Revision bb4eafc6
b/pithos/api/short_url.py | ||
---|---|---|
1 |
# Copyright (C) 2009 by Michael Fogleman |
|
2 |
# |
|
3 |
# Permission is hereby granted, free of charge, to any person obtaining a copy |
|
4 |
# of this software and associated documentation files (the "Software"), to deal |
|
5 |
# in the Software without restriction, including without limitation the rights |
|
6 |
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
|
7 |
# copies of the Software, and to permit persons to whom the Software is |
|
8 |
# furnished to do so, subject to the following conditions: |
|
9 |
# |
|
10 |
# The above copyright notice and this permission notice shall be included in |
|
11 |
# all copies or substantial portions of the Software. |
|
12 |
# |
|
13 |
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
|
14 |
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
|
15 |
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
|
16 |
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
|
17 |
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
|
18 |
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
|
19 |
# THE SOFTWARE. |
|
20 |
|
|
21 |
''' |
|
22 |
Short URL Generator |
|
23 |
=================== |
|
24 |
|
|
25 |
Python implementation for generating Tiny URL- and bit.ly-like URLs. |
|
26 |
|
|
27 |
A bit-shuffling approach is used to avoid generating consecutive, predictable |
|
28 |
URLs. However, the algorithm is deterministic and will guarantee that no |
|
29 |
collisions will occur. |
|
30 |
|
|
31 |
The URL alphabet is fully customizable and may contain any number of |
|
32 |
characters. By default, digits and lower-case letters are used, with |
|
33 |
some removed to avoid confusion between characters like o, O and 0. The |
|
34 |
default alphabet is shuffled and has a prime number of characters to further |
|
35 |
improve the results of the algorithm. |
|
36 |
|
|
37 |
The block size specifies how many bits will be shuffled. The lower BLOCK_SIZE |
|
38 |
bits are reversed. Any bits higher than BLOCK_SIZE will remain as is. |
|
39 |
BLOCK_SIZE of 0 will leave all bits unaffected and the algorithm will simply |
|
40 |
be converting your integer to a different base. |
|
41 |
|
|
42 |
The intended use is that incrementing, consecutive integers will be used as |
|
43 |
keys to generate the short URLs. For example, when creating a new URL, the |
|
44 |
unique integer ID assigned by a database could be used to generate the URL |
|
45 |
by using this module. Or a simple counter may be used. As long as the same |
|
46 |
integer is not used twice, the same short URL will not be generated twice. |
|
47 |
|
|
48 |
The module supports both encoding and decoding of URLs. The min_length |
|
49 |
parameter allows you to pad the URL if you want it to be a specific length. |
|
50 |
|
|
51 |
Sample Usage: |
|
52 |
|
|
53 |
>>> import short_url |
|
54 |
>>> url = short_url.encode_url(12) |
|
55 |
>>> print url |
|
56 |
LhKA |
|
57 |
>>> key = short_url.decode_url(url) |
|
58 |
>>> print key |
|
59 |
12 |
|
60 |
|
|
61 |
Use the functions in the top-level of the module to use the default encoder. |
|
62 |
Otherwise, you may create your own UrlEncoder object and use its encode_url |
|
63 |
and decode_url methods. |
|
64 |
|
|
65 |
Author: Michael Fogleman |
|
66 |
License: MIT |
|
67 |
Link: http://code.activestate.com/recipes/576918/ |
|
68 |
''' |
|
69 |
|
|
70 |
DEFAULT_ALPHABET = 'mn6j2c4rv8bpygw95z7hsdaetxuk3fq' |
|
71 |
DEFAULT_BLOCK_SIZE = 24 |
|
72 |
MIN_LENGTH = 5 |
|
73 |
|
|
74 |
class UrlEncoder(object): |
|
75 |
def __init__(self, alphabet=DEFAULT_ALPHABET, block_size=DEFAULT_BLOCK_SIZE): |
|
76 |
self.alphabet = alphabet |
|
77 |
self.block_size = block_size |
|
78 |
self.mask = (1 << block_size) - 1 |
|
79 |
self.mapping = range(block_size) |
|
80 |
self.mapping.reverse() |
|
81 |
def encode_url(self, n, min_length=MIN_LENGTH): |
|
82 |
return self.enbase(self.encode(n), min_length) |
|
83 |
def decode_url(self, n): |
|
84 |
return self.decode(self.debase(n)) |
|
85 |
def encode(self, n): |
|
86 |
return (n & ~self.mask) | self._encode(n & self.mask) |
|
87 |
def _encode(self, n): |
|
88 |
result = 0 |
|
89 |
for i, b in enumerate(self.mapping): |
|
90 |
if n & (1 << i): |
|
91 |
result |= (1 << b) |
|
92 |
return result |
|
93 |
def decode(self, n): |
|
94 |
return (n & ~self.mask) | self._decode(n & self.mask) |
|
95 |
def _decode(self, n): |
|
96 |
result = 0 |
|
97 |
for i, b in enumerate(self.mapping): |
|
98 |
if n & (1 << b): |
|
99 |
result |= (1 << i) |
|
100 |
return result |
|
101 |
def enbase(self, x, min_length=MIN_LENGTH): |
|
102 |
result = self._enbase(x) |
|
103 |
padding = self.alphabet[0] * (min_length - len(result)) |
|
104 |
return '%s%s' % (padding, result) |
|
105 |
def _enbase(self, x): |
|
106 |
n = len(self.alphabet) |
|
107 |
if x < n: |
|
108 |
return self.alphabet[x] |
|
109 |
return self._enbase(x / n) + self.alphabet[x % n] |
|
110 |
def debase(self, x): |
|
111 |
n = len(self.alphabet) |
|
112 |
result = 0 |
|
113 |
for i, c in enumerate(reversed(x)): |
|
114 |
result += self.alphabet.index(c) * (n ** i) |
|
115 |
return result |
|
116 |
|
|
117 |
DEFAULT_ENCODER = UrlEncoder() |
|
118 |
|
|
119 |
def encode(n): |
|
120 |
return DEFAULT_ENCODER.encode(n) |
|
121 |
|
|
122 |
def decode(n): |
|
123 |
return DEFAULT_ENCODER.decode(n) |
|
124 |
|
|
125 |
def enbase(n, min_length=MIN_LENGTH): |
|
126 |
return DEFAULT_ENCODER.enbase(n, min_length) |
|
127 |
|
|
128 |
def debase(n): |
|
129 |
return DEFAULT_ENCODER.debase(n) |
|
130 |
|
|
131 |
def encode_url(n, min_length=MIN_LENGTH): |
|
132 |
return DEFAULT_ENCODER.encode_url(n, min_length) |
|
133 |
|
|
134 |
def decode_url(n): |
|
135 |
return DEFAULT_ENCODER.decode_url(n) |
|
136 |
|
|
137 |
if __name__ == '__main__': |
|
138 |
for a in range(0, 200000, 37): |
|
139 |
b = encode(a) |
|
140 |
c = enbase(b) |
|
141 |
d = debase(c) |
|
142 |
e = decode(d) |
|
143 |
assert a == e |
|
144 |
assert b == d |
|
145 |
c = (' ' * (7 - len(c))) + c |
|
146 |
print '%6d %12d %s %12d %6d' % (a, b, c, d, e) |
b/pithos/api/util.py | ||
---|---|---|
50 | 50 |
from pithos.api.faults import (Fault, NotModified, BadRequest, Unauthorized, Forbidden, ItemNotFound, |
51 | 51 |
Conflict, LengthRequired, PreconditionFailed, RequestEntityTooLarge, |
52 | 52 |
RangeNotSatisfiable, ServiceUnavailable) |
53 |
from pithos.api.short_url import encode_url |
|
53 | 54 |
from pithos.backends import connect_backend |
54 | 55 |
from pithos.backends.base import NotAllowedError, QuotaError |
55 | 56 |
|
... | ... | |
241 | 242 |
def update_public_meta(public, meta): |
242 | 243 |
if not public: |
243 | 244 |
return |
244 |
meta['X-Object-Public'] = public
|
|
245 |
meta['X-Object-Public'] = '/public/' + encode_url(public)
|
|
245 | 246 |
|
246 | 247 |
def validate_modification_preconditions(request, meta): |
247 | 248 |
"""Check that the modified timestamp conforms with the preconditions set.""" |
b/pithos/backends/base.py | ||
---|---|---|
388 | 388 |
return |
389 | 389 |
|
390 | 390 |
def get_object_public(self, user, account, container, name): |
391 |
"""Return the public URL of the object if applicable.
|
|
391 |
"""Return the public id of the object if applicable.
|
|
392 | 392 |
|
393 | 393 |
Raises: |
394 | 394 |
NotAllowedError: Operation not permitted |
... | ... | |
513 | 513 |
""" |
514 | 514 |
return [] |
515 | 515 |
|
516 |
def get_public(self, user, public): |
|
517 |
"""Return the (account, container, name) for the public id given. |
|
518 |
|
|
519 |
Raises: |
|
520 |
NotAllowedError: Operation not permitted |
|
521 |
|
|
522 |
NameError: Public id does not exist |
|
523 |
""" |
|
524 |
return None |
|
525 |
|
|
516 | 526 |
def get_block(self, hash): |
517 | 527 |
"""Return a block's data. |
518 | 528 |
|
b/pithos/backends/lib/sqlalchemy/permissions.py | ||
---|---|---|
79 | 79 |
def access_check(self, path, access, member): |
80 | 80 |
"""Return true if the member has this access to the path.""" |
81 | 81 |
|
82 |
if access == READ and self.public_check(path):
|
|
82 |
if access == READ and self.public_get(path) is not None:
|
|
83 | 83 |
return True |
84 | 84 |
|
85 | 85 |
r = self.xfeature_inherit(path) |
b/pithos/backends/lib/sqlalchemy/public.py | ||
---|---|---|
32 | 32 |
# or implied, of GRNET S.A. |
33 | 33 |
|
34 | 34 |
from dbworker import DBWorker |
35 |
from sqlalchemy import Table, Column, String, MetaData |
|
35 |
from sqlalchemy import Table, Column, String, Integer, MetaData
|
|
36 | 36 |
from sqlalchemy.sql import select |
37 |
from sqlalchemy.schema import Index |
|
38 |
|
|
37 | 39 |
|
38 | 40 |
class Public(DBWorker): |
39 | 41 |
"""Paths can be marked as public.""" |
... | ... | |
42 | 44 |
DBWorker.__init__(self, **params) |
43 | 45 |
metadata = MetaData() |
44 | 46 |
columns=[] |
45 |
columns.append(Column('path', String(2048), index=True)) |
|
46 |
self.public = Table('public', metadata, *columns, mysql_engine='InnoDB') |
|
47 |
columns.append(Column('public_id', Integer, primary_key=True)) |
|
48 |
columns.append(Column('path', String(2048))) |
|
49 |
self.public = Table('public', metadata, *columns, mysql_engine='InnoDB', sqlite_autoincrement=True) |
|
50 |
# place an index on path |
|
51 |
Index('idx_public_path', self.public.c.path) |
|
47 | 52 |
metadata.create_all(self.engine) |
48 | 53 |
|
49 |
|
|
50 | 54 |
def public_set(self, path): |
51 | 55 |
s = self.public.select() |
52 | 56 |
s = s.where(self.public.c.path == path) |
... | ... | |
63 | 67 |
r = self.conn.execute(s) |
64 | 68 |
r.close() |
65 | 69 |
|
66 |
def public_check(self, path): |
|
67 |
s = select([self.public.c.path], self.public.c.path == path) |
|
70 |
def public_get(self, path): |
|
71 |
s = select([self.public.c.public_id], self.public.c.path == path) |
|
72 |
r = self.conn.execute(s) |
|
73 |
row = r.fetchone() |
|
74 |
r.close() |
|
75 |
if row: |
|
76 |
return row[0] |
|
77 |
return None |
|
78 |
|
|
79 |
def public_path(self, public): |
|
80 |
s = select([self.public.c.path], self.public.c.public_id == public) |
|
68 | 81 |
r = self.conn.execute(s) |
69 |
l = r.fetchone()
|
|
82 |
row = r.fetchone()
|
|
70 | 83 |
r.close() |
71 |
return bool(l) |
|
84 |
if row: |
|
85 |
return row[0] |
|
86 |
return None |
b/pithos/backends/lib/sqlite/permissions.py | ||
---|---|---|
76 | 76 |
def access_check(self, path, access, member): |
77 | 77 |
"""Return true if the member has this access to the path.""" |
78 | 78 |
|
79 |
if access == READ and self.public_check(path):
|
|
79 |
if access == READ and self.public_get(path) is not None:
|
|
80 | 80 |
return True |
81 | 81 |
|
82 | 82 |
r = self.xfeature_inherit(path) |
b/pithos/backends/lib/sqlite/public.py | ||
---|---|---|
42 | 42 |
execute = self.execute |
43 | 43 |
|
44 | 44 |
execute(""" create table if not exists public |
45 |
( path text primary key ) """) |
|
45 |
( public_id integer primary key autoincrement, |
|
46 |
path text ) """) |
|
47 |
execute(""" create unique index if not exists idx_public_path |
|
48 |
on public(path) """) |
|
46 | 49 |
|
47 | 50 |
def public_set(self, path): |
48 | 51 |
q = "insert or ignore into public (path) values (?)" |
... | ... | |
52 | 55 |
q = "delete from public where path = ?" |
53 | 56 |
self.execute(q, (path,)) |
54 | 57 |
|
55 |
def public_check(self, path):
|
|
56 |
q = "select 1 from public where path = ?"
|
|
58 |
def public_get(self, path):
|
|
59 |
q = "select public_id from public where path = ?"
|
|
57 | 60 |
self.execute(q, (path,)) |
58 |
return bool(self.fetchone()) |
|
61 |
row = self.fetchone() |
|
62 |
if row: |
|
63 |
return row[0] |
|
64 |
return None |
|
65 |
|
|
66 |
def public_path(self, public): |
|
67 |
q = "select path from public where public_id = ?" |
|
68 |
self.execute(q, (public,)) |
|
69 |
row = self.fetchone() |
|
70 |
if row: |
|
71 |
return row[0] |
|
72 |
return None |
b/pithos/backends/modular.py | ||
---|---|---|
44 | 44 |
|
45 | 45 |
inf = float('inf') |
46 | 46 |
|
47 |
ULTIMATE_ANSWER = 42 |
|
48 |
|
|
47 | 49 |
|
48 | 50 |
logger = logging.getLogger(__name__) |
49 | 51 |
|
... | ... | |
492 | 494 |
|
493 | 495 |
@backend_method |
494 | 496 |
def get_object_public(self, user, account, container, name): |
495 |
"""Return the public URL of the object if applicable."""
|
|
497 |
"""Return the public id of the object if applicable."""
|
|
496 | 498 |
|
497 | 499 |
logger.debug("get_object_public: %s %s %s", account, container, name) |
498 | 500 |
self._can_read(user, account, container, name) |
499 | 501 |
path = self._lookup_object(account, container, name)[0] |
500 |
if self.permissions.public_check(path): |
|
501 |
return '/public/' + path |
|
502 |
return None |
|
502 |
p = self.permissions.public_get(path) |
|
503 |
if p is not None: |
|
504 |
p += ULTIMATE_ANSWER |
|
505 |
return p |
|
503 | 506 |
|
504 | 507 |
@backend_method |
505 | 508 |
def update_object_public(self, user, account, container, name, public): |
... | ... | |
651 | 654 |
versions = self.node.node_get_versions(node) |
652 | 655 |
return [[x[self.SERIAL], x[self.MTIME]] for x in versions if x[self.CLUSTER] != CLUSTER_DELETED] |
653 | 656 |
|
657 |
@backend_method |
|
658 |
def get_public(self, user, public): |
|
659 |
"""Return the (account, container, name) for the public id given.""" |
|
660 |
logger.debug("get_public: %s", public) |
|
661 |
if public is None or public < ULTIMATE_ANSWER: |
|
662 |
raise NameError |
|
663 |
path = self.permissions.public_path(public - ULTIMATE_ANSWER) |
|
664 |
account, container, name = path.split('/', 2) |
|
665 |
self._can_read(user, account, container, name) |
|
666 |
return (account, container, name) |
|
667 |
|
|
654 | 668 |
@backend_method(autocommit=0) |
655 | 669 |
def get_block(self, hash): |
656 | 670 |
"""Return a block's data.""" |
b/pithos/public/functions.py | ||
---|---|---|
39 | 39 |
from pithos.api.util import (put_object_headers, update_manifest_meta, |
40 | 40 |
validate_modification_preconditions, validate_matching_preconditions, |
41 | 41 |
object_data_response, api_method) |
42 |
from pithos.api.short_url import decode_url |
|
42 | 43 |
|
43 | 44 |
|
44 | 45 |
logger = logging.getLogger(__name__) |
45 | 46 |
|
46 | 47 |
|
47 |
def object_demux(request, v_account, v_container, v_object):
|
|
48 |
def public_demux(request, v_public):
|
|
48 | 49 |
if request.method == 'HEAD': |
49 |
return object_meta(request, v_account, v_container, v_object)
|
|
50 |
return public_meta(request, v_public)
|
|
50 | 51 |
elif request.method == 'GET': |
51 |
return object_read(request, v_account, v_container, v_object)
|
|
52 |
return public_read(request, v_public)
|
|
52 | 53 |
else: |
53 | 54 |
return method_not_allowed(request) |
54 | 55 |
|
55 | 56 |
@api_method('HEAD', user_required=False) |
56 |
def object_meta(request, v_account, v_container, v_object):
|
|
57 |
def public_meta(request, v_public):
|
|
57 | 58 |
# Normal Response Codes: 204 |
58 | 59 |
# Error Response Codes: serviceUnavailable (503), |
59 | 60 |
# itemNotFound (404), |
60 | 61 |
# badRequest (400) |
61 | 62 |
|
62 | 63 |
try: |
64 |
v_account, v_container, v_object = request.backend.get_public(request.user_uniq, |
|
65 |
decode_url(v_public)) |
|
63 | 66 |
meta = request.backend.get_object_meta(request.user_uniq, v_account, |
64 |
v_container, v_object) |
|
67 |
v_container, v_object)
|
|
65 | 68 |
public = request.backend.get_object_public(request.user_uniq, v_account, |
66 | 69 |
v_container, v_object) |
67 | 70 |
except: |
... | ... | |
76 | 79 |
return response |
77 | 80 |
|
78 | 81 |
@api_method('GET', user_required=False) |
79 |
def object_read(request, v_account, v_container, v_object):
|
|
82 |
def public_read(request, v_public):
|
|
80 | 83 |
# Normal Response Codes: 200, 206 |
81 | 84 |
# Error Response Codes: serviceUnavailable (503), |
82 | 85 |
# rangeNotSatisfiable (416), |
... | ... | |
86 | 89 |
# notModified (304) |
87 | 90 |
|
88 | 91 |
try: |
92 |
v_account, v_container, v_object = request.backend.get_public(request.user_uniq, |
|
93 |
decode_url(v_public)) |
|
89 | 94 |
meta = request.backend.get_object_meta(request.user_uniq, v_account, |
90 |
v_container, v_object) |
|
95 |
v_container, v_object)
|
|
91 | 96 |
public = request.backend.get_object_public(request.user_uniq, v_account, |
92 | 97 |
v_container, v_object) |
93 | 98 |
except: |
... | ... | |
133 | 138 |
except: |
134 | 139 |
raise ItemNotFound('Object does not exist') |
135 | 140 |
|
141 |
if 'Content-Disposition' not in meta: |
|
142 |
name = v_object.rstrip('/').split('/')[-1] |
|
143 |
if not name: |
|
144 |
name = v_public |
|
145 |
meta['Content-Disposition'] = 'attachment; filename=%s' % (name,) |
|
146 |
|
|
136 | 147 |
return object_data_response(request, sizes, hashmaps, meta, True) |
137 | 148 |
|
138 | 149 |
@api_method(user_required=False) |
b/pithos/public/urls.py | ||
---|---|---|
33 | 33 |
|
34 | 34 |
from django.conf.urls.defaults import * |
35 | 35 |
|
36 |
# TODO: This only works when in this order. |
|
37 | 36 |
urlpatterns = patterns('pithos.public.functions', |
38 | 37 |
(r'^$', 'method_not_allowed'), |
39 |
(r'^(?P<v_account>.+?)/(?P<v_container>.+?)/(?P<v_object>.+?)$', 'object_demux'), |
|
40 |
(r'^(?P<v_account>.+?)/(?P<v_container>.+?)/?$', 'method_not_allowed'), |
|
41 |
(r'^(?P<v_account>.+?)/?$', 'method_not_allowed') |
|
38 |
(r'^(?P<v_public>.+?)/?$', 'public_demux') |
|
42 | 39 |
) |
Also available in: Unified diff