Revision 4a105ce2
| b/contrib/migrate-db | ||
|---|---|---|
| 168 | 168 |
if public: |
| 169 | 169 |
self.backend.permissions.public_set( |
| 170 | 170 |
object, |
| 171 |
self.backend.public_url_min_length,
|
|
| 171 |
self.backend.public_url_security,
|
|
| 172 | 172 |
self.backend.public_url_alphabet |
| 173 | 173 |
) |
| 174 | 174 |
#set object's permissions |
| b/snf-pithos-app/README | ||
|---|---|---|
| 57 | 57 |
PITHOS_USE_QUOTAHOLDER True Enable quotaholder |
| 58 | 58 |
PITHOS_QUOTAHOLDER_URL '' Quotaholder URL |
| 59 | 59 |
PITHOS_QUOTAHOLDER_TOKEN '' Quotaholder token |
| 60 |
PITHOS_PUBLIC_URL_MIN_LENGTH 8 Public URL minimun length
|
|
| 61 |
PITHOS_PUBLIC_URL_ALPHABET '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' Public URL alphabet
|
|
| 60 |
PITHOS_PUBLIC_URL_SECURITY 16 How many random bytes to use for constructing the URL of Pithos public files
|
|
| 61 |
PITHOS_PUBLIC_URL_ALPHABET '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' The alphabet to use for constructing the URL of Pithos public files
|
|
| 62 | 62 |
=============================== ================================================================ ============================================================ |
| 63 | 63 |
|
| 64 | 64 |
To update checksums asynchronously, enable the queue, install snf-pithos-tools and use ``pithos-dispatcher``:: |
| b/snf-pithos-app/conf/20-snf-pithos-app-settings.conf | ||
|---|---|---|
| 56 | 56 |
# |
| 57 | 57 |
#PITHOS_QUOTAHOLDER_POOLSIZE = 200 |
| 58 | 58 |
# |
| 59 |
# Set public url length |
|
| 60 |
#PITHOS_PUBLIC_URL_MIN_LENGTH = 8 |
|
| 59 |
# How many random bytes to use for constructing the URL of Pithos public files. |
|
| 60 |
# Lower values mean accidental reuse of (discarded) URLs is more probable. |
|
| 61 |
# Note: the active public URLs will always be unique. |
|
| 62 |
# Only the old and discarded URLs can ever be reused. |
|
| 63 |
# Higher values mean more safety and longer URLs |
|
| 64 |
# |
|
| 65 |
#PITHOS_PUBLIC_URL_SECURITY = 16 |
|
| b/snf-pithos-app/pithos/api/settings.py | ||
|---|---|---|
| 66 | 66 |
QUOTAHOLDER_TOKEN = getattr(settings, 'PITHOS_QUOTAHOLDER_TOKEN', '') |
| 67 | 67 |
QUOTAHOLDER_POOLSIZE = getattr(settings, 'PITHOS_QUOTAHOLDER_POOLSIZE', 200) |
| 68 | 68 |
|
| 69 |
# Set public url length and alphabet |
|
| 70 |
PUBLIC_URL_MIN_LENGTH = getattr(settings, 'PITHOS_PUBLIC_URL_MIN_LENGTH', 8) |
|
| 69 |
# Set how many random bytes to use for constructing the URL of Pithos public files |
|
| 70 |
PUBLIC_URL_SECURITY = getattr(settings, 'PITHOS_PUBLIC_URL_SECURITY', 16) |
|
| 71 |
# Set the alphabet to use for constructing the URL of Pithos public files |
|
| 71 | 72 |
PUBLIC_URL_ALPHABET = getattr( |
| 72 | 73 |
settings, |
| 73 | 74 |
'PITHOS_PUBLIC_URL_ALPHABET', |
| b/snf-pithos-app/pithos/api/swiss_army/__init__.py | ||
|---|---|---|
| 196 | 196 |
fullpath = '/'.join([dest_account, dest_container, dest_name]) |
| 197 | 197 |
self.backend.permissions.public_set( |
| 198 | 198 |
fullpath, |
| 199 |
self.backend.public_url_min_length,
|
|
| 199 |
self.backend.public_url_security,
|
|
| 200 | 200 |
self.backend.public_url_alphabet |
| 201 | 201 |
) |
| 202 | 202 |
|
| b/snf-pithos-app/pithos/api/tests.py | ||
|---|---|---|
| 77 | 77 |
account, account, container, object |
| 78 | 78 |
) |
| 79 | 79 |
self.assertTrue(public != None) |
| 80 |
self.assertTrue(len(public) >= settings.PUBLIC_URL_MIN_LENGTH)
|
|
| 80 |
self.assertTrue(len(public) >= settings.PUBLIC_URL_SECURITY)
|
|
| 81 | 81 |
self.assertTrue(set(public) <= set(settings.PUBLIC_URL_ALPHABET)) |
| 82 | 82 |
self.assertEqual( |
| 83 | 83 |
self.backend.get_public('$$account$$', public),
|
| ... | ... | |
| 103 | 103 |
|
| 104 | 104 |
self.backend.permissions.public_set( |
| 105 | 105 |
'account/container/object', |
| 106 |
self.backend.public_url_min_length,
|
|
| 106 |
self.backend.public_url_security,
|
|
| 107 | 107 |
self.backend.public_url_alphabet |
| 108 | 108 |
) |
| 109 | 109 |
self.assert_public_object('account', 'container', 'object')
|
| ... | ... | |
| 120 | 120 |
) |
| 121 | 121 |
self.backend.permissions.public_set( |
| 122 | 122 |
'account/container/object', |
| 123 |
self.backend.public_url_min_length,
|
|
| 123 |
self.backend.public_url_security,
|
|
| 124 | 124 |
self.backend.public_url_alphabet |
| 125 | 125 |
) |
| 126 | 126 |
public = self.assert_public_object('account', 'container', 'object')
|
| 127 | 127 |
|
| 128 | 128 |
self.backend.permissions.public_set( |
| 129 | 129 |
'account/container/object', |
| 130 |
self.backend.public_url_min_length,
|
|
| 130 |
self.backend.public_url_security,
|
|
| 131 | 131 |
self.backend.public_url_alphabet |
| 132 | 132 |
) |
| 133 | 133 |
public2 = self.assert_public_object('account', 'container', 'object')
|
| ... | ... | |
| 146 | 146 |
) |
| 147 | 147 |
self.backend.permissions.public_set( |
| 148 | 148 |
'account/container/object', |
| 149 |
self.backend.public_url_min_length,
|
|
| 149 |
self.backend.public_url_security,
|
|
| 150 | 150 |
self.backend.public_url_alphabet |
| 151 | 151 |
) |
| 152 | 152 |
public = self.assert_public_object('account', 'container', 'object')
|
| ... | ... | |
| 156 | 156 |
|
| 157 | 157 |
self.backend.permissions.public_set( |
| 158 | 158 |
'account/container/object', |
| 159 |
self.backend.public_url_min_length,
|
|
| 159 |
self.backend.public_url_security,
|
|
| 160 | 160 |
self.backend.public_url_alphabet |
| 161 | 161 |
) |
| 162 | 162 |
public3 = self.assert_public_object('account', 'container', 'object')
|
| ... | ... | |
| 225 | 225 |
|
| 226 | 226 |
self.backend.permissions.public_set( |
| 227 | 227 |
'account/container/object', |
| 228 |
self.backend.public_url_min_length,
|
|
| 228 |
self.backend.public_url_security,
|
|
| 229 | 229 |
self.backend.public_url_alphabet |
| 230 | 230 |
) |
| 231 | 231 |
self.assert_public_object('account', 'container', 'object')
|
| ... | ... | |
| 253 | 253 |
|
| 254 | 254 |
self.backend.permissions.public_set( |
| 255 | 255 |
'account/container/object', |
| 256 |
self.backend.public_url_min_length,
|
|
| 256 |
self.backend.public_url_security,
|
|
| 257 | 257 |
self.backend.public_url_alphabet |
| 258 | 258 |
) |
| 259 | 259 |
public = self.assert_public_object('account', 'container', 'object')
|
| b/snf-pithos-app/pithos/api/util.py | ||
|---|---|---|
| 68 | 68 |
COOKIE_NAME, USER_CATALOG_URL, |
| 69 | 69 |
RADOS_STORAGE, RADOS_POOL_BLOCKS, |
| 70 | 70 |
RADOS_POOL_MAPS, TRANSLATE_UUIDS, |
| 71 |
PUBLIC_URL_MIN_LENGTH,
|
|
| 71 |
PUBLIC_URL_SECURITY,
|
|
| 72 | 72 |
PUBLIC_URL_ALPHABET) |
| 73 | 73 |
from pithos.backends import connect_backend |
| 74 | 74 |
from pithos.backends.base import (NotAllowedError, QuotaError, ItemNotExists, |
| ... | ... | |
| 985 | 985 |
quotaholder_client_poolsize=QUOTAHOLDER_POOLSIZE, |
| 986 | 986 |
free_versioning=BACKEND_FREE_VERSIONING, |
| 987 | 987 |
block_params=BLOCK_PARAMS, |
| 988 |
public_url_min_length=PUBLIC_URL_MIN_LENGTH,
|
|
| 988 |
public_url_security=PUBLIC_URL_SECURITY,
|
|
| 989 | 989 |
public_url_alphabet=PUBLIC_URL_ALPHABET) |
| 990 | 990 |
|
| 991 | 991 |
def get_backend(): |
| b/snf-pithos-backend/pithos/backends/lib/sqlalchemy/public.py | ||
|---|---|---|
| 37 | 37 |
from sqlalchemy.schema import Index |
| 38 | 38 |
from sqlalchemy.exc import NoSuchTableError |
| 39 | 39 |
|
| 40 |
from pithos.backends.random_word import get_word |
|
| 40 |
from pithos.backends.random_word import get_random_word |
|
| 41 |
|
|
| 42 |
import logging |
|
| 43 |
|
|
| 44 |
logger = logging.getLogger(__name__) |
|
| 41 | 45 |
|
| 42 | 46 |
def create_tables(engine): |
| 43 | 47 |
metadata = MetaData() |
| ... | ... | |
| 68 | 72 |
tables = create_tables(self.engine) |
| 69 | 73 |
map(lambda t: self.__setattr__(t.name, t), tables) |
| 70 | 74 |
|
| 71 |
def get_unique_url(self, serial, public_url_min_length, public_url_alphabet):
|
|
| 72 |
l = public_url_min_length
|
|
| 75 |
def get_unique_url(self, public_security, public_url_alphabet):
|
|
| 76 |
l = public_security
|
|
| 73 | 77 |
while 1: |
| 74 |
candidate = get_word(serial, length=l, alphabet=public_url_alphabet)
|
|
| 78 |
candidate = get_random_word(length=l, alphabet=public_url_alphabet)
|
|
| 75 | 79 |
if self.public_path(candidate) is None: |
| 76 | 80 |
return candidate |
| 77 | 81 |
l +=1 |
| 78 | 82 |
|
| 79 |
def public_set(self, path, public_url_min_length, public_url_alphabet):
|
|
| 83 |
def public_set(self, path, public_security, public_url_alphabet):
|
|
| 80 | 84 |
s = select([self.public.c.public_id]) |
| 81 | 85 |
s = s.where(self.public.c.path == path) |
| 82 | 86 |
r = self.conn.execute(s) |
| ... | ... | |
| 84 | 88 |
r.close() |
| 85 | 89 |
|
| 86 | 90 |
if not row: |
| 91 |
url = self.get_unique_url( |
|
| 92 |
public_security, public_url_alphabet |
|
| 93 |
) |
|
| 87 | 94 |
s = self.public.insert() |
| 88 |
s = s.values(path=path, active=True) |
|
| 95 |
s = s.values(path=path, active=True, url=url)
|
|
| 89 | 96 |
r = self.conn.execute(s) |
| 90 |
serial = r.inserted_primary_key[0] |
|
| 91 | 97 |
r.close() |
| 92 |
|
|
| 93 |
url = self.get_unique_url( |
|
| 94 |
serial, public_url_min_length, public_url_alphabet |
|
| 95 |
) |
|
| 96 |
s = self.public.update().where(self.public.c.public_id == serial) |
|
| 97 |
s = s.values(url=url) |
|
| 98 |
self.conn.execute(s).close() |
|
| 98 |
logger.info('Public url: %s set for path: %s' % (url, path))
|
|
| 99 | 99 |
|
| 100 | 100 |
def public_unset(self, path): |
| 101 | 101 |
s = self.public.delete() |
| 102 | 102 |
s = s.where(self.public.c.path == path) |
| 103 | 103 |
self.conn.execute(s).close() |
| 104 |
logger.info('Public url unset for path: %s' % (path))
|
|
| 104 | 105 |
|
| 105 | 106 |
def public_unset_bulk(self, paths): |
| 106 | 107 |
if not paths: |
| b/snf-pithos-backend/pithos/backends/lib/sqlite/public.py | ||
|---|---|---|
| 33 | 33 |
|
| 34 | 34 |
from dbworker import DBWorker |
| 35 | 35 |
|
| 36 |
from pithos.backends.random_word import get_word |
|
| 36 |
from pithos.backends.random_word import get_random_word |
|
| 37 |
|
|
| 38 |
import logging |
|
| 39 |
|
|
| 40 |
logger = logging.getLogger(__name__) |
|
| 37 | 41 |
|
| 38 | 42 |
class Public(DBWorker): |
| 39 | 43 |
"""Paths can be marked as public.""" |
| ... | ... | |
| 52 | 56 |
execute(""" create unique index if not exists idx_public_url
|
| 53 | 57 |
on public(url) """) |
| 54 | 58 |
|
| 55 |
def get_unique_url(self, serial, public_url_min_length, public_url_alphabet):
|
|
| 56 |
l = public_url_min_length
|
|
| 59 |
def get_unique_url(self, public_url_security, public_url_alphabet):
|
|
| 60 |
l = public_url_security
|
|
| 57 | 61 |
while 1: |
| 58 |
candidate = get_word(serial, length=l, alphabet=public_url_alphabet)
|
|
| 62 |
candidate = get_random_word(length=l, alphabet=public_url_alphabet)
|
|
| 59 | 63 |
if self.public_path(candidate) is None: |
| 60 | 64 |
return candidate |
| 61 | 65 |
l +=1 |
| 62 | 66 |
|
| 63 |
def public_set(self, path, public_url_min_length, public_url_alphabet):
|
|
| 67 |
def public_set(self, path, public_url_security, public_url_alphabet):
|
|
| 64 | 68 |
q = "select public_id from public where path = ?" |
| 65 | 69 |
self.execute(q, (path,)) |
| 66 | 70 |
row = self.fetchone() |
| 67 | 71 |
|
| 68 | 72 |
if not row: |
| 69 |
q = "insert into public(path, active) values(?, ?)" |
|
| 70 |
serial = self.execute(q, (path, active)).lastrowid |
|
| 71 | 73 |
url = self.get_unique_url( |
| 72 |
serial, public_url_min_length, public_url_alphabet
|
|
| 74 |
public_url_security, public_url_alphabet
|
|
| 73 | 75 |
) |
| 74 |
q = "update public set url=url where public_id = ?" |
|
| 75 |
self.execute(q, (serial,)) |
|
| 76 |
q = "insert into public(path, active, url) values(?, 1, ?)" |
|
| 77 |
self.execute(q, (path, url)) |
|
| 78 |
logger.info('Public url: %s set for path: %s' % (url, path))
|
|
| 76 | 79 |
|
| 77 | 80 |
def public_unset(self, path): |
| 78 | 81 |
q = "delete from public where path = ?" |
| 79 | 82 |
self.execute(q, (path,)) |
| 83 |
logger.info('Public url unset for path: %s' % (path))
|
|
| 80 | 84 |
|
| 81 | 85 |
def public_unset_bulk(self, paths): |
| 82 | 86 |
placeholders = ','.join('?' for path in paths)
|
| 83 |
q = "delete from public where path in (%s)" |
|
| 87 |
q = "delete from public where path in (%s)" % placeholders
|
|
| 84 | 88 |
self.execute(q, paths) |
| 85 | 89 |
|
| 86 | 90 |
def public_get(self, path): |
| b/snf-pithos-backend/pithos/backends/modular.py | ||
|---|---|---|
| 84 | 84 |
DEFAULT_BLOCK_PARAMS = { 'mappool': None, 'blockpool': None }
|
| 85 | 85 |
#DEFAULT_QUEUE_HOSTS = '[amqp://guest:guest@localhost:5672]' |
| 86 | 86 |
#DEFAULT_QUEUE_EXCHANGE = 'pithos' |
| 87 |
DEFAULT_ALPHABET = ('0123456789'
|
|
| 88 |
'abcdefghijklmnopqrstuvwxyz' |
|
| 89 |
'ABCDEFGHIJKLMNOPQRSTUVWXYZ') |
|
| 90 |
DEFAULT_MIN_LENGTH = 8
|
|
| 87 |
DEFAULT_PUBLIC_URL_ALPHABET = ('0123456789'
|
|
| 88 |
'abcdefghijklmnopqrstuvwxyz'
|
|
| 89 |
'ABCDEFGHIJKLMNOPQRSTUVWXYZ')
|
|
| 90 |
DEFAULT_PUBLIC_URL_SECURITY = 8
|
|
| 91 | 91 |
|
| 92 | 92 |
QUEUE_MESSAGE_KEY_PREFIX = 'pithos.%s' |
| 93 | 93 |
QUEUE_CLIENT_ID = 'pithos' |
| ... | ... | |
| 153 | 153 |
quotaholder_url=None, quotaholder_token=None, |
| 154 | 154 |
quotaholder_client_poolsize=None, |
| 155 | 155 |
free_versioning=True, block_params=None, |
| 156 |
public_url_min_length=None,
|
|
| 156 |
public_url_security=None,
|
|
| 157 | 157 |
public_url_alphabet=None): |
| 158 | 158 |
db_module = db_module or DEFAULT_DB_MODULE |
| 159 | 159 |
db_connection = db_connection or DEFAULT_DB_CONNECTION |
| ... | ... | |
| 167 | 167 |
#queue_hosts = queue_hosts or DEFAULT_QUEUE_HOSTS |
| 168 | 168 |
#queue_exchange = queue_exchange or DEFAULT_QUEUE_EXCHANGE |
| 169 | 169 |
|
| 170 |
self.public_url_min_length = public_url_min_length or DEFAULT_MIN_LENGTH
|
|
| 171 |
self.public_url_alphabet = public_url_alphabet or DEFAULT_ALPHABET |
|
| 170 |
self.public_url_security = public_url_security or DEFAULT_PUBLIC_URL_SECURITY
|
|
| 171 |
self.public_url_alphabet = public_url_alphabet or DEFAULT_PUBLIC_URL_ALPHABET
|
|
| 172 | 172 |
|
| 173 | 173 |
self.hash_algorithm = 'sha256' |
| 174 | 174 |
self.block_size = 4 * 1024 * 1024 # 4MB |
| ... | ... | |
| 838 | 838 |
self.permissions.public_unset(path) |
| 839 | 839 |
else: |
| 840 | 840 |
self.permissions.public_set( |
| 841 |
path, self.public_url_min_length, self.public_url_alphabet
|
|
| 841 |
path, self.public_url_security, self.public_url_alphabet
|
|
| 842 | 842 |
) |
| 843 | 843 |
|
| 844 | 844 |
@backend_method |
| b/snf-pithos-backend/pithos/backends/random_word.py | ||
|---|---|---|
| 31 | 31 |
# interpreted as representing official policies, either expressed |
| 32 | 32 |
# or implied, of GRNET S.A. |
| 33 | 33 |
|
| 34 |
DEFAULT_ALPHABET = ('0123456789'
|
|
| 35 |
'abcdefghijklmnopqrstuvwxyz' |
|
| 36 |
'ABCDEFGHIJKLMNOPQRSTUVWXYZ') |
|
| 37 |
DEFAULT_MIN_LENGTH = 8 |
|
| 34 |
import random |
|
| 38 | 35 |
|
| 39 |
from random import randrange |
|
| 40 |
from math import log |
|
| 36 |
getrandbits = random.SystemRandom().getrandbits |
|
| 41 | 37 |
|
| 42 |
def get_random_word(length=DEFAULT_MIN_LENGTH, alphabet=DEFAULT_ALPHABET): |
|
| 43 |
alphabet_length = len(alphabet) |
|
| 44 |
word = ''.join(alphabet[randrange(alphabet_length)] |
|
| 45 |
for _ in xrange(length)) |
|
| 46 |
return word |
|
| 38 |
DEFAULT_ALPHABET = ("0123456789"
|
|
| 39 |
"ABCDEFGHIJKLMNOPQRSTUVWXYZ" |
|
| 40 |
"abcdefghijklmnopqrstuvwxyz") |
|
| 47 | 41 |
|
| 48 |
def encode_serial(serial, alphabet=DEFAULT_ALPHABET): |
|
| 49 |
i = 0 |
|
| 42 |
def get_random_word(length, alphabet=DEFAULT_ALPHABET): |
|
| 43 |
remainder = getrandbits(length * 8) |
|
| 44 |
return encode_word(remainder, alphabet=alphabet) |
|
| 45 |
|
|
| 46 |
def encode_word(number, alphabet=DEFAULT_ALPHABET): |
|
| 50 | 47 |
base = len(alphabet) |
| 51 |
quotient = int(serial) |
|
| 52 | 48 |
digits = [] |
| 53 | 49 |
append = digits.append |
| 54 |
while quotient > 0: |
|
| 50 |
quotient = number |
|
| 51 |
while True: |
|
| 55 | 52 |
quotient, remainder = divmod(quotient, base) |
| 56 | 53 |
append(alphabet[remainder]) |
| 57 |
word = ''.join(reversed(digits))
|
|
| 58 |
return word
|
|
| 54 |
if quotient <= 0:
|
|
| 55 |
break
|
|
| 59 | 56 |
|
| 60 |
def get_word(serial, length=DEFAULT_MIN_LENGTH, alphabet=DEFAULT_ALPHABET): |
|
| 61 |
word = encode_serial(serial, alphabet) |
|
| 62 |
word += get_random_word(length, alphabet) |
|
| 63 |
return word |
|
| 57 |
return ''.join(digits) |
|
| b/snf-pithos-backend/pithos/backends/util.py | ||
|---|---|---|
| 49 | 49 |
quotaholder_url=None, quotaholder_token=None, |
| 50 | 50 |
quotaholder_client_poolsize=None, |
| 51 | 51 |
block_params=None, |
| 52 |
public_url_min_length=None,
|
|
| 52 |
public_url_security=None,
|
|
| 53 | 53 |
public_url_alphabet=None): |
| 54 | 54 |
super(PithosBackendPool, self).__init__(size=size) |
| 55 | 55 |
self.db_module = db_module |
| ... | ... | |
| 66 | 66 |
self.quotaholder_token = quotaholder_token |
| 67 | 67 |
self.quotaholder_client_poolsize = quotaholder_client_poolsize |
| 68 | 68 |
self.free_versioning = free_versioning |
| 69 |
self.public_url_min_length=public_url_min_length
|
|
| 69 |
self.public_url_security=public_url_security
|
|
| 70 | 70 |
self.public_url_alphabet=public_url_alphabet |
| 71 | 71 |
|
| 72 | 72 |
def _pool_create(self): |
| ... | ... | |
| 85 | 85 |
quotaholder_token=self.quotaholder_token, |
| 86 | 86 |
quotaholder_client_poolsize=self.quotaholder_client_poolsize, |
| 87 | 87 |
free_versioning=self.free_versioning, |
| 88 |
public_url_min_length=self.public_url_min_length,
|
|
| 88 |
public_url_security=self.public_url_security,
|
|
| 89 | 89 |
public_url_alphabet=self.public_url_alphabet) |
| 90 | 90 |
|
| 91 | 91 |
backend._real_close = backend.close |
Also available in: Unified diff