Revision b1dadd0e
| b/snf-pithos-app/README | ||
|---|---|---|
| 42 | 42 |
PITHOS_BACKEND_QUEUE_EXCHANGE pithos |
| 43 | 43 |
PITHOS_BACKEND_QUOTA 50 GB (50 * 1024 ** 3) Default user quota |
| 44 | 44 |
PITHOS_BACKEND_VERSIONING auto Default versioning policy for containers |
| 45 |
PITHOS_BACKEND_FREE_VERSIONING True Default versioning debit policy (default free) |
|
| 45 | 46 |
PITHOS_UPDATE_MD5 True Update object checksums when using hashmaps |
| 46 | 47 |
PITHOS_SERVICE_TOKEN '' Service token acquired by the identity provider (astakos) |
| 47 | 48 |
=============================== ================================================ ============================================================ |
| b/snf-pithos-app/conf/20-snf-pithos-app-settings.conf | ||
|---|---|---|
| 28 | 28 |
# Default setting for new accounts. |
| 29 | 29 |
#PITHOS_BACKEND_QUOTA = 50 * 1024 * 1024 * 1024 |
| 30 | 30 |
#PITHOS_BACKEND_VERSIONING = 'auto' |
| 31 |
#PITHOS_BACKEND_FREE_VERSIONING = True |
|
| 31 | 32 |
|
| 32 | 33 |
# Disable if checksums are not required or are computed asynchronously by an external process. |
| 33 | 34 |
#PITHOS_UPDATE_MD5 = True |
| b/snf-pithos-app/pithos/api/dispatch.py | ||
|---|---|---|
| 1 |
from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION, |
|
| 2 |
BACKEND_BLOCK_MODULE, BACKEND_BLOCK_PATH, |
|
| 3 |
BACKEND_BLOCK_UMASK, |
|
| 4 |
BACKEND_QUEUE_MODULE, BACKEND_QUEUE_HOSTS, |
|
| 5 |
BACKEND_QUEUE_EXCHANGE, |
|
| 6 |
BACKEND_QUOTA, BACKEND_VERSIONING) |
|
| 7 |
from pithos.backends import connect_backend |
|
| 8 |
from pithos.api.util import hashmap_md5 |
|
| 1 |
# from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION, |
|
| 2 |
# BACKEND_BLOCK_MODULE, BACKEND_BLOCK_PATH, |
|
| 3 |
# BACKEND_BLOCK_UMASK, |
|
| 4 |
# BACKEND_QUEUE_MODULE, BACKEND_QUEUE_HOSTS, |
|
| 5 |
# BACKEND_QUEUE_EXCHANGE, |
|
| 6 |
# BACKEND_QUOTA, BACKEND_VERSIONING, |
|
| 7 |
# BACKEND_FREE_VERSIONING) |
|
| 8 |
from pithos.api.settings import (BACKEND_QUOTA, BACKEND_VERSIONING) |
|
| 9 |
|
|
| 10 |
#from pithos.backends import connect_backend |
|
| 11 |
from pithos.api.util import hashmap_md5, get_backend |
|
| 9 | 12 |
|
| 10 | 13 |
from django.core.mail import send_mail |
| 11 | 14 |
from django.utils.translation import ugettext as _ |
| ... | ... | |
| 20 | 23 |
if m['resource'] != 'object' or m['details']['action'] != 'object update': |
| 21 | 24 |
return |
| 22 | 25 |
|
| 23 |
backend = connect_backend(db_module=BACKEND_DB_MODULE, |
|
| 24 |
db_connection=BACKEND_DB_CONNECTION, |
|
| 25 |
block_module=BACKEND_BLOCK_MODULE, |
|
| 26 |
block_path=BACKEND_BLOCK_PATH, |
|
| 27 |
block_umask=BACKEND_BLOCK_UMASK, |
|
| 28 |
queue_module=BACKEND_QUEUE_MODULE, |
|
| 29 |
queue_hosts=BACKEND_QUEUE_HOSTS, |
|
| 30 |
queue_exchange=BACKEND_QUEUE_EXHANGE) |
|
| 26 |
# backend = connect_backend(db_module=BACKEND_DB_MODULE, |
|
| 27 |
# db_connection=BACKEND_DB_CONNECTION, |
|
| 28 |
# block_module=BACKEND_BLOCK_MODULE, |
|
| 29 |
# block_path=BACKEND_BLOCK_PATH, |
|
| 30 |
# block_umask=BACKEND_BLOCK_UMASK, |
|
| 31 |
# queue_module=BACKEND_QUEUE_MODULE, |
|
| 32 |
# queue_hosts=BACKEND_QUEUE_HOSTS, |
|
| 33 |
# queue_exchange=BACKEND_QUEUE_EXHANGE |
|
| 34 |
# free_versioning=BACKEND_FREE_VERSIONING) |
|
| 35 |
backend = get_backend() |
|
| 31 | 36 |
backend.default_policy['quota'] = BACKEND_QUOTA |
| 32 | 37 |
backend.default_policy['versioning'] = BACKEND_VERSIONING |
| 33 | 38 |
|
| b/snf-pithos-app/pithos/api/management/commands/storagequota.py | ||
|---|---|---|
| 35 | 35 |
|
| 36 | 36 |
from django.core.management.base import BaseCommand, CommandError |
| 37 | 37 |
|
| 38 |
from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION, |
|
| 39 |
BACKEND_BLOCK_MODULE, BACKEND_BLOCK_PATH, |
|
| 40 |
BACKEND_BLOCK_UMASK, |
|
| 41 |
BACKEND_QUEUE_MODULE, BACKEND_QUEUE_HOSTS, |
|
| 42 |
BACKEND_QUEUE_EXCHANGE, |
|
| 43 |
BACKEND_QUOTA, BACKEND_VERSIONING) |
|
| 44 |
from pithos.backends import connect_backend |
|
| 38 |
from pithos.api.settings import (BACKEND_QUOTA, BACKEND_VERSIONING) |
|
| 45 | 39 |
|
| 40 |
from pithos.api.util import get_backend |
|
| 46 | 41 |
|
| 47 | 42 |
class Command(BaseCommand): |
| 48 | 43 |
args = "<user>" |
| ... | ... | |
| 67 | 62 |
except ValueError: |
| 68 | 63 |
raise CommandError("Invalid quota")
|
| 69 | 64 |
|
| 70 |
backend = connect_backend(db_module=BACKEND_DB_MODULE, |
|
| 71 |
db_connection=BACKEND_DB_CONNECTION, |
|
| 72 |
block_module=BACKEND_BLOCK_MODULE, |
|
| 73 |
block_path=BACKEND_BLOCK_PATH, |
|
| 74 |
block_umask=BACKEND_BLOCK_UMASK, |
|
| 75 |
queue_module=BACKEND_QUEUE_MODULE, |
|
| 76 |
queue_connection=BACKEND_QUEUE_HOSTS, |
|
| 77 |
queue_exchange=BACKEND_QUEUE_EXCHANGE) |
|
| 65 |
backend = get_backend() |
|
| 78 | 66 |
backend.default_policy['quota'] = BACKEND_QUOTA |
| 79 | 67 |
backend.default_policy['versioning'] = BACKEND_VERSIONING |
| 80 | 68 |
if quota is not None: |
| b/snf-pithos-app/pithos/api/settings.py | ||
|---|---|---|
| 44 | 44 |
BACKEND_QUOTA = getattr( |
| 45 | 45 |
settings, 'PITHOS_BACKEND_QUOTA', 50 * 1024 * 1024 * 1024) |
| 46 | 46 |
BACKEND_VERSIONING = getattr(settings, 'PITHOS_BACKEND_VERSIONING', 'auto') |
| 47 |
BACKEND_FREE_VERSIONING = getattr(settings, 'PITHOS_BACKEND_FREE_VERSIONING', True) |
|
| 47 | 48 |
|
| 48 | 49 |
# Update object checksums when using hashmaps. |
| 49 | 50 |
UPDATE_MD5 = getattr(settings, 'PITHOS_UPDATE_MD5', True) |
| b/snf-pithos-app/pithos/api/util.py | ||
|---|---|---|
| 62 | 62 |
BACKEND_QUEUE_MODULE, BACKEND_QUEUE_HOSTS, |
| 63 | 63 |
BACKEND_QUEUE_EXCHANGE, |
| 64 | 64 |
BACKEND_QUOTA, BACKEND_VERSIONING, |
| 65 |
BACKEND_FREE_VERSIONING, |
|
| 65 | 66 |
AUTHENTICATION_URL, AUTHENTICATION_USERS, |
| 66 | 67 |
SERVICE_TOKEN, COOKIE_NAME) |
| 67 | 68 |
|
| ... | ... | |
| 860 | 861 |
block_path=BACKEND_BLOCK_PATH, |
| 861 | 862 |
block_umask=BACKEND_BLOCK_UMASK, |
| 862 | 863 |
queue_module=BACKEND_QUEUE_MODULE, |
| 863 |
queue_connection=BACKEND_QUEUE_CONNECTION) |
|
| 864 |
queue_hosts=BACKEND_QUEUE_HOSTS, |
|
| 865 |
queue_exchange=BACKEND_QUEUE_EXCHANGE, |
|
| 866 |
free_versioning=BACKEND_FREE_VERSIONING) |
|
| 864 | 867 |
|
| 865 | 868 |
|
| 866 | 869 |
def get_backend(): |
| b/snf-pithos-backend/pithos/backends/modular.py | ||
|---|---|---|
| 129 | 129 |
def __init__(self, db_module=None, db_connection=None, |
| 130 | 130 |
block_module=None, block_path=None, block_umask=None, |
| 131 | 131 |
queue_module=None, queue_hosts=None, |
| 132 |
queue_exchange=None): |
|
| 132 |
queue_exchange=None, |
|
| 133 |
free_versioning=True): |
|
| 133 | 134 |
db_module = db_module or DEFAULT_DB_MODULE |
| 134 | 135 |
db_connection = db_connection or DEFAULT_DB_CONNECTION |
| 135 | 136 |
block_module = block_module or DEFAULT_BLOCK_MODULE |
| ... | ... | |
| 138 | 139 |
#queue_module = queue_module or DEFAULT_QUEUE_MODULE |
| 139 | 140 |
#queue_hosts = queue_hosts or DEFAULT_QUEUE_HOSTS |
| 140 | 141 |
#queue_exchange = queue_exchange or DEFAULT_QUEUE_EXCHANGE |
| 141 |
|
|
| 142 |
|
|
| 142 | 143 |
self.hash_algorithm = 'sha256' |
| 143 | 144 |
self.block_size = 4 * 1024 * 1024 # 4MB |
| 145 |
self.free_versioning = free_versioning |
|
| 144 | 146 |
|
| 145 | 147 |
self.default_policy = {'quota': DEFAULT_QUOTA,
|
| 146 | 148 |
'versioning': DEFAULT_VERSIONING} |
| ... | ... | |
| 171 | 173 |
if queue_module and queue_hosts: |
| 172 | 174 |
self.queue_module = load_module(queue_module) |
| 173 | 175 |
params = {'hosts': queue_hosts,
|
| 174 |
'exchange': queue_exchange,
|
|
| 176 |
'exchange': queue_exchange,
|
|
| 175 | 177 |
'client_id': QUEUE_CLIENT_ID} |
| 176 | 178 |
self.queue = self.queue_module.Queue(**params) |
| 177 | 179 |
else: |
| ... | ... | |
| 489 | 491 |
self.store.map_delete(h) |
| 490 | 492 |
self.node.node_purge_children(node, until, CLUSTER_DELETED) |
| 491 | 493 |
self._report_size_change(user, account, -size, |
| 492 |
{'action':'container purge', 'path': path,
|
|
| 493 |
'versions': ','.join(str(i) for i in serials)})
|
|
| 494 |
{'action':'container purge', 'path': path,
|
|
| 495 |
'versions': ','.join(str(i) for i in serials)})
|
|
| 494 | 496 |
return |
| 495 | 497 |
|
| 496 | 498 |
if not delimiter: |
| ... | ... | |
| 503 | 505 |
self.node.node_purge_children(node, inf, CLUSTER_DELETED) |
| 504 | 506 |
self.node.node_remove(node) |
| 505 | 507 |
self._report_size_change(user, account, -size, |
| 506 |
{'action': 'container delete',
|
|
| 507 |
'path': path,
|
|
| 508 |
'versions': ','.join(str(i) for i in serials)})
|
|
| 508 |
{'action': 'container delete',
|
|
| 509 |
'path': path,
|
|
| 510 |
'versions': ','.join(str(i) for i in serials)})
|
|
| 509 | 511 |
else: |
| 510 |
# remove only contents
|
|
| 512 |
# remove only contents |
|
| 511 | 513 |
src_names = self._list_objects_no_limit(user, account, container, prefix='', delimiter=None, virtual=False, domain=None, keys=[], shared=False, until=None, size_range=None, all_props=True, public=False) |
| 512 | 514 |
paths = [] |
| 513 | 515 |
for t in src_names: |
| ... | ... | |
| 516 | 518 |
src_version_id, dest_version_id = self._put_version_duplicate(user, node, size=0, type='', hash=None, checksum='', cluster=CLUSTER_DELETED) |
| 517 | 519 |
del_size = self._apply_versioning( |
| 518 | 520 |
account, container, src_version_id) |
| 519 |
if del_size: |
|
| 520 |
self._report_size_change(user, account, -del_size, |
|
| 521 |
{'action': 'object delete',
|
|
| 522 |
'path': path, |
|
| 523 |
'versions': ','.join([str(dest_version_id)])}) |
|
| 521 |
self._report_size_change( |
|
| 522 |
user, account, -del_size, {
|
|
| 523 |
'action': 'object delete', |
|
| 524 |
'path': path, |
|
| 525 |
'versions': ','.join([str(dest_version_id)]) |
|
| 526 |
} |
|
| 527 |
) |
|
| 524 | 528 |
self._report_object_change( |
| 525 | 529 |
user, account, path, details={'action': 'object delete'})
|
| 526 | 530 |
paths.append(path) |
| ... | ... | |
| 800 | 804 |
self._put_metadata_duplicate( |
| 801 | 805 |
src_version_id, dest_version_id, domain, meta, replace_meta) |
| 802 | 806 |
|
| 803 |
# Check quota. |
|
| 804 | 807 |
del_size = self._apply_versioning(account, container, pre_version_id) |
| 805 | 808 |
size_delta = size - del_size |
| 806 |
if size_delta > 0: |
|
| 807 |
account_quota = long(self._get_policy(account_node)['quota']) |
|
| 808 |
container_quota = long(self._get_policy(container_node)['quota']) |
|
| 809 |
if (account_quota > 0 and self._get_statistics(account_node)[1] + size_delta > account_quota) or \ |
|
| 810 |
(container_quota > 0 and self._get_statistics(container_node)[1] + size_delta > container_quota): |
|
| 811 |
# This must be executed in a transaction, so the version is never created if it fails. |
|
| 812 |
raise QuotaError |
|
| 809 |
# # Check quota. |
|
| 810 |
# if size_delta > 0: |
|
| 811 |
# account_quota = long(self._get_policy(account_node)['quota']) |
|
| 812 |
# container_quota = long(self._get_policy(container_node)['quota']) |
|
| 813 |
# if (account_quota > 0 and self._get_statistics(account_node)[1] + size_delta > account_quota) or \ |
|
| 814 |
# (container_quota > 0 and self._get_statistics(container_node)[1] + size_delta > container_quota): |
|
| 815 |
# # This must be executed in a transaction, so the version is never created if it fails. |
|
| 816 |
# raise QuotaError |
|
| 813 | 817 |
self._report_size_change(user, account, size_delta, |
| 814 |
{'action': 'object update', 'path': path,
|
|
| 815 |
'versions': ','.join([str(dest_version_id)])})
|
|
| 818 |
{'action': 'object update', 'path': path,
|
|
| 819 |
'versions': ','.join([str(dest_version_id)])})
|
|
| 816 | 820 |
|
| 817 | 821 |
if permissions is not None: |
| 818 | 822 |
self.permissions.access_set(path, permissions) |
| ... | ... | |
| 943 | 947 |
except NameError: |
| 944 | 948 |
self.permissions.access_clear(path) |
| 945 | 949 |
self._report_size_change(user, account, -size, |
| 946 |
{'action': 'object purge', 'path': path,
|
|
| 947 |
'versions': ','.join(str(i) for i in serials)})
|
|
| 950 |
{'action': 'object purge', 'path': path,
|
|
| 951 |
'versions': ','.join(str(i) for i in serials)})
|
|
| 948 | 952 |
return |
| 949 | 953 |
|
| 950 | 954 |
path, node = self._lookup_object(account, container, name) |
| 951 | 955 |
src_version_id, dest_version_id = self._put_version_duplicate(user, node, size=0, type='', hash=None, checksum='', cluster=CLUSTER_DELETED) |
| 952 | 956 |
del_size = self._apply_versioning(account, container, src_version_id) |
| 953 |
if del_size: |
|
| 954 |
self._report_size_change(user, account, -del_size, |
|
| 955 |
{'action': 'object delete', 'path': path,
|
|
| 956 |
'versions': ','.join([str(dest_version_id)])}) |
|
| 957 |
self._report_size_change(user, account, -del_size, |
|
| 958 |
{'action': 'object delete', 'path': path,
|
|
| 959 |
'versions': ','.join([str(dest_version_id)])}) |
|
| 957 | 960 |
self._report_object_change( |
| 958 | 961 |
user, account, path, details={'action': 'object delete'})
|
| 959 | 962 |
self.permissions.access_clear(path) |
| ... | ... | |
| 968 | 971 |
src_version_id, dest_version_id = self._put_version_duplicate(user, node, size=0, type='', hash=None, checksum='', cluster=CLUSTER_DELETED) |
| 969 | 972 |
del_size = self._apply_versioning( |
| 970 | 973 |
account, container, src_version_id) |
| 971 |
if del_size: |
|
| 972 |
self._report_size_change(user, account, -del_size, |
|
| 973 |
{'action': 'object delete',
|
|
| 974 |
'path': path, |
|
| 975 |
'versions': ','.join([str(dest_version_id)])}) |
|
| 974 |
self._report_size_change(user, account, -del_size, |
|
| 975 |
{'action': 'object delete',
|
|
| 976 |
'path': path, |
|
| 977 |
'versions': ','.join([str(dest_version_id)])}) |
|
| 976 | 978 |
self._report_object_change( |
| 977 | 979 |
user, account, path, details={'action': 'object delete'})
|
| 978 | 980 |
paths.append(path) |
| ... | ... | |
| 1230 | 1232 |
logger.debug( |
| 1231 | 1233 |
"_report_size_change: %s %s %s %s", user, account, size, details) |
| 1232 | 1234 |
self.messages.append((QUEUE_MESSAGE_KEY_PREFIX % ('resource.diskspace',),
|
| 1233 |
account, QUEUE_INSTANCE_ID, 'diskspace',
|
|
| 1234 |
float(size), details))
|
|
| 1235 |
account, QUEUE_INSTANCE_ID, 'diskspace',
|
|
| 1236 |
float(size), details))
|
|
| 1235 | 1237 |
|
| 1236 | 1238 |
def _report_object_change(self, user, account, path, details={}):
|
| 1237 | 1239 |
details.update({'user': user})
|
| 1238 | 1240 |
logger.debug("_report_object_change: %s %s %s %s", user,
|
| 1239 | 1241 |
account, path, details) |
| 1240 | 1242 |
self.messages.append((QUEUE_MESSAGE_KEY_PREFIX % ('object',),
|
| 1241 |
account, QUEUE_INSTANCE_ID, 'object', path, details))
|
|
| 1243 |
account, QUEUE_INSTANCE_ID, 'object', path, details))
|
|
| 1242 | 1244 |
|
| 1243 | 1245 |
def _report_sharing_change(self, user, account, path, details={}):
|
| 1244 | 1246 |
logger.debug("_report_permissions_change: %s %s %s %s",
|
| 1245 | 1247 |
user, account, path, details) |
| 1246 | 1248 |
details.update({'user': user})
|
| 1247 | 1249 |
self.messages.append((QUEUE_MESSAGE_KEY_PREFIX % ('sharing',),
|
| 1248 |
account, QUEUE_INSTANCE_ID, 'sharing', path, details))
|
|
| 1250 |
account, QUEUE_INSTANCE_ID, 'sharing', path, details))
|
|
| 1249 | 1251 |
|
| 1250 | 1252 |
# Policy functions. |
| 1251 | 1253 |
|
| ... | ... | |
| 1285 | 1287 |
return 0 |
| 1286 | 1288 |
path, node = self._lookup_container(account, container) |
| 1287 | 1289 |
versioning = self._get_policy(node)['versioning'] |
| 1288 |
if versioning != 'auto': |
|
| 1290 |
if versioning != 'auto' or self.free_versioning:
|
|
| 1289 | 1291 |
hash, size = self.node.version_remove(version_id) |
| 1290 | 1292 |
self.store.map_delete(hash) |
| 1291 | 1293 |
return size |
| b/snf-pithos-backend/pithos/backends/util.py | ||
|---|---|---|
| 43 | 43 |
class PithosBackendPool(ObjectPool): |
| 44 | 44 |
def __init__(self, size=None, db_module=None, db_connection=None, |
| 45 | 45 |
block_module=None, block_path=None, block_umask=None, |
| 46 |
queue_module=None, queue_connection=None): |
|
| 46 |
queue_module=None, queue_hosts=None, |
|
| 47 |
queue_exchange=None, |
|
| 48 |
free_versioning=True): |
|
| 47 | 49 |
super(PithosBackendPool, self).__init__(size=size) |
| 48 | 50 |
self.db_module = db_module |
| 49 | 51 |
self.db_connection = db_connection |
| ... | ... | |
| 51 | 53 |
self.block_path = block_path |
| 52 | 54 |
self.block_umask = block_umask |
| 53 | 55 |
self.queue_module = queue_module |
| 54 |
self.queue_connection = queue_connection |
|
| 55 |
|
|
| 56 |
self.queue_hosts = queue_hosts |
|
| 57 |
self.queue_exchange = queue_exchange |
|
| 58 |
self.free_versioning = free_versioning |
|
| 59 |
|
|
| 56 | 60 |
def _pool_create(self): |
| 57 | 61 |
backend = connect_backend(db_module=self.db_module, |
| 58 | 62 |
db_connection=self.db_connection, |
| ... | ... | |
| 60 | 64 |
block_path=self.block_path, |
| 61 | 65 |
block_umask=self.block_umask, |
| 62 | 66 |
queue_module=self.queue_module, |
| 63 |
queue_connection=self.queue_connection) |
|
| 67 |
queue_hosts=self.queue_hosts, |
|
| 68 |
queue_exchange=self.queue_exchange, |
|
| 69 |
free_versioning=self.free_versioning) |
|
| 64 | 70 |
|
| 65 | 71 |
backend._real_close = backend.close |
| 66 | 72 |
backend.close = instancemethod(_pooled_backend_close, backend, |
Also available in: Unified diff