Revision 16f2673e
b/Changelog | ||
---|---|---|
14 | 14 |
|
15 | 15 |
Released: UNRELEASED |
16 | 16 |
|
17 |
* Incorporate quotaholder in astakos |
|
18 |
* obsolete settings: |
|
19 |
PITHOS_USE_QUOTAHOLDER, PITHOS_QUOTAHOLDER_URL, PITHOS_QUOTAHOLDER_TOKEN |
|
20 |
* new settins: |
|
21 |
PITHOS_ASTAKOSCLIENT_POOLSIZE |
|
22 |
|
|
17 | 23 |
Synnefo-wide |
18 | 24 |
------------ |
19 | 25 |
* Create 'snf_django' Python package to hold common code for all Synnefo |
b/snf-cyclades-app/synnefo/plankton/backend.py | ||
---|---|---|
76 | 76 |
_pithos_backend_pool = \ |
77 | 77 |
PithosBackendPool( |
78 | 78 |
POOL_SIZE, |
79 |
quotaholder_enabled=True, |
|
80 |
quotaholder_url=settings.ASTAKOS_URL, |
|
81 |
quotaholder_token=settings.CYCLADES_ASTAKOS_SERVICE_TOKEN, |
|
82 |
quotaholder_client_poolsize=settings.CYCLADES_QUOTAHOLDER_POOLSIZE, |
|
79 |
astakos_url=settings.ASTAKOS_URL, |
|
80 |
service_token=settings.CYCLADES_ASTAKOS_SERVICE_TOKEN, |
|
81 |
astakosclient_poolsize=settings.CYCLADES_QUOTAHOLDER_POOLSIZE, |
|
83 | 82 |
db_connection=settings.BACKEND_DB_CONNECTION, |
84 | 83 |
block_path=settings.BACKEND_BLOCK_PATH) |
85 | 84 |
|
b/snf-pithos-app/README | ||
---|---|---|
31 | 31 |
Name Default value Description |
32 | 32 |
=============================== ================================================================ ============================================================ |
33 | 33 |
ASTAKOS_URL \https://<astakos.host>/ Astakos Authentication URL |
34 |
PITHOS_ASTAKOSCLIENT_POOLSIZE 200 The size of the http pool for the astakos client |
|
34 | 35 |
PITHOS_ASTAKOS_COOKIE_NAME _pithos2_a Cookie name to retrieve fallback token |
35 | 36 |
PITHOS_BACKEND_DB_MODULE pithos.backends.lib.sqlalchemy |
36 | 37 |
PITHOS_BACKEND_DB_CONNECTION sqlite:////tmp/pithos-backend.db SQLAlchemy database connection string |
... | ... | |
55 | 56 |
PITHOS_USER_CATALOG_URL \https://<astakos.host>/user_catalogs/ Astakos User Catalog URL |
56 | 57 |
PITHOS_USER_FEEDBACK_URL \https://<astakos.host>/feedback/ Astakos User Feedback URL |
57 | 58 |
PITHOS_USER_LOGIN_URL \https://<astakos.host>/login/ Astakos User Login URL |
58 |
PITHOS_USE_QUOTAHOLDER True Enable quotaholder |
|
59 |
PITHOS_QUOTAHOLDER_URL '' Quotaholder URL |
|
60 |
PITHOS_QUOTAHOLDER_TOKEN '' Quotaholder token |
|
61 | 59 |
PITHOS_PUBLIC_URL_SECURITY 16 How many random bytes to use for constructing the URL of Pithos public files |
62 | 60 |
PITHOS_PUBLIC_URL_ALPHABET '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' The alphabet to use for constructing the URL of Pithos public files |
63 | 61 |
=============================== ================================================================ ============================================================ |
b/snf-pithos-app/conf/20-snf-pithos-app-settings.conf | ||
---|---|---|
45 | 45 |
#PITHOS_USER_FEEDBACK_URL = 'https://accounts.example.synnefo.org/feedback/' |
46 | 46 |
#PITHOS_USER_LOGIN_URL = 'https://accounts.example.synnefo.org/login/' |
47 | 47 |
|
48 |
# Enable and configure quotaholder |
|
49 |
#PITHOS_USE_QUOTAHOLDER = False |
|
50 |
#PITHOS_QUOTAHOLDER_URL = '' |
|
51 |
#PITHOS_QUOTAHOLDER_TOKEN = '' |
|
52 |
# |
|
53 |
# Tune the size of the http pool for the quotaholder client. |
|
48 |
# Tune the size of the http pool for the astakos client. |
|
54 | 49 |
# It limits the maximum number of quota changing requests |
55 | 50 |
# that pithos can serve. Extra requests will be blocked |
56 | 51 |
# until another has completed. |
57 | 52 |
# |
58 |
#PITHOS_QUOTAHOLDER_POOLSIZE = 200
|
|
53 |
#PITHOS_ASTAKOSCLIENT_POOLSIZE = 200
|
|
59 | 54 |
# |
60 | 55 |
# How many random bytes to use for constructing the URL of Pithos public files. |
61 | 56 |
# Lower values mean accidental reuse of (discarded) URLs is more probable. |
b/snf-pithos-app/pithos/api/management/commands/reconcile-commissions-pithos.py | ||
---|---|---|
56 | 56 |
def handle_noargs(self, **options): |
57 | 57 |
b = get_backend() |
58 | 58 |
try: |
59 |
pending_commissions = b.quotaholder.get_pending_commissions(
|
|
60 |
token=b.quotaholder_token)
|
|
59 |
pending_commissions = b.astakosclient.get_pending_commissions(
|
|
60 |
token=b.service_token)
|
|
61 | 61 |
|
62 | 62 |
if pending_commissions: |
63 | 63 |
self.stdout.write( |
... | ... | |
68 | 68 |
return |
69 | 69 |
|
70 | 70 |
if options['fix']: |
71 |
to_accept = b.quotaholder_serials.lookup(pending_commissions)
|
|
71 |
to_accept = b.commision_serials.lookup(pending_commissions)
|
|
72 | 72 |
to_reject = list(set(pending_commissions) - set(to_accept)) |
73 |
response = b.quotaholder.resolve_commissions(
|
|
74 |
token=b.quotaholder_token,
|
|
73 |
response = b.astakosclient.resolve_commissions(
|
|
74 |
token=b.service_token,
|
|
75 | 75 |
accept_serials=to_accept, |
76 | 76 |
reject_serials=to_reject |
77 | 77 |
) |
... | ... | |
84 | 84 |
for i in failed: |
85 | 85 |
self.stdout.write('%s\n' % i) |
86 | 86 |
|
87 |
b.quotaholder_serials.delete_many(accepted)
|
|
87 |
b.commision_serials.delete_many(accepted)
|
|
88 | 88 |
except Exception, e: |
89 | 89 |
logger.exception(e) |
90 | 90 |
raise CommandError(e) |
b/snf-pithos-app/pithos/api/management/commands/reconcile-resources-pithos.py | ||
---|---|---|
99 | 99 |
size = backend.node.node_account_usage(node, CLUSTER_NORMAL) |
100 | 100 |
db_usage[path] = size or 0 |
101 | 101 |
|
102 |
result = backend.quotaholder.service_get_quotas(
|
|
103 |
backend.quotaholder_token,
|
|
102 |
result = backend.astakosclient.service_get_quotas(
|
|
103 |
backend.service_token,
|
|
104 | 104 |
) |
105 | 105 |
|
106 | 106 |
qh_usage = {} |
... | ... | |
149 | 149 |
request['auto_accept'] = True |
150 | 150 |
request['provisions'] = provisions |
151 | 151 |
try: |
152 |
serial = backend.quotaholder.issue_commission(
|
|
153 |
backend.quotaholder_token, request
|
|
152 |
serial = backend.astakosclient.issue_commission(
|
|
153 |
backend.service_token, request
|
|
154 | 154 |
) |
155 | 155 |
except AstakosClientException, e: |
156 | 156 |
self.stdout.write(e.details) |
157 | 157 |
else: |
158 |
backend.quotaholder_serials.insert_many([serial])
|
|
158 |
backend.commission_serials.insert_many([serial])
|
|
159 | 159 |
elif options['list'] and table: |
160 | 160 |
output_format = options["output_format"] |
161 | 161 |
if output_format != "json" and not options["headers"]: |
b/snf-pithos-app/pithos/api/settings.py | ||
---|---|---|
3 | 3 |
|
4 | 4 |
ASTAKOS_URL = getattr(settings, "ASTAKOS_URL", |
5 | 5 |
"https://accounts.example.synnefo.org/") |
6 |
ASTAKOSCLIENT_POOLSIZE = getattr(settings, 'PITHOS_ASTAKOSCLIENT_POOLSIZE', 200) |
|
6 | 7 |
|
7 | 8 |
COOKIE_NAME = getattr(settings, 'PITHOS_ASTAKOS_COOKIE_NAME', '_pithos2_a') |
8 | 9 |
|
... | ... | |
64 | 65 |
USER_LOGIN_URL = getattr(settings, 'PITHOS_USER_LOGIN_URL', |
65 | 66 |
'https://accounts.example.synnefo.org/login/') |
66 | 67 |
|
67 |
# Set the quota holder component URI |
|
68 |
USE_QUOTAHOLDER = getattr(settings, 'PITHOS_USE_QUOTAHOLDER', False) |
|
69 |
QUOTAHOLDER_URL = getattr(settings, 'PITHOS_QUOTAHOLDER_URL', '') |
|
70 |
QUOTAHOLDER_TOKEN = getattr(settings, 'PITHOS_QUOTAHOLDER_TOKEN', '') |
|
71 |
QUOTAHOLDER_POOLSIZE = getattr(settings, 'PITHOS_QUOTAHOLDER_POOLSIZE', 200) |
|
72 |
|
|
73 | 68 |
# Set how many random bytes to use for constructing the URL of Pithos public files |
74 | 69 |
PUBLIC_URL_SECURITY = getattr(settings, 'PITHOS_PUBLIC_URL_SECURITY', 16) |
75 | 70 |
# Set the alphabet to use for constructing the URL of Pithos public files |
b/snf-pithos-app/pithos/api/util.py | ||
---|---|---|
55 | 55 |
BACKEND_BLOCK_UMASK, |
56 | 56 |
BACKEND_QUEUE_MODULE, BACKEND_QUEUE_HOSTS, |
57 | 57 |
BACKEND_QUEUE_EXCHANGE, |
58 |
QUOTAHOLDER_POOLSIZE,
|
|
58 |
ASTAKOSCLIENT_POOLSIZE,
|
|
59 | 59 |
SERVICE_TOKEN, |
60 | 60 |
ASTAKOS_URL, |
61 | 61 |
BACKEND_ACCOUNT_QUOTA, BACKEND_CONTAINER_QUOTA, |
... | ... | |
953 | 953 |
queue_module=BACKEND_QUEUE_MODULE, |
954 | 954 |
queue_hosts=BACKEND_QUEUE_HOSTS, |
955 | 955 |
queue_exchange=BACKEND_QUEUE_EXCHANGE, |
956 |
quotaholder_enabled=True, |
|
957 |
quotaholder_url=ASTAKOS_URL, |
|
958 |
quotaholder_token=SERVICE_TOKEN, |
|
959 |
quotaholder_client_poolsize=QUOTAHOLDER_POOLSIZE, |
|
956 |
astakos_url=ASTAKOS_URL, |
|
957 |
service_token=SERVICE_TOKEN, |
|
958 |
astakosclient_poolsize=ASTAKOSCLIENT_POOLSIZE, |
|
960 | 959 |
free_versioning=BACKEND_FREE_VERSIONING, |
961 | 960 |
block_params=BLOCK_PARAMS, |
962 | 961 |
public_url_security=PUBLIC_URL_SECURITY, |
b/snf-pithos-backend/pithos/backends/modular.py | ||
---|---|---|
141 | 141 |
return ret |
142 | 142 |
except: |
143 | 143 |
if serials: |
144 |
self.quotaholder.resolve_commissions(
|
|
145 |
token=self.quotaholder_token,
|
|
144 |
self.astakosclient.resolve_commissions(
|
|
145 |
token=self.service_token,
|
|
146 | 146 |
accept_serials=[], |
147 | 147 |
reject_serials=serials) |
148 | 148 |
self.wrapper.rollback() |
... | ... | |
159 | 159 |
def __init__(self, db_module=None, db_connection=None, |
160 | 160 |
block_module=None, block_path=None, block_umask=None, |
161 | 161 |
queue_module=None, queue_hosts=None, queue_exchange=None, |
162 |
quotaholder_enabled=False, |
|
163 |
quotaholder_url=None, quotaholder_token=None, |
|
164 |
quotaholder_client_poolsize=None, |
|
162 |
astakos_url=None, service_token=None, |
|
163 |
astakosclient_poolsize=None, |
|
165 | 164 |
free_versioning=True, block_params=None, |
166 | 165 |
public_url_security=None, |
167 | 166 |
public_url_alphabet=None, |
... | ... | |
205 | 204 |
params = {'wrapper': self.wrapper} |
206 | 205 |
self.permissions = self.db_module.Permissions(**params) |
207 | 206 |
self.config = self.db_module.Config(**params) |
208 |
self.quotaholder_serials = self.db_module.QuotaholderSerial(**params)
|
|
207 |
self.commission_serials = self.db_module.QuotaholderSerial(**params)
|
|
209 | 208 |
for x in ['READ', 'WRITE']: |
210 | 209 |
setattr(self, x, getattr(self.db_module, x)) |
211 | 210 |
self.node = self.db_module.Node(**params) |
... | ... | |
237 | 236 |
|
238 | 237 |
self.queue = NoQueue() |
239 | 238 |
|
240 |
self.quotaholder_enabled = quotaholder_enabled |
|
241 |
if quotaholder_enabled: |
|
242 |
self.quotaholder_url = quotaholder_url |
|
243 |
self.quotaholder_token = quotaholder_token |
|
244 |
self.quotaholder = AstakosClient( |
|
245 |
quotaholder_url, |
|
246 |
use_pool=True, |
|
247 |
pool_size=quotaholder_client_poolsize) |
|
239 |
self.astakos_url = astakos_url |
|
240 |
self.service_token = service_token |
|
241 |
self.astakosclient = AstakosClient( |
|
242 |
astakos_url, |
|
243 |
use_pool=True, |
|
244 |
pool_size=astakosclient_poolsize) |
|
248 | 245 |
|
249 | 246 |
self.serials = [] |
250 | 247 |
self.messages = [] |
... | ... | |
255 | 252 |
|
256 | 253 |
@property |
257 | 254 |
def using_external_quotaholder(self): |
258 |
return self.quotaholder_enabled
|
|
255 |
return True
|
|
259 | 256 |
|
260 | 257 |
@backend_method |
261 | 258 |
def list_accounts(self, user, marker=None, limit=10000): |
... | ... | |
1366 | 1363 |
|
1367 | 1364 |
try: |
1368 | 1365 |
name = details['path'] if 'path' in details else '' |
1369 |
serial = self.quotaholder.issue_one_commission(
|
|
1370 |
token=self.quotaholder_token,
|
|
1366 |
serial = self.astakosclient.issue_one_commission(
|
|
1367 |
token=self.service_token,
|
|
1371 | 1368 |
holder=account, |
1372 | 1369 |
source=DEFAULT_SOURCE, |
1373 | 1370 |
provisions={'pithos.diskspace': size}, |
b/snf-pithos-backend/pithos/backends/util.py | ||
---|---|---|
45 | 45 |
block_module=None, block_path=None, block_umask=None, |
46 | 46 |
queue_module=None, queue_hosts=None, |
47 | 47 |
queue_exchange=None, free_versioning=True, |
48 |
quotaholder_enabled=True, |
|
49 |
quotaholder_url=None, quotaholder_token=None, |
|
50 |
quotaholder_client_poolsize=None, |
|
48 |
astakos_url=None, service_token=None, |
|
49 |
astakosclient_poolsize=None, |
|
51 | 50 |
block_params=None, |
52 | 51 |
public_url_security=None, |
53 | 52 |
public_url_alphabet=None, |
... | ... | |
65 | 64 |
self.block_params = block_params |
66 | 65 |
self.queue_hosts = queue_hosts |
67 | 66 |
self.queue_exchange = queue_exchange |
68 |
self.quotaholder_enabled = quotaholder_enabled |
|
69 |
self.quotaholder_url = quotaholder_url |
|
70 |
self.quotaholder_token = quotaholder_token |
|
71 |
self.quotaholder_client_poolsize = quotaholder_client_poolsize |
|
67 |
self.astakos_url = astakos_url |
|
68 |
self.service_token = service_token |
|
69 |
self.astakosclient_poolsize = astakosclient_poolsize |
|
72 | 70 |
self.free_versioning = free_versioning |
73 | 71 |
self.public_url_security = public_url_security |
74 | 72 |
self.public_url_alphabet = public_url_alphabet |
... | ... | |
87 | 85 |
block_params=self.block_params, |
88 | 86 |
queue_hosts=self.queue_hosts, |
89 | 87 |
queue_exchange=self.queue_exchange, |
90 |
quotaholder_enabled=self.quotaholder_enabled, |
|
91 |
quotaholder_url=self.quotaholder_url, |
|
92 |
quotaholder_token=self.quotaholder_token, |
|
93 |
quotaholder_client_poolsize=self.quotaholder_client_poolsize, |
|
88 |
astakos_url=self.astakos_url, |
|
89 |
service_token=self.service_token, |
|
90 |
astakosclient_poolsize=self.astakosclient_poolsize, |
|
94 | 91 |
free_versioning=self.free_versioning, |
95 | 92 |
public_url_security=self.public_url_security, |
96 | 93 |
public_url_alphabet=self.public_url_alphabet, |
Also available in: Unified diff