Revision 16f2673e

b/Changelog
14 14

  
15 15
Released: UNRELEASED
16 16

  
17
* Incorporate quotaholder in astakos
18
  * obsolete settings:
19
    PITHOS_USE_QUOTAHOLDER, PITHOS_QUOTAHOLDER_URL, PITHOS_QUOTAHOLDER_TOKEN
20
  * new settins:
21
    PITHOS_ASTAKOSCLIENT_POOLSIZE
22

  
17 23
Synnefo-wide
18 24
------------
19 25
* Create 'snf_django' Python package to hold common code for all Synnefo
b/snf-cyclades-app/synnefo/plankton/backend.py
76 76
_pithos_backend_pool = \
77 77
    PithosBackendPool(
78 78
        POOL_SIZE,
79
        quotaholder_enabled=True,
80
        quotaholder_url=settings.ASTAKOS_URL,
81
        quotaholder_token=settings.CYCLADES_ASTAKOS_SERVICE_TOKEN,
82
        quotaholder_client_poolsize=settings.CYCLADES_QUOTAHOLDER_POOLSIZE,
79
        astakos_url=settings.ASTAKOS_URL,
80
        service_token=settings.CYCLADES_ASTAKOS_SERVICE_TOKEN,
81
        astakosclient_poolsize=settings.CYCLADES_QUOTAHOLDER_POOLSIZE,
83 82
        db_connection=settings.BACKEND_DB_CONNECTION,
84 83
        block_path=settings.BACKEND_BLOCK_PATH)
85 84

  
b/snf-pithos-app/README
31 31
Name                             Default value                                                      Description
32 32
===============================  ================================================================   ============================================================
33 33
ASTAKOS_URL                      \https://<astakos.host>/                                           Astakos Authentication URL
34
PITHOS_ASTAKOSCLIENT_POOLSIZE    200                                                                The size of the http pool for the astakos client
34 35
PITHOS_ASTAKOS_COOKIE_NAME       _pithos2_a                                                         Cookie name to retrieve fallback token
35 36
PITHOS_BACKEND_DB_MODULE         pithos.backends.lib.sqlalchemy
36 37
PITHOS_BACKEND_DB_CONNECTION     sqlite:////tmp/pithos-backend.db                                   SQLAlchemy database connection string
......
55 56
PITHOS_USER_CATALOG_URL          \https://<astakos.host>/user_catalogs/                             Astakos User Catalog URL
56 57
PITHOS_USER_FEEDBACK_URL         \https://<astakos.host>/feedback/                                  Astakos User Feedback URL
57 58
PITHOS_USER_LOGIN_URL            \https://<astakos.host>/login/                                     Astakos User Login URL
58
PITHOS_USE_QUOTAHOLDER           True                                                               Enable quotaholder
59
PITHOS_QUOTAHOLDER_URL           ''                                                                 Quotaholder URL
60
PITHOS_QUOTAHOLDER_TOKEN         ''                                                                 Quotaholder token
61 59
PITHOS_PUBLIC_URL_SECURITY       16                                                                 How many random bytes to use for constructing the URL of Pithos public files
62 60
PITHOS_PUBLIC_URL_ALPHABET       '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'   The alphabet to use for constructing the URL of Pithos public files
63 61
===============================  ================================================================   ============================================================
b/snf-pithos-app/conf/20-snf-pithos-app-settings.conf
45 45
#PITHOS_USER_FEEDBACK_URL = 'https://accounts.example.synnefo.org/feedback/'
46 46
#PITHOS_USER_LOGIN_URL = 'https://accounts.example.synnefo.org/login/'
47 47

  
48
# Enable and configure quotaholder
49
#PITHOS_USE_QUOTAHOLDER = False
50
#PITHOS_QUOTAHOLDER_URL = ''
51
#PITHOS_QUOTAHOLDER_TOKEN = ''
52
#
53
# Tune the size of the http pool for the quotaholder client.
48
# Tune the size of the http pool for the astakos client.
54 49
# It limits the maximum number of quota changing requests
55 50
# that pithos can serve. Extra requests will be blocked
56 51
# until another has completed.
57 52
#
58
#PITHOS_QUOTAHOLDER_POOLSIZE = 200
53
#PITHOS_ASTAKOSCLIENT_POOLSIZE = 200
59 54
#
60 55
# How many random bytes to use for constructing the URL of Pithos public files.
61 56
# Lower values mean accidental reuse of (discarded) URLs is more probable.
b/snf-pithos-app/pithos/api/management/commands/reconcile-commissions-pithos.py
56 56
    def handle_noargs(self, **options):
57 57
        b = get_backend()
58 58
        try:
59
            pending_commissions = b.quotaholder.get_pending_commissions(
60
                token=b.quotaholder_token)
59
            pending_commissions = b.astakosclient.get_pending_commissions(
60
                token=b.service_token)
61 61

  
62 62
            if pending_commissions:
63 63
                self.stdout.write(
......
68 68
                return
69 69

  
70 70
            if options['fix']:
71
                to_accept = b.quotaholder_serials.lookup(pending_commissions)
71
                to_accept = b.commision_serials.lookup(pending_commissions)
72 72
                to_reject = list(set(pending_commissions) - set(to_accept))
73
                response = b.quotaholder.resolve_commissions(
74
                    token=b.quotaholder_token,
73
                response = b.astakosclient.resolve_commissions(
74
                    token=b.service_token,
75 75
                    accept_serials=to_accept,
76 76
                    reject_serials=to_reject
77 77
                )
......
84 84
                for i in failed:
85 85
                    self.stdout.write('%s\n' % i)
86 86

  
87
                b.quotaholder_serials.delete_many(accepted)
87
                b.commision_serials.delete_many(accepted)
88 88
        except Exception, e:
89 89
            logger.exception(e)
90 90
            raise CommandError(e)
b/snf-pithos-app/pithos/api/management/commands/reconcile-resources-pithos.py
99 99
                size = backend.node.node_account_usage(node, CLUSTER_NORMAL)
100 100
                db_usage[path] = size or 0
101 101

  
102
            result = backend.quotaholder.service_get_quotas(
103
                backend.quotaholder_token,
102
            result = backend.astakosclient.service_get_quotas(
103
                backend.service_token,
104 104
            )
105 105

  
106 106
            qh_usage = {}
......
149 149
                request['auto_accept'] = True
150 150
                request['provisions'] = provisions
151 151
                try:
152
                    serial = backend.quotaholder.issue_commission(
153
                        backend.quotaholder_token, request
152
                    serial = backend.astakosclient.issue_commission(
153
                        backend.service_token, request
154 154
                    )
155 155
                except AstakosClientException, e:
156 156
                    self.stdout.write(e.details)
157 157
                else:
158
                    backend.quotaholder_serials.insert_many([serial])
158
                    backend.commission_serials.insert_many([serial])
159 159
            elif options['list'] and table:
160 160
                output_format = options["output_format"]
161 161
                if output_format != "json" and not options["headers"]:
b/snf-pithos-app/pithos/api/settings.py
3 3

  
4 4
ASTAKOS_URL = getattr(settings, "ASTAKOS_URL",
5 5
                      "https://accounts.example.synnefo.org/")
6
ASTAKOSCLIENT_POOLSIZE = getattr(settings, 'PITHOS_ASTAKOSCLIENT_POOLSIZE', 200)
6 7

  
7 8
COOKIE_NAME = getattr(settings, 'PITHOS_ASTAKOS_COOKIE_NAME', '_pithos2_a')
8 9

  
......
64 65
USER_LOGIN_URL = getattr(settings, 'PITHOS_USER_LOGIN_URL',
65 66
                         'https://accounts.example.synnefo.org/login/')
66 67

  
67
# Set the quota holder component URI
68
USE_QUOTAHOLDER = getattr(settings, 'PITHOS_USE_QUOTAHOLDER', False)
69
QUOTAHOLDER_URL = getattr(settings, 'PITHOS_QUOTAHOLDER_URL', '')
70
QUOTAHOLDER_TOKEN = getattr(settings, 'PITHOS_QUOTAHOLDER_TOKEN', '')
71
QUOTAHOLDER_POOLSIZE = getattr(settings, 'PITHOS_QUOTAHOLDER_POOLSIZE', 200)
72

  
73 68
# Set how many random bytes to use for constructing the URL of Pithos public files
74 69
PUBLIC_URL_SECURITY =  getattr(settings, 'PITHOS_PUBLIC_URL_SECURITY', 16)
75 70
# Set the alphabet to use for constructing the URL of Pithos public files
b/snf-pithos-app/pithos/api/util.py
55 55
                                 BACKEND_BLOCK_UMASK,
56 56
                                 BACKEND_QUEUE_MODULE, BACKEND_QUEUE_HOSTS,
57 57
                                 BACKEND_QUEUE_EXCHANGE,
58
                                 QUOTAHOLDER_POOLSIZE,
58
                                 ASTAKOSCLIENT_POOLSIZE,
59 59
                                 SERVICE_TOKEN,
60 60
                                 ASTAKOS_URL,
61 61
                                 BACKEND_ACCOUNT_QUOTA, BACKEND_CONTAINER_QUOTA,
......
953 953
        queue_module=BACKEND_QUEUE_MODULE,
954 954
        queue_hosts=BACKEND_QUEUE_HOSTS,
955 955
        queue_exchange=BACKEND_QUEUE_EXCHANGE,
956
        quotaholder_enabled=True,
957
        quotaholder_url=ASTAKOS_URL,
958
        quotaholder_token=SERVICE_TOKEN,
959
        quotaholder_client_poolsize=QUOTAHOLDER_POOLSIZE,
956
        astakos_url=ASTAKOS_URL,
957
        service_token=SERVICE_TOKEN,
958
        astakosclient_poolsize=ASTAKOSCLIENT_POOLSIZE,
960 959
        free_versioning=BACKEND_FREE_VERSIONING,
961 960
        block_params=BLOCK_PARAMS,
962 961
        public_url_security=PUBLIC_URL_SECURITY,
b/snf-pithos-backend/pithos/backends/modular.py
141 141
            return ret
142 142
        except:
143 143
            if serials:
144
                self.quotaholder.resolve_commissions(
145
                    token=self.quotaholder_token,
144
                self.astakosclient.resolve_commissions(
145
                    token=self.service_token,
146 146
                    accept_serials=[],
147 147
                    reject_serials=serials)
148 148
            self.wrapper.rollback()
......
159 159
    def __init__(self, db_module=None, db_connection=None,
160 160
                 block_module=None, block_path=None, block_umask=None,
161 161
                 queue_module=None, queue_hosts=None, queue_exchange=None,
162
                 quotaholder_enabled=False,
163
                 quotaholder_url=None, quotaholder_token=None,
164
                 quotaholder_client_poolsize=None,
162
                 astakos_url=None, service_token=None,
163
                 astakosclient_poolsize=None,
165 164
                 free_versioning=True, block_params=None,
166 165
                 public_url_security=None,
167 166
                 public_url_alphabet=None,
......
205 204
        params = {'wrapper': self.wrapper}
206 205
        self.permissions = self.db_module.Permissions(**params)
207 206
        self.config = self.db_module.Config(**params)
208
        self.quotaholder_serials = self.db_module.QuotaholderSerial(**params)
207
        self.commission_serials = self.db_module.QuotaholderSerial(**params)
209 208
        for x in ['READ', 'WRITE']:
210 209
            setattr(self, x, getattr(self.db_module, x))
211 210
        self.node = self.db_module.Node(**params)
......
237 236

  
238 237
            self.queue = NoQueue()
239 238

  
240
        self.quotaholder_enabled = quotaholder_enabled
241
        if quotaholder_enabled:
242
            self.quotaholder_url = quotaholder_url
243
            self.quotaholder_token = quotaholder_token
244
            self.quotaholder = AstakosClient(
245
                quotaholder_url,
246
                use_pool=True,
247
                pool_size=quotaholder_client_poolsize)
239
        self.astakos_url = astakos_url
240
        self.service_token = service_token
241
        self.astakosclient = AstakosClient(
242
            astakos_url,
243
            use_pool=True,
244
            pool_size=astakosclient_poolsize)
248 245

  
249 246
        self.serials = []
250 247
        self.messages = []
......
255 252

  
256 253
    @property
257 254
    def using_external_quotaholder(self):
258
        return self.quotaholder_enabled
255
        return True
259 256

  
260 257
    @backend_method
261 258
    def list_accounts(self, user, marker=None, limit=10000):
......
1366 1363

  
1367 1364
        try:
1368 1365
            name = details['path'] if 'path' in details else ''
1369
            serial = self.quotaholder.issue_one_commission(
1370
                token=self.quotaholder_token,
1366
            serial = self.astakosclient.issue_one_commission(
1367
                token=self.service_token,
1371 1368
                holder=account,
1372 1369
                source=DEFAULT_SOURCE,
1373 1370
                provisions={'pithos.diskspace': size},
b/snf-pithos-backend/pithos/backends/util.py
45 45
                 block_module=None, block_path=None, block_umask=None,
46 46
                 queue_module=None, queue_hosts=None,
47 47
                 queue_exchange=None, free_versioning=True,
48
                 quotaholder_enabled=True,
49
                 quotaholder_url=None, quotaholder_token=None,
50
                 quotaholder_client_poolsize=None,
48
                 astakos_url=None, service_token=None,
49
                 astakosclient_poolsize=None,
51 50
                 block_params=None,
52 51
                 public_url_security=None,
53 52
                 public_url_alphabet=None,
......
65 64
        self.block_params = block_params
66 65
        self.queue_hosts = queue_hosts
67 66
        self.queue_exchange = queue_exchange
68
        self.quotaholder_enabled = quotaholder_enabled
69
        self.quotaholder_url = quotaholder_url
70
        self.quotaholder_token = quotaholder_token
71
        self.quotaholder_client_poolsize = quotaholder_client_poolsize
67
        self.astakos_url = astakos_url
68
        self.service_token = service_token
69
        self.astakosclient_poolsize = astakosclient_poolsize
72 70
        self.free_versioning = free_versioning
73 71
        self.public_url_security = public_url_security
74 72
        self.public_url_alphabet = public_url_alphabet
......
87 85
                block_params=self.block_params,
88 86
                queue_hosts=self.queue_hosts,
89 87
                queue_exchange=self.queue_exchange,
90
                quotaholder_enabled=self.quotaholder_enabled,
91
                quotaholder_url=self.quotaholder_url,
92
                quotaholder_token=self.quotaholder_token,
93
                quotaholder_client_poolsize=self.quotaholder_client_poolsize,
88
                astakos_url=self.astakos_url,
89
                service_token=self.service_token,
90
                astakosclient_poolsize=self.astakosclient_poolsize,
94 91
                free_versioning=self.free_versioning,
95 92
                public_url_security=self.public_url_security,
96 93
                public_url_alphabet=self.public_url_alphabet,

Also available in: Unified diff