Revision 2ff02341
| b/snf-pithos-app/pithos/api/management/commands/reconcile-resources-pithos.py | ||
|---|---|---|
| 31 | 31 |
# interpreted as representing official policies, either expressed |
| 32 | 32 |
# or implied, of GRNET S.A. |
| 33 | 33 |
|
| 34 |
from django.core.management.base import NoArgsCommand, CommandError
|
|
| 34 |
from django.core.management.base import NoArgsCommand |
|
| 35 | 35 |
|
| 36 | 36 |
from optparse import make_option |
| 37 | 37 |
|
| 38 | 38 |
from pithos.api.util import get_backend |
| 39 | 39 |
from pithos.api.resources import resources |
| 40 |
from pithos.backends.modular import CLUSTER_NORMAL, DEFAULT_SOURCE
|
|
| 40 |
from pithos.backends.modular import DEFAULT_SOURCE |
|
| 41 | 41 |
|
| 42 | 42 |
from synnefo.webproject.management import utils |
| 43 | 43 |
|
| 44 |
from astakosclient.errors import QuotaLimit |
|
| 44 |
from astakosclient.errors import QuotaLimit, NotFound
|
|
| 45 | 45 |
|
| 46 | 46 |
backend = get_backend() |
| 47 | 47 |
|
| ... | ... | |
| 70 | 70 |
|
| 71 | 71 |
def handle_noargs(self, **options): |
| 72 | 72 |
try: |
| 73 |
qh_result = backend.astakosclient.service_get_quotas( |
|
| 74 |
backend.service_token) |
|
| 73 |
userid = options['userid'] |
|
| 75 | 74 |
|
| 76 |
users = (options['userid'],) if options['userid'] else None |
|
| 77 |
account_nodes = backend.node.node_accounts(users) |
|
| 78 |
if not account_nodes: |
|
| 79 |
raise CommandError('No users found.')
|
|
| 75 |
# Get holding from Pithos DB |
|
| 76 |
db_usage = backend.node.node_account_usage(userid) |
|
| 80 | 77 |
|
| 81 |
db_usage = {}
|
|
| 82 |
for path, node in account_nodes:
|
|
| 83 |
size = backend.node.node_account_usage(node, CLUSTER_NORMAL)
|
|
| 84 |
db_usage[path] = size or 0
|
|
| 78 |
users = set(db_usage.keys())
|
|
| 79 |
if userid and userid not in users:
|
|
| 80 |
self.stdout.write("User '%s' does not exist in DB!\n" % userid)
|
|
| 81 |
return
|
|
| 85 | 82 |
|
| 86 |
users = set(qh_result.keys()) |
|
| 87 |
users.update(db_usage.keys()) |
|
| 83 |
# Get holding from Quotaholder |
|
| 84 |
try: |
|
| 85 |
qh_result = backend.astakosclient.service_get_quotas( |
|
| 86 |
backend.service_token, userid) |
|
| 87 |
except NotFound: |
|
| 88 |
self.stdout.write( |
|
| 89 |
"User '%s' does not exist in Quotaholder!\n" % userid) |
|
| 90 |
return |
|
| 91 |
|
|
| 92 |
users.update(qh_result.keys()) |
|
| 88 | 93 |
|
| 89 | 94 |
pending_exists = False |
| 90 | 95 |
unknown_user_exists = False |
| ... | ... | |
| 95 | 100 |
qh_all = qh_result[uuid] |
| 96 | 101 |
except KeyError: |
| 97 | 102 |
self.stdout.write( |
| 98 |
"User '%s' does not exist in Quotaholder!\n" % uuid |
|
| 99 |
) |
|
| 103 |
"User '%s' does not exist in Quotaholder!\n" % uuid) |
|
| 100 | 104 |
unknown_user_exists = True |
| 101 | 105 |
continue |
| 102 | 106 |
else: |
| ... | ... | |
| 114 | 118 |
self.stdout.write( |
| 115 | 119 |
"Pending commission. " |
| 116 | 120 |
"User '%s', resource '%s'.\n" % |
| 117 |
(uuid, resource) |
|
| 118 |
) |
|
| 121 |
(uuid, resource)) |
|
| 119 | 122 |
pending_exists = True |
| 120 | 123 |
continue |
| 121 | 124 |
|
| ... | ... | |
| 146 | 149 |
if pending_exists: |
| 147 | 150 |
self.stdout.write( |
| 148 | 151 |
"Found pending commissions. Run 'snf-manage" |
| 149 |
" reconcile-commissions-pithos'\n" |
|
| 150 |
) |
|
| 152 |
" reconcile-commissions-pithos'\n") |
|
| 151 | 153 |
elif not (unsynced or unknown_user_exists): |
| 152 | 154 |
self.stdout.write("Everything in sync.\n")
|
| 153 | 155 |
finally: |
| b/snf-pithos-backend/pithos/backends/lib/sqlalchemy/node.py | ||
|---|---|---|
| 480 | 480 |
r.close() |
| 481 | 481 |
return dict(rows) |
| 482 | 482 |
|
| 483 |
def node_account_usage(self, account_node, cluster): |
|
| 484 |
select_children = select( |
|
| 485 |
[self.nodes.c.node]).where(self.nodes.c.parent == account_node) |
|
| 486 |
select_descendants = select([self.nodes.c.node]).where( |
|
| 487 |
or_(self.nodes.c.parent.in_(select_children), |
|
| 488 |
self.nodes.c.node.in_(select_children))) |
|
| 489 |
s = select([func.sum(self.versions.c.size)]) |
|
| 490 |
s = s.where(self.nodes.c.node == self.versions.c.node) |
|
| 491 |
s = s.where(self.nodes.c.node.in_(select_descendants)) |
|
| 483 |
def node_account_usage(self, account=None, cluster=0): |
|
| 484 |
"""Return usage for a specific account. |
|
| 485 |
|
|
| 486 |
Keyword arguments: |
|
| 487 |
account -- (default None: list usage for all the accounts) |
|
| 488 |
cluster -- list current, history or deleted usage (default 0: normal) |
|
| 489 |
""" |
|
| 490 |
|
|
| 491 |
n1 = self.nodes.alias('n1')
|
|
| 492 |
n2 = self.nodes.alias('n2')
|
|
| 493 |
n3 = self.nodes.alias('n3')
|
|
| 494 |
|
|
| 495 |
s = select([n3.c.path, func.sum(self.versions.c.size)]) |
|
| 496 |
s = s.where(n1.c.node == self.versions.c.node) |
|
| 492 | 497 |
s = s.where(self.versions.c.cluster == cluster) |
| 498 |
s = s.where(n1.c.parent == n2.c.node) |
|
| 499 |
s = s.where(n2.c.parent == n3.c.node) |
|
| 500 |
s = s.where(n3.c.parent == 0) |
|
| 501 |
s = s.where(n3.c.node != 0) |
|
| 502 |
if account: |
|
| 503 |
s = s.where(n3.c.path == account) |
|
| 504 |
s = s.group_by(n3.c.path) |
|
| 493 | 505 |
r = self.conn.execute(s) |
| 494 |
usage = r.fetchone()[0]
|
|
| 506 |
usage = r.fetchall()
|
|
| 495 | 507 |
r.close() |
| 496 |
return usage
|
|
| 508 |
return dict(usage)
|
|
| 497 | 509 |
|
| 498 | 510 |
def policy_get(self, node): |
| 499 | 511 |
s = select([self.policy.c.key, self.policy.c.value], |
| b/snf-pithos-backend/pithos/backends/lib/sqlite/node.py | ||
|---|---|---|
| 302 | 302 |
for r in self.fetchall(): |
| 303 | 303 |
hashes += [r[0]] |
| 304 | 304 |
serials += [r[1]] |
| 305 |
|
|
| 305 |
|
|
| 306 | 306 |
q = ("delete from versions "
|
| 307 | 307 |
"where node in (select node " |
| 308 | 308 |
"from nodes " |
| ... | ... | |
| 402 | 402 |
) |
| 403 | 403 |
return dict(self.execute(q).fetchall()) |
| 404 | 404 |
|
| 405 |
def node_account_usage(self, account_node, cluster): |
|
| 406 |
select_children = ("select node from nodes where parent = ?")
|
|
| 407 |
select_descedents = ("select node from nodes "
|
|
| 408 |
"where parent in (%s) " |
|
| 409 |
"or node in (%s) ") % ((select_children,)*2) |
|
| 410 |
args = [account_node]*2 |
|
| 411 |
q = ("select sum(v.size) from versions v, nodes n "
|
|
| 412 |
"where v.node = n.node " |
|
| 413 |
"and n.node in (%s) " |
|
| 414 |
"and v.cluster = ?") % select_descedents |
|
| 415 |
args += [cluster] |
|
| 405 |
def node_account_usage(self, account=None, cluster=0): |
|
| 406 |
"""Return usage for a specific account. |
|
| 416 | 407 |
|
| 408 |
Keyword arguments: |
|
| 409 |
account -- (default None: list usage for all the accounts) |
|
| 410 |
cluster -- list current, history or deleted usage (default 0: normal) |
|
| 411 |
""" |
|
| 412 |
|
|
| 413 |
q = ("select n3.path, sum(v.size) from "
|
|
| 414 |
"versions v, nodes n1, nodes n2, nodes n3 " |
|
| 415 |
"where v.node = n1.node " |
|
| 416 |
"and v.cluster = ? " |
|
| 417 |
"and n1.parent = n2.node " |
|
| 418 |
"and n2.parent = n3.node " |
|
| 419 |
"and n3.parent = 0 " |
|
| 420 |
"and n3.node != 0 ") |
|
| 421 |
args = [cluster] |
|
| 422 |
if account: |
|
| 423 |
q += ("and n3.path = ? ")
|
|
| 424 |
args += [account] |
|
| 425 |
q += ("group by n3.path")
|
|
| 426 |
|
|
| 427 |
print '###', q, args |
|
| 417 | 428 |
self.execute(q, args) |
| 418 |
return self.fetchone()[0]
|
|
| 429 |
return dict(self.fetchall())
|
|
| 419 | 430 |
|
| 420 | 431 |
def policy_get(self, node): |
| 421 | 432 |
q = "select key, value from policy where node = ?" |
Also available in: Unified diff