Revision 989576a6
| b/snf-pithos-app/pithos/api/management/commands/pithos-usage.py | ||
|---|---|---|
| 38 | 38 |
from sqlalchemy import func |
| 39 | 39 |
from sqlalchemy.sql import select, and_, or_ |
| 40 | 40 |
|
| 41 |
from synnefo.util.number import strbigdec |
|
| 42 |
|
|
| 43 | 41 |
from pithos.api.util import get_backend |
| 44 | 42 |
from pithos.backends.modular import ( |
| 45 | 43 |
CLUSTER_NORMAL, CLUSTER_HISTORY, CLUSTER_DELETED |
| 46 | 44 |
) |
| 47 | 45 |
clusters = (CLUSTER_NORMAL, CLUSTER_HISTORY, CLUSTER_DELETED) |
| 48 | 46 |
|
| 49 |
Statistics = namedtuple('Statistics', ('node', 'path', 'size', 'cluster'))
|
|
| 47 |
Usage = namedtuple('Usage', ('node', 'path', 'size', 'cluster'))
|
|
| 48 |
GetQuota = namedtuple('GetQuota', ('entity', 'resource', 'key'))
|
|
| 50 | 49 |
|
| 51 | 50 |
class ResetHoldingPayload(namedtuple('ResetHoldingPayload', (
|
| 52 | 51 |
'entity', 'resource', 'key', 'imported', 'exported', 'returned', 'released' |
| ... | ... | |
| 63 | 62 |
table = {}
|
| 64 | 63 |
table['nodes'] = backend.node.nodes |
| 65 | 64 |
table['versions'] = backend.node.versions |
| 66 |
table['statistics'] = backend.node.statistics |
|
| 67 | 65 |
table['policy'] = backend.node.policy |
| 68 | 66 |
conn = backend.node.conn |
| 69 | 67 |
|
| ... | ... | |
| 75 | 73 |
s = s.where(table['nodes'].c.path.in_(users)) |
| 76 | 74 |
return conn.execute(s).fetchall() |
| 77 | 75 |
|
| 78 |
def _compute_statistics(nodes):
|
|
| 79 |
statistics = []
|
|
| 80 |
append = statistics.append
|
|
| 76 |
def _compute_usage(nodes):
|
|
| 77 |
usage = []
|
|
| 78 |
append = usage.append
|
|
| 81 | 79 |
for path, node in nodes: |
| 82 | 80 |
select_children = select( |
| 83 | 81 |
[table['nodes'].c.node]).where(table['nodes'].c.parent == node) |
| ... | ... | |
| 96 | 94 |
size = d2[CLUSTER_NORMAL] |
| 97 | 95 |
except KeyError: |
| 98 | 96 |
size = 0 |
| 99 |
append(Statistics(
|
|
| 97 |
append(Usage(
|
|
| 100 | 98 |
node=node, |
| 101 | 99 |
path=path, |
| 102 | 100 |
size=size, |
| 103 | 101 |
cluster=CLUSTER_NORMAL)) |
| 104 |
return statistics |
|
| 105 |
|
|
| 106 |
def _verify_statistics(item): |
|
| 107 |
"""Verify statistics""" |
|
| 108 |
s = select([table['statistics'].c.size]) |
|
| 109 |
s = s.where(table['statistics'].c.node == item.node) |
|
| 110 |
s = s.where(table['statistics'].c.cluster == item.cluster) |
|
| 111 |
db_item = conn.execute(s).fetchone() |
|
| 112 |
if not db_item: |
|
| 113 |
return |
|
| 114 |
try: |
|
| 115 |
assert item.size == db_item.size, \ |
|
| 116 |
'%d[%s][%d], size: %d != %d' % ( |
|
| 117 |
item.node, item.path, item.cluster, |
|
| 118 |
item.size, db_item.size) |
|
| 119 |
except AssertionError, e: |
|
| 120 |
print e |
|
| 121 |
|
|
| 122 |
def _prepare_reset_holding(statistics, verify=False): |
|
| 123 |
"""Verify statistics and set quotaholder user usage""" |
|
| 102 |
return usage |
|
| 103 |
|
|
| 104 |
def _get_quotaholder_usage(usage): |
|
| 105 |
payload = [] |
|
| 106 |
append = payload.append |
|
| 107 |
[append(GetQuota( |
|
| 108 |
entity=item.path, |
|
| 109 |
resource='pithos+.diskspace', |
|
| 110 |
key=ENTITY_KEY |
|
| 111 |
)) for item in usage] |
|
| 112 |
|
|
| 113 |
result = backend.quotaholder.get_quota( |
|
| 114 |
context={}, clientkey='pithos', get_quota=payload
|
|
| 115 |
) |
|
| 116 |
return dict((entity, imported - exported + returned - released) for ( |
|
| 117 |
entity, resource, quantity, capacity, import_limit, export_limit, |
|
| 118 |
imported, exported, returned, released, flags |
|
| 119 |
) in result) |
|
| 120 |
|
|
| 121 |
|
|
| 122 |
def _prepare_reset_holding(usage, only_diverging=False): |
|
| 123 |
"""Verify usage and set quotaholder user usage""" |
|
| 124 | 124 |
reset_holding = [] |
| 125 | 125 |
append = reset_holding.append |
| 126 |
for item in statistics: |
|
| 127 |
if verify: |
|
| 128 |
_verify_statistics(item) |
|
| 126 |
|
|
| 127 |
quotaholder_usage = {}
|
|
| 128 |
if only_diverging: |
|
| 129 |
quotaholder_usage = _get_quotaholder_usage(usage) |
|
| 130 |
|
|
| 131 |
for item in(usage): |
|
| 132 |
if only_diverging and quotaholder_usage.get(item.path) == item.size: |
|
| 133 |
continue |
|
| 134 |
|
|
| 129 | 135 |
if item.cluster == CLUSTER_NORMAL: |
| 130 | 136 |
append(ResetHoldingPayload( |
| 131 | 137 |
entity=item.path, |
| ... | ... | |
| 152 | 158 |
action="store_true", |
| 153 | 159 |
default=False, |
| 154 | 160 |
help="Reset usage for all or specified users"), |
| 155 |
make_option('--verify',
|
|
| 156 |
dest='verify',
|
|
| 161 |
make_option('--diverging',
|
|
| 162 |
dest='diverging',
|
|
| 157 | 163 |
action="store_true", |
| 158 | 164 |
default=False, |
| 159 |
help=("Verify statistics consistency for all "
|
|
| 160 |
"or specified users")), |
|
| 165 |
help=("List or reset diverging usages")),
|
|
| 161 | 166 |
make_option('--user',
|
| 162 | 167 |
dest='users', |
| 163 | 168 |
action='append', |
| 164 | 169 |
metavar='USER_UUID', |
| 165 |
help="Specify which users --list or --reset applies. This option can be repeated several times. If no user is specified --list or --reset will be applied globally."), |
|
| 170 |
help=("Specify which users --list or --reset applies.",
|
|
| 171 |
"This option can be repeated several times.", |
|
| 172 |
"If no user is specified --list or --reset will be applied globally.")), |
|
| 166 | 173 |
) |
| 167 | 174 |
|
| 168 | 175 |
def handle_noargs(self, **options): |
| 169 | 176 |
try: |
| 177 |
user_nodes = _retrieve_user_nodes(options['users']) |
|
| 178 |
if not user_nodes: |
|
| 179 |
raise CommandError('No users found.')
|
|
| 180 |
usage = _compute_usage(user_nodes) |
|
| 181 |
reset_holding = _prepare_reset_holding( |
|
| 182 |
usage, only_diverging=options['diverging'] |
|
| 183 |
) |
|
| 184 |
|
|
| 170 | 185 |
if options['list']: |
| 171 |
user_nodes = _retrieve_user_nodes(options['users']) |
|
| 172 |
if not user_nodes: |
|
| 173 |
raise CommandError('No users found.')
|
|
| 174 |
statistics = _compute_statistics(user_nodes) |
|
| 175 |
reset_holding = _prepare_reset_holding( |
|
| 176 |
statistics, verify=options['verify'] |
|
| 177 |
) |
|
| 178 | 186 |
print '\n'.join([str(i) for i in reset_holding]) |
| 179 | 187 |
|
| 180 | 188 |
if options['reset']: |
Also available in: Unified diff