root / snf-pithos-app / pithos / api / management / commands / pithos-reset-usage.py @ 7c740d3c
History | View | Annotate | Download (6.1 kB)
1 | 4c03c942 | Sofia Papagiannaki | # Copyright 2012 GRNET S.A. All rights reserved.
|
---|---|---|---|
2 | 4c03c942 | Sofia Papagiannaki | #
|
3 | 4c03c942 | Sofia Papagiannaki | # Redistribution and use in source and binary forms, with or
|
4 | 4c03c942 | Sofia Papagiannaki | # without modification, are permitted provided that the following
|
5 | 4c03c942 | Sofia Papagiannaki | # conditions are met:
|
6 | 4c03c942 | Sofia Papagiannaki | #
|
7 | 4c03c942 | Sofia Papagiannaki | # 1. Redistributions of source code must retain the above
|
8 | 4c03c942 | Sofia Papagiannaki | # copyright notice, this list of conditions and the following
|
9 | 4c03c942 | Sofia Papagiannaki | # disclaimer.
|
10 | 4c03c942 | Sofia Papagiannaki | #
|
11 | 4c03c942 | Sofia Papagiannaki | # 2. Redistributions in binary form must reproduce the above
|
12 | 4c03c942 | Sofia Papagiannaki | # copyright notice, this list of conditions and the following
|
13 | 4c03c942 | Sofia Papagiannaki | # disclaimer in the documentation and/or other materials
|
14 | 4c03c942 | Sofia Papagiannaki | # provided with the distribution.
|
15 | 4c03c942 | Sofia Papagiannaki | #
|
16 | 4c03c942 | Sofia Papagiannaki | # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
|
17 | 4c03c942 | Sofia Papagiannaki | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
18 | 4c03c942 | Sofia Papagiannaki | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
19 | 4c03c942 | Sofia Papagiannaki | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
|
20 | 4c03c942 | Sofia Papagiannaki | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
21 | 4c03c942 | Sofia Papagiannaki | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
22 | 4c03c942 | Sofia Papagiannaki | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
23 | 4c03c942 | Sofia Papagiannaki | # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
24 | 4c03c942 | Sofia Papagiannaki | # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
25 | 4c03c942 | Sofia Papagiannaki | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
26 | 4c03c942 | Sofia Papagiannaki | # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
27 | 4c03c942 | Sofia Papagiannaki | # POSSIBILITY OF SUCH DAMAGE.
|
28 | 4c03c942 | Sofia Papagiannaki | #
|
29 | 4c03c942 | Sofia Papagiannaki | # The views and conclusions contained in the software and
|
30 | 4c03c942 | Sofia Papagiannaki | # documentation are those of the authors and should not be
|
31 | 4c03c942 | Sofia Papagiannaki | # interpreted as representing official policies, either expressed
|
32 | 4c03c942 | Sofia Papagiannaki | # or implied, of GRNET S.A.
|
33 | 4c03c942 | Sofia Papagiannaki | |
34 | 4c03c942 | Sofia Papagiannaki | from django.core.management.base import NoArgsCommand, CommandError |
35 | 4c03c942 | Sofia Papagiannaki | |
36 | 4c03c942 | Sofia Papagiannaki | from collections import namedtuple |
37 | 4c03c942 | Sofia Papagiannaki | from sqlalchemy import func |
38 | 4c03c942 | Sofia Papagiannaki | from sqlalchemy.sql import select, and_, or_ |
39 | 4c03c942 | Sofia Papagiannaki | |
40 | 4c03c942 | Sofia Papagiannaki | from pithos.api.util import get_backend |
41 | 4c03c942 | Sofia Papagiannaki | from pithos.backends.modular import ( |
42 | 4c03c942 | Sofia Papagiannaki | CLUSTER_NORMAL, CLUSTER_HISTORY, CLUSTER_DELETED |
43 | 4c03c942 | Sofia Papagiannaki | ) |
44 | 4c03c942 | Sofia Papagiannaki | clusters = (CLUSTER_NORMAL, CLUSTER_HISTORY, CLUSTER_DELETED) |
45 | 4c03c942 | Sofia Papagiannaki | |
46 | 4c03c942 | Sofia Papagiannaki | Statistics = namedtuple('Statistics', ('node', 'path', 'size', 'cluster')) |
47 | 4c03c942 | Sofia Papagiannaki | |
48 | a7447c76 | Sofia Papagiannaki | ResetHoldingPayload = namedtuple('ResetHoldingPayload', (
|
49 | 09e69b8a | Georgios D. Tsoukalas | 'entity', 'resource', 'key', |
50 | 09e69b8a | Georgios D. Tsoukalas | 'imported', 'exported', 'returned', 'released')) |
51 | 09e69b8a | Georgios D. Tsoukalas | ENTITY_KEY = '1'
|
52 | 4c03c942 | Sofia Papagiannaki | |
53 | 4c03c942 | Sofia Papagiannaki | backend = get_backend() |
54 | 4c03c942 | Sofia Papagiannaki | table = {} |
55 | 4c03c942 | Sofia Papagiannaki | table['nodes'] = backend.node.nodes
|
56 | 4c03c942 | Sofia Papagiannaki | table['versions'] = backend.node.versions
|
57 | 4c03c942 | Sofia Papagiannaki | table['statistics'] = backend.node.statistics
|
58 | 6c997921 | Sofia Papagiannaki | table['policy'] = backend.node.policy
|
59 | 4c03c942 | Sofia Papagiannaki | conn = backend.node.conn |
60 | 4c03c942 | Sofia Papagiannaki | |
61 | 09e69b8a | Georgios D. Tsoukalas | |
62 | 4c03c942 | Sofia Papagiannaki | def _compute_statistics(nodes): |
63 | 4c03c942 | Sofia Papagiannaki | statistics = [] |
64 | 4c03c942 | Sofia Papagiannaki | append = statistics.append |
65 | 4c03c942 | Sofia Papagiannaki | for path, node in nodes: |
66 | 4c03c942 | Sofia Papagiannaki | select_children = select( |
67 | 09e69b8a | Georgios D. Tsoukalas | [table['nodes'].c.node]).where(table['nodes'].c.parent == node) |
68 | 4c03c942 | Sofia Papagiannaki | select_descendants = select([table['nodes'].c.node]).where(
|
69 | 4c03c942 | Sofia Papagiannaki | or_(table['nodes'].c.parent.in_(select_children),
|
70 | 4c03c942 | Sofia Papagiannaki | table['nodes'].c.node.in_(select_children)))
|
71 | 4c03c942 | Sofia Papagiannaki | s = select([table['versions'].c.cluster,
|
72 | 4c03c942 | Sofia Papagiannaki | func.sum(table['versions'].c.size)])
|
73 | 4c03c942 | Sofia Papagiannaki | s = s.group_by(table['versions'].c.cluster)
|
74 | 4c03c942 | Sofia Papagiannaki | s = s.where(table['nodes'].c.node == table['versions'].c.node) |
75 | 4c03c942 | Sofia Papagiannaki | s = s.where(table['nodes'].c.node.in_(select_descendants))
|
76 | 4c03c942 | Sofia Papagiannaki | d2 = dict(conn.execute(s).fetchall())
|
77 | d776b3c3 | Sofia Papagiannaki | |
78 | d776b3c3 | Sofia Papagiannaki | for cluster in clusters: |
79 | 4c03c942 | Sofia Papagiannaki | try:
|
80 | 4c03c942 | Sofia Papagiannaki | size = d2[cluster] |
81 | 4c03c942 | Sofia Papagiannaki | except KeyError: |
82 | d776b3c3 | Sofia Papagiannaki | size = 0
|
83 | 4c03c942 | Sofia Papagiannaki | append(Statistics( |
84 | 4c03c942 | Sofia Papagiannaki | node=node, |
85 | 4c03c942 | Sofia Papagiannaki | path=path, |
86 | 4c03c942 | Sofia Papagiannaki | size=size, |
87 | 6674f31d | Sofia Papagiannaki | cluster=cluster)) |
88 | 4c03c942 | Sofia Papagiannaki | return statistics
|
89 | 4c03c942 | Sofia Papagiannaki | |
90 | 09e69b8a | Georgios D. Tsoukalas | |
91 | a7447c76 | Sofia Papagiannaki | def _get_verified_usage(statistics): |
92 | 09e69b8a | Georgios D. Tsoukalas | """Verify statistics and set quotaholder account usage"""
|
93 | a7447c76 | Sofia Papagiannaki | reset_holding = [] |
94 | a7447c76 | Sofia Papagiannaki | append = reset_holding.append |
95 | 4c03c942 | Sofia Papagiannaki | for item in statistics: |
96 | 4c03c942 | Sofia Papagiannaki | s = select([table['statistics'].c.size])
|
97 | 4c03c942 | Sofia Papagiannaki | s = s.where(table['statistics'].c.node == item.node)
|
98 | 4c03c942 | Sofia Papagiannaki | s = s.where(table['statistics'].c.cluster == item.cluster)
|
99 | 4c03c942 | Sofia Papagiannaki | db_item = conn.execute(s).fetchone() |
100 | 4c03c942 | Sofia Papagiannaki | if not db_item: |
101 | 4c03c942 | Sofia Papagiannaki | continue
|
102 | 4c03c942 | Sofia Papagiannaki | try:
|
103 | 4c03c942 | Sofia Papagiannaki | assert item.size == db_item.size, \
|
104 | 09e69b8a | Georgios D. Tsoukalas | '%d[%s][%d], size: %d != %d' % (
|
105 | 09e69b8a | Georgios D. Tsoukalas | item.node, item.path, item.cluster, |
106 | 09e69b8a | Georgios D. Tsoukalas | item.size, db_item.size) |
107 | 4c03c942 | Sofia Papagiannaki | except AssertionError, e: |
108 | 4c03c942 | Sofia Papagiannaki | print e
|
109 | 6674f31d | Sofia Papagiannaki | if item.cluster == CLUSTER_NORMAL:
|
110 | 6674f31d | Sofia Papagiannaki | append(ResetHoldingPayload( |
111 | 6674f31d | Sofia Papagiannaki | entity=item.path, |
112 | 6674f31d | Sofia Papagiannaki | resource='pithos+.diskspace',
|
113 | 6674f31d | Sofia Papagiannaki | key=ENTITY_KEY, |
114 | 6674f31d | Sofia Papagiannaki | imported=item.size, |
115 | 6674f31d | Sofia Papagiannaki | exported=0,
|
116 | 6674f31d | Sofia Papagiannaki | returned=0,
|
117 | 6674f31d | Sofia Papagiannaki | released=0))
|
118 | a7447c76 | Sofia Papagiannaki | return reset_holding
|
119 | d776b3c3 | Sofia Papagiannaki | |
120 | d776b3c3 | Sofia Papagiannaki | |
121 | 4c03c942 | Sofia Papagiannaki | class Command(NoArgsCommand): |
122 | a7447c76 | Sofia Papagiannaki | help = "Set quotaholder account usage"
|
123 | 4c03c942 | Sofia Papagiannaki | |
124 | 4c03c942 | Sofia Papagiannaki | def handle_noargs(self, **options): |
125 | 4c03c942 | Sofia Papagiannaki | try:
|
126 | 4c03c942 | Sofia Papagiannaki | if not backend.quotaholder_url: |
127 | 4c03c942 | Sofia Papagiannaki | raise CommandError('Quotaholder component url is not set') |
128 | d776b3c3 | Sofia Papagiannaki | |
129 | 4c03c942 | Sofia Papagiannaki | if not backend.quotaholder_token: |
130 | 4c03c942 | Sofia Papagiannaki | raise CommandError('Quotaholder component token is not set') |
131 | 4c03c942 | Sofia Papagiannaki | |
132 | 4c03c942 | Sofia Papagiannaki | # retrieve account nodes
|
133 | 4c03c942 | Sofia Papagiannaki | s = select([table['nodes'].c.path, table['nodes'].c.node]) |
134 | 4c03c942 | Sofia Papagiannaki | s = s.where(and_(table['nodes'].c.node != 0, |
135 | 4c03c942 | Sofia Papagiannaki | table['nodes'].c.parent == 0)) |
136 | 4c03c942 | Sofia Papagiannaki | account_nodes = conn.execute(s).fetchall() |
137 | d776b3c3 | Sofia Papagiannaki | |
138 | 4c03c942 | Sofia Papagiannaki | # compute account statistics
|
139 | 9a0a8eca | Sofia Papagiannaki | statistics = _compute_statistics(account_nodes) |
140 | 4c03c942 | Sofia Papagiannaki | |
141 | a7447c76 | Sofia Papagiannaki | # verify and send usage
|
142 | a7447c76 | Sofia Papagiannaki | reset_holding = _get_verified_usage(statistics) |
143 | 9a0a8eca | Sofia Papagiannaki | |
144 | d776b3c3 | Sofia Papagiannaki | while True: |
145 | a7447c76 | Sofia Papagiannaki | result = backend.quotaholder.reset_holding( |
146 | d776b3c3 | Sofia Papagiannaki | context={}, |
147 | d776b3c3 | Sofia Papagiannaki | clientkey='pithos',
|
148 | a7447c76 | Sofia Papagiannaki | reset_holding=reset_holding) |
149 | d776b3c3 | Sofia Papagiannaki | |
150 | d776b3c3 | Sofia Papagiannaki | if not result: |
151 | d776b3c3 | Sofia Papagiannaki | break
|
152 | d776b3c3 | Sofia Papagiannaki | |
153 | a7447c76 | Sofia Papagiannaki | missing_entities = [reset_holding[x].entity for x in result] |
154 | d776b3c3 | Sofia Papagiannaki | self.stdout.write(
|
155 | a7447c76 | Sofia Papagiannaki | 'Unknown quotaholder accounts: %s\n' %
|
156 | a7447c76 | Sofia Papagiannaki | ','.join(missing_entities))
|
157 | 09e69b8a | Georgios D. Tsoukalas | m = 'Retrying sending quota usage for the rest...\n'
|
158 | 09e69b8a | Georgios D. Tsoukalas | self.stdout.write(m)
|
159 | a7447c76 | Sofia Papagiannaki | missing_indexes = set(result)
|
160 | a7447c76 | Sofia Papagiannaki | reset_holding = [x for i, x in enumerate(reset_holding) |
161 | a7447c76 | Sofia Papagiannaki | if i not in missing_indexes] |
162 | d776b3c3 | Sofia Papagiannaki | finally:
|
163 | 4c03c942 | Sofia Papagiannaki | backend.close() |