Revision 363a5cdb
b/contrib/astakos_conflicting_users.py | ||
---|---|---|
1 |
import os |
|
2 |
import sys |
|
3 |
os.environ['DJANGO_SETTINGS_MODULE'] = 'synnefo.settings' |
|
4 |
|
|
5 |
from django.conf import settings |
|
6 |
from astakos.im.models import AstakosUser |
|
7 |
|
|
8 |
def duplicate_users(): |
|
9 |
for u in AstakosUser.objects.filter(): |
|
10 |
if AstakosUser.objects.filter(email__iexact=u.email).count() > 1: |
|
11 |
print AstakosUser.objects.filter(email__iexact=u.email).values('pk', |
|
12 |
'email', |
|
13 |
'is_active') |
|
14 |
|
|
15 |
if len(sys.argv) == 2: |
|
16 |
pk = int(sys.argv[1]) |
|
17 |
user = AstakosUser.objects.get(pk=pk) |
|
18 |
if AstakosUser.objects.filter(email__iexact=user.email).count() == 1: |
|
19 |
print "No duplicate emails found for user %s" % (user) |
|
20 |
exit() |
|
21 |
user = AstakosUser.objects.get(pk=pk) |
|
22 |
print "Deleting user %r" % (user) |
|
23 |
user.delete() |
|
24 |
exit() |
|
25 |
else: |
|
26 |
duplicate_users() |
|
27 |
|
b/contrib/migrate-data | ||
---|---|---|
1 |
#!/usr/bin/env python |
|
2 |
|
|
3 |
# Copyright 2011-2012 GRNET S.A. All rights reserved. |
|
4 |
# |
|
5 |
# Redistribution and use in source and binary forms, with or |
|
6 |
# without modification, are permitted provided that the following |
|
7 |
# conditions are met: |
|
8 |
# |
|
9 |
# 1. Redistributions of source code must retain the above |
|
10 |
# copyright notice, this list of conditions and the following |
|
11 |
# disclaimer. |
|
12 |
# |
|
13 |
# 2. Redistributions in binary form must reproduce the above |
|
14 |
# copyright notice, this list of conditions and the following |
|
15 |
# disclaimer in the documentation and/or other materials |
|
16 |
# provided with the distribution. |
|
17 |
# |
|
18 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
|
19 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
|
20 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
|
21 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
|
22 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
23 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
24 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
|
25 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
|
26 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
|
27 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
|
28 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
|
29 |
# POSSIBILITY OF SUCH DAMAGE. |
|
30 |
# |
|
31 |
# The views and conclusions contained in the software and |
|
32 |
# documentation are those of the authors and should not be |
|
33 |
# interpreted as representing official policies, either expressed |
|
34 |
# or implied, of GRNET S.A. |
|
35 |
|
|
36 |
from binascii import hexlify |
|
37 |
|
|
38 |
from sqlalchemy import Table |
|
39 |
from sqlalchemy.sql import select |
|
40 |
|
|
41 |
from pithos import settings |
|
42 |
from pithos.backends.modular import ModularBackend |
|
43 |
|
|
44 |
from pithos.tools.lib.hashmap import HashMap |
|
45 |
|
|
46 |
from migrate import Migration, Cache |
|
47 |
|
|
48 |
import os |
|
49 |
|
|
50 |
class DataMigration(Migration): |
|
51 |
def __init__(self, pithosdb, db): |
|
52 |
Migration.__init__(self, pithosdb) |
|
53 |
self.cache = Cache(db) |
|
54 |
|
|
55 |
def retrieve_files(self): |
|
56 |
# Loop for all available files. |
|
57 |
filebody = Table('filebody', self.metadata, autoload=True) |
|
58 |
s = select([filebody.c.storedfilepath]) |
|
59 |
rp = self.conn.execute(s) |
|
60 |
path = rp.fetchone() |
|
61 |
while path: |
|
62 |
yield path |
|
63 |
path = rp.fetchone() |
|
64 |
rp.close() |
|
65 |
|
|
66 |
def execute(self): |
|
67 |
blocksize = self.backend.block_size |
|
68 |
blockhash = self.backend.hash_algorithm |
|
69 |
|
|
70 |
for (path,) in self.retrieve_files(): |
|
71 |
map = HashMap(blocksize, blockhash) |
|
72 |
try: |
|
73 |
map.load(open(path)) |
|
74 |
except Exception, e: |
|
75 |
print e |
|
76 |
continue |
|
77 |
hash = hexlify(map.hash()) |
|
78 |
|
|
79 |
if hash != self.cache.get(path): |
|
80 |
missing = self.backend.blocker.block_ping(map) # XXX Backend hack... |
|
81 |
status = '[>] ' + path |
|
82 |
if missing: |
|
83 |
status += ' - %d block(s) missing' % len(missing) |
|
84 |
with open(path) as fp: |
|
85 |
for h in missing: |
|
86 |
offset = map.index(h) * blocksize |
|
87 |
fp.seek(offset) |
|
88 |
block = fp.read(blocksize) |
|
89 |
self.backend.put_block(block) |
|
90 |
else: |
|
91 |
status += ' - no blocks missing' |
|
92 |
self.cache.put(path, hash) |
|
93 |
else: |
|
94 |
status = '[-] ' + path |
|
95 |
print status |
|
96 |
|
|
97 |
if __name__ == "__main__": |
|
98 |
pithosdb = 'postgresql://gss@127.0.0.1/pithos' |
|
99 |
db = 'sqlite:///migrate.db' |
|
100 |
|
|
101 |
dt = DataMigration(pithosdb, db) |
|
102 |
dt.execute() |
b/contrib/migrate-db | ||
---|---|---|
1 |
#!/usr/bin/env python |
|
2 |
|
|
3 |
# Copyright 2011-2012 GRNET S.A. All rights reserved. |
|
4 |
# |
|
5 |
# Redistribution and use in source and binary forms, with or |
|
6 |
# without modification, are permitted provided that the following |
|
7 |
# conditions are met: |
|
8 |
# |
|
9 |
# 1. Redistributions of source code must retain the above |
|
10 |
# copyright notice, this list of conditions and the following |
|
11 |
# disclaimer. |
|
12 |
# |
|
13 |
# 2. Redistributions in binary form must reproduce the above |
|
14 |
# copyright notice, this list of conditions and the following |
|
15 |
# disclaimer in the documentation and/or other materials |
|
16 |
# provided with the distribution. |
|
17 |
# |
|
18 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
|
19 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
|
20 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
|
21 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
|
22 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
23 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
24 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
|
25 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
|
26 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
|
27 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
|
28 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
|
29 |
# POSSIBILITY OF SUCH DAMAGE. |
|
30 |
# |
|
31 |
# The views and conclusions contained in the software and |
|
32 |
# documentation are those of the authors and should not be |
|
33 |
# interpreted as representing official policies, either expressed |
|
34 |
# or implied, of GRNET S.A. |
|
35 |
|
|
36 |
from sqlalchemy import Table |
|
37 |
from sqlalchemy.sql import select, and_ |
|
38 |
|
|
39 |
from binascii import hexlify |
|
40 |
|
|
41 |
from pithos.backends.lib.hashfiler import Blocker |
|
42 |
from pithos.backends.lib.sqlalchemy import Node |
|
43 |
|
|
44 |
from django.conf import settings |
|
45 |
|
|
46 |
from pithos.backends.modular import CLUSTER_NORMAL, CLUSTER_HISTORY, CLUSTER_DELETED |
|
47 |
from pithos.backends.lib.sqlalchemy.node import Node, ROOTNODE |
|
48 |
|
|
49 |
from pithos.tools.lib.transfer import upload |
|
50 |
from pithos.tools.lib.hashmap import HashMap |
|
51 |
from pithos.tools.lib.client import Fault |
|
52 |
|
|
53 |
from migrate import Migration, Cache |
|
54 |
|
|
55 |
from calendar import timegm |
|
56 |
from decimal import Decimal |
|
57 |
from collections import defaultdict |
|
58 |
|
|
59 |
import json |
|
60 |
import os |
|
61 |
import sys |
|
62 |
import hashlib |
|
63 |
import mimetypes |
|
64 |
import time |
|
65 |
import datetime |
|
66 |
|
|
67 |
(ID, CREATIONDATE, MODIFICATIONDATE, DELETED, ICON, NAME, VERSION, CREATEDBY_ID, MODIFIEDBY_ID, OWNER_ID, PARENT_ID, READFORALL, SHARED, USER) = range(14) |
|
68 |
|
|
69 |
class ObjectMigration(Migration): |
|
70 |
def __init__(self, old_db, db, f): |
|
71 |
Migration.__init__(self, old_db) |
|
72 |
self.cache = Cache(db) |
|
73 |
|
|
74 |
def create_node(self, username, container, object): |
|
75 |
node = self.backend.node.node_lookup(object) |
|
76 |
if not node: |
|
77 |
parent_path = '%s/%s' %(username, container) |
|
78 |
parent_node = self.backend.node.node_lookup(parent_path) |
|
79 |
if not parent_node: |
|
80 |
raise Exception('Missing node') |
|
81 |
node = self.backend.node.node_create(parent_node, object) |
|
82 |
return node |
|
83 |
|
|
84 |
def create_history(self, header_id, node_id, deleted=False): |
|
85 |
i = 0 |
|
86 |
map = HashMap(self.backend.block_size, self.backend.hash_algorithm) |
|
87 |
v = [] |
|
88 |
stored_versions = self.backend.node.node_get_versions(node_id, ['mtime']) |
|
89 |
stored_versions_mtime = [datetime.datetime.utcfromtimestamp(elem[0]) for elem in stored_versions] |
|
90 |
for t, rowcount in self.retrieve_node_versions(header_id): |
|
91 |
size, modyfied_by, filepath, mimetype, mdate = t |
|
92 |
if mdate in stored_versions_mtime: |
|
93 |
continue |
|
94 |
cluster = CLUSTER_HISTORY if i < rowcount - 1 else CLUSTER_NORMAL |
|
95 |
cluster = cluster if not deleted else CLUSTER_DELETED |
|
96 |
hash = self.cache.get(filepath) |
|
97 |
if hash == None: |
|
98 |
raise Exception("Missing hash") |
|
99 |
args = node_id, hash, size, modyfied_by, cluster, mimetype, mdate |
|
100 |
v.append(self.create_version(*args)) |
|
101 |
i += 1 |
|
102 |
return v |
|
103 |
|
|
104 |
def create_version(self, node_id, hash, size, modyfied_by, cluster, mimetype, mdate): |
|
105 |
args = (node_id, hash, size, None, modyfied_by, cluster) |
|
106 |
serial = self.backend.node.version_create(*args)[0] |
|
107 |
meta = {'hash':hash, |
|
108 |
'content-type':mimetype} |
|
109 |
self.backend.node.attribute_set(serial, ((k, v) for k, v in meta.iteritems())) |
|
110 |
timestamp = timegm(mdate.timetuple()) |
|
111 |
microseconds = mdate.time().microsecond |
|
112 |
values = timestamp, microseconds, serial |
|
113 |
f.write('update versions set mtime=\'%10d.%6d\' where serial=%s;' %values) |
|
114 |
return serial |
|
115 |
|
|
116 |
def create_tags(self, header_id, node_id, vserials): |
|
117 |
tags = self.retrieve_tags(header_id) |
|
118 |
if not tags: |
|
119 |
return |
|
120 |
for v in vserials: |
|
121 |
self.backend.node.attribute_set(v, (('X-Object-Meta-Tag', tags),)) |
|
122 |
|
|
123 |
def create_permissions(self, fid, path, owner, is_folder=True): |
|
124 |
fpath, fpermissions = self.backend.permissions.access_inherit(path) |
|
125 |
permissions = self.retrieve_permissions(fid, is_folder) |
|
126 |
if not fpermissions: |
|
127 |
keys = ('read', 'write') |
|
128 |
for k in keys: |
|
129 |
if owner in permissions[k]: |
|
130 |
permissions[k].remove(owner) |
|
131 |
self.backend.permissions.access_set(path, permissions) |
|
132 |
else: |
|
133 |
keys = ('read', 'write') |
|
134 |
common_p = {} |
|
135 |
for k in keys: |
|
136 |
if owner in permissions[k]: |
|
137 |
permissions[k].remove(owner) |
|
138 |
common = set(fpermissions[k]).intersection(set(permissions[k])) |
|
139 |
common_p[k] = list(common) |
|
140 |
#keep only the common permissions |
|
141 |
#trade off for securing access only to explicitly authorized users |
|
142 |
self.backend.permissions.access_set(fpath, common_p) |
|
143 |
|
|
144 |
def create_objects(self): |
|
145 |
for t in self.retrieve_current_nodes(): |
|
146 |
username, headerid, folderid, filename, deleted, filepath, mimetype, public, owner_id = t |
|
147 |
containers = ['pithos', 'trash'] |
|
148 |
|
|
149 |
for c in containers: |
|
150 |
#create container if it does not exist |
|
151 |
try: |
|
152 |
self.backend._lookup_container(username, c) |
|
153 |
except NameError, e: |
|
154 |
self.backend.put_container(username, username, c) |
|
155 |
|
|
156 |
container = 'pithos' if not deleted else 'trash' |
|
157 |
path = self.build_path(folderid) |
|
158 |
#create node |
|
159 |
object = '%s/%s' %(username, container) |
|
160 |
object = '%s/%s/%s' %(object, path, filename) if path else '%s/%s' %(object, filename) |
|
161 |
args = username, container, object |
|
162 |
nodeid = self.create_node(*args) |
|
163 |
#create node history |
|
164 |
vserials = self.create_history(headerid, nodeid, deleted) |
|
165 |
#set object tags |
|
166 |
self.create_tags(headerid, nodeid, vserials) |
|
167 |
#set object's publicity |
|
168 |
if public: |
|
169 |
self.backend.permissions.public_set(object) |
|
170 |
#set object's permissions |
|
171 |
self.create_permissions(headerid, object, username, is_folder=False) |
|
172 |
|
|
173 |
def build_path(self, child_id): |
|
174 |
folder = Table('folder', self.metadata, autoload=True) |
|
175 |
user = Table('gss_user', self.metadata, autoload=True) |
|
176 |
j = folder.join(user, folder.c.owner_id == user.c.id) |
|
177 |
s = select([folder, user.c.username], from_obj=j) |
|
178 |
s = s.where(folder.c.id == child_id) |
|
179 |
s.order_by(folder.c.modificationdate) |
|
180 |
rp = self.conn.execute(s) |
|
181 |
t = rp.fetchone() |
|
182 |
md5 = hashlib.md5() |
|
183 |
hash = md5.hexdigest().lower() |
|
184 |
size = 0 |
|
185 |
if not t[PARENT_ID]: |
|
186 |
return '' |
|
187 |
else: |
|
188 |
container_path = t[USER] |
|
189 |
container_path += '/trash' if t[DELETED] else '/pithos' |
|
190 |
parent_node = self.backend.node.node_lookup(container_path) |
|
191 |
if not parent_node: |
|
192 |
raise Exception('Missing node:', container_path) |
|
193 |
parent_path = self.build_path(t[PARENT_ID]) |
|
194 |
path = '%s/%s/%s' %(container_path, parent_path, t[NAME]) if parent_path else '%s/%s' %(container_path, t[NAME]) |
|
195 |
node = self.backend.node.node_lookup(path) |
|
196 |
if not node: |
|
197 |
node = self.backend.node.node_create(parent_node, path) |
|
198 |
if not node: |
|
199 |
raise Exception('Unable to create node:', path) |
|
200 |
|
|
201 |
#create versions |
|
202 |
v = self.create_version(node, hash, size, t[USER], CLUSTER_NORMAL, 'application/directory', t[CREATIONDATE]) |
|
203 |
if t[CREATIONDATE] != t[MODIFICATIONDATE]: |
|
204 |
self.backend.node.version_recluster(v, CLUSTER_HISTORY) |
|
205 |
self.create_version(node, hash, size, t[USER], CLUSTER_NORMAL, 'application/directory', t[MODIFICATIONDATE]) |
|
206 |
|
|
207 |
#set permissions |
|
208 |
self.create_permissions(t[ID], path, t[USER], is_folder=True) |
|
209 |
return '%s/%s' %(parent_path, t[NAME]) if parent_path else t[NAME] |
|
210 |
|
|
211 |
def retrieve_current_nodes(self): |
|
212 |
fileheader = Table('fileheader', self.metadata, autoload=True) |
|
213 |
filebody = Table('filebody', self.metadata, autoload=True) |
|
214 |
folder = Table('folder', self.metadata, autoload=True) |
|
215 |
gss_user = Table('gss_user', self.metadata, autoload=True) |
|
216 |
j = filebody.join(fileheader, filebody.c.id == fileheader.c.currentbody_id) |
|
217 |
j = j.join(folder, fileheader.c.folder_id == folder.c.id) |
|
218 |
j = j.join(gss_user, fileheader.c.owner_id == gss_user.c.id) |
|
219 |
s = select([gss_user.c.username, fileheader.c.id, fileheader.c.folder_id, |
|
220 |
fileheader.c.name, fileheader.c.deleted, |
|
221 |
filebody.c.storedfilepath, filebody.c.mimetype, |
|
222 |
fileheader.c.readforall, fileheader.c.owner_id], from_obj=j) |
|
223 |
rp = self.conn.execute(s) |
|
224 |
object = rp.fetchone() |
|
225 |
while object: |
|
226 |
yield object |
|
227 |
object = rp.fetchone() |
|
228 |
rp.close() |
|
229 |
|
|
230 |
def retrieve_node_versions(self, header_id): |
|
231 |
filebody = Table('filebody', self.metadata, autoload=True) |
|
232 |
gss_user = Table('gss_user', self.metadata, autoload=True) |
|
233 |
j = filebody.join(gss_user, filebody.c.modifiedby_id == gss_user.c.id) |
|
234 |
s = select([filebody.c.filesize, gss_user.c.username, |
|
235 |
filebody.c.storedfilepath, filebody.c.mimetype, |
|
236 |
filebody.c.modificationdate], from_obj=j) |
|
237 |
s = s.where(filebody.c.header_id == header_id) |
|
238 |
s = s.order_by(filebody.c.version) |
|
239 |
rp = self.conn.execute(s) |
|
240 |
version = rp.fetchone() |
|
241 |
while version: |
|
242 |
yield version, rp.rowcount |
|
243 |
version = rp.fetchone() |
|
244 |
rp.close() |
|
245 |
|
|
246 |
def retrieve_tags(self, header_id): |
|
247 |
filetag = Table('filetag', self.metadata, autoload=True) |
|
248 |
s = select([filetag.c.tag], filetag.c.fileid == header_id) |
|
249 |
rp = self.conn.execute(s) |
|
250 |
tags = rp.fetchall() if rp.returns_rows else [] |
|
251 |
tags = [elem[0] for elem in tags] |
|
252 |
rp.close() |
|
253 |
return ','.join(tags) if tags else '' |
|
254 |
|
|
255 |
def retrieve_permissions(self, id, is_folder=True): |
|
256 |
permissions = {} |
|
257 |
if is_folder: |
|
258 |
ftable = Table('folder_permission', self.metadata, autoload=True) |
|
259 |
else: |
|
260 |
ftable = Table('fileheader_permission', self.metadata, autoload=True) |
|
261 |
permission = Table('permission', self.metadata, autoload=True) |
|
262 |
group = Table('gss_group', self.metadata, autoload=True) |
|
263 |
user = Table('gss_user', self.metadata, autoload=True) |
|
264 |
j = ftable.join(permission, ftable.c.permissions_id == permission.c.id) |
|
265 |
j1 = j.join(group, group.c.id == permission.c.group_id) |
|
266 |
j2 = j.join(user, user.c.id == permission.c.user_id) |
|
267 |
|
|
268 |
permissions = defaultdict(list) |
|
269 |
|
|
270 |
def _get_permissions(self, action='read', get_groups=True): |
|
271 |
if get_groups: |
|
272 |
col, j = group.c.name, j1 |
|
273 |
cond2 = permission.c.group_id != None |
|
274 |
else: |
|
275 |
col, j = user.c.username, j2 |
|
276 |
cond2 = permission.c.user_id != None |
|
277 |
s = select([col], from_obj=j) |
|
278 |
if is_folder: |
|
279 |
s = s.where(ftable.c.folder_id == id) |
|
280 |
else: |
|
281 |
s = s.where(ftable.c.fileheader_id == id) |
|
282 |
if action == 'read': |
|
283 |
cond1 = permission.c.read == True |
|
284 |
else: |
|
285 |
cond1 = permission.c.write == True |
|
286 |
s = s.where(and_(cond1, cond2)) |
|
287 |
print '>', s, s.compile().params |
|
288 |
rp = self.conn.execute(s) |
|
289 |
p = permissions[action].extend([e[0] for e in rp.fetchall()]) |
|
290 |
rp.close() |
|
291 |
return p |
|
292 |
|
|
293 |
#get object read groups |
|
294 |
_get_permissions(self, action='read', get_groups=True) |
|
295 |
|
|
296 |
#get object read users |
|
297 |
_get_permissions(self, action='read', get_groups=False) |
|
298 |
|
|
299 |
#get object write groups |
|
300 |
_get_permissions(self, action='write', get_groups=True) |
|
301 |
|
|
302 |
#get object write groups |
|
303 |
_get_permissions(self, action='write', get_groups=False) |
|
304 |
|
|
305 |
return permissions |
|
306 |
|
|
307 |
if __name__ == "__main__": |
|
308 |
old_db = '' |
|
309 |
db = '' |
|
310 |
|
|
311 |
f = open('fixdates.sql', 'w') |
|
312 |
ot = ObjectMigration(old_db, db, f) |
|
313 |
ot.create_objects() |
|
314 |
f.close() |
|
315 |
|
|
316 |
|
b/contrib/migrate-users | ||
---|---|---|
1 |
#!/usr/bin/env python |
|
2 |
|
|
3 |
# Copyright 2011-2012 GRNET S.A. All rights reserved. |
|
4 |
# |
|
5 |
# Redistribution and use in source and binary forms, with or |
|
6 |
# without modification, are permitted provided that the following |
|
7 |
# conditions are met: |
|
8 |
# |
|
9 |
# 1. Redistributions of source code must retain the above |
|
10 |
# copyright notice, this list of conditions and the following |
|
11 |
# disclaimer. |
|
12 |
# |
|
13 |
# 2. Redistributions in binary form must reproduce the above |
|
14 |
# copyright notice, this list of conditions and the following |
|
15 |
# disclaimer in the documentation and/or other materials |
|
16 |
# provided with the distribution. |
|
17 |
# |
|
18 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
|
19 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
|
20 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
|
21 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
|
22 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
23 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
24 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
|
25 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
|
26 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
|
27 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
|
28 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
|
29 |
# POSSIBILITY OF SUCH DAMAGE. |
|
30 |
# |
|
31 |
# The views and conclusions contained in the software and |
|
32 |
# documentation are those of the authors and should not be |
|
33 |
# interpreted as representing official policies, either expressed |
|
34 |
# or implied, of GRNET S.A. |
|
35 |
|
|
36 |
from sqlalchemy import Table |
|
37 |
from sqlalchemy.sql import select |
|
38 |
|
|
39 |
from pithos.im.models import User |
|
40 |
|
|
41 |
from migrate import Migration |
|
42 |
|
|
43 |
import base64 |
|
44 |
|
|
45 |
class UserMigration(Migration): |
|
46 |
def __init__(self, db): |
|
47 |
Migration.__init__(self, db) |
|
48 |
self.gss_users = Table('gss_user', self.metadata, autoload=True) |
|
49 |
|
|
50 |
def execute(self): |
|
51 |
for u in self.retrieve_users(): |
|
52 |
user = User() |
|
53 |
user.pk = u['id'] |
|
54 |
user.uniq = u['username'] |
|
55 |
user.realname = u['name'] |
|
56 |
user.affiliation = u['homeorganization'] if u['homeorganization'] else '' |
|
57 |
user.auth_token = base64.b64encode(u['authtoken']) |
|
58 |
user.auth_token_created = u['creationdate'] |
|
59 |
user.auth_token_expires = u['authtokenexpirydate'] |
|
60 |
user.created = u['creationdate'] |
|
61 |
user.updated = u['modificationdate'] |
|
62 |
user.email = u['email'] |
|
63 |
user.active = 'ACTIVE' if u['active'] else 'SUSPENDED' |
|
64 |
print '#', user |
|
65 |
user.save(update_timestamps=False) |
|
66 |
|
|
67 |
#create user groups |
|
68 |
for (owner, group, members) in self.retrieve_groups(u['username']): |
|
69 |
self.backend.permissions.group_addmany(owner, group, members) |
|
70 |
|
|
71 |
|
|
72 |
def retrieve_users(self): |
|
73 |
s = self.gss_users.select() |
|
74 |
rp = self.conn.execute(s) |
|
75 |
user = rp.fetchone() |
|
76 |
while user: |
|
77 |
yield user |
|
78 |
user = rp.fetchone() |
|
79 |
rp.close() |
|
80 |
|
|
81 |
def retrieve_groups(self, owner): |
|
82 |
gss_group = Table('gss_group', self.metadata, autoload=True) |
|
83 |
gss_user = Table('gss_user', self.metadata, autoload=True) |
|
84 |
group_user = Table('gss_group_gss_user', self.metadata, autoload=True) |
|
85 |
j1 = gss_group.join(gss_user, gss_group.c.owner_id == gss_user.c.id) |
|
86 |
j2 = group_user.join(gss_user, group_user.c.members_id == gss_user.c.id) |
|
87 |
s = select([gss_group.c.id, gss_group.c.name, gss_user.c.username], from_obj=j1) |
|
88 |
s = s.where(gss_user.c.username == owner) |
|
89 |
rp = self.conn.execute(s) |
|
90 |
gr = rp.fetchone() |
|
91 |
while gr: |
|
92 |
id, group, owner = gr |
|
93 |
s = select([gss_user.c.username], from_obj=j2) |
|
94 |
s = s.where(group_user.c.groupsmember_id == id) |
|
95 |
rp2 = self.conn.execute(s) |
|
96 |
members = rp2.fetchall() |
|
97 |
rp2.close() |
|
98 |
yield owner, group, (m[0] for m in members) |
|
99 |
gr = rp.fetchone() |
|
100 |
rp.close() |
|
101 |
|
|
102 |
if __name__ == "__main__": |
|
103 |
db = '' |
|
104 |
m = UserMigration(db) |
|
105 |
m.execute() |
b/contrib/migrate.py | ||
---|---|---|
1 |
#!/usr/bin/env python |
|
2 |
|
|
3 |
# Copyright 2011-2012 GRNET S.A. All rights reserved. |
|
4 |
# |
|
5 |
# Redistribution and use in source and binary forms, with or |
|
6 |
# without modification, are permitted provided that the following |
|
7 |
# conditions are met: |
|
8 |
# |
|
9 |
# 1. Redistributions of source code must retain the above |
|
10 |
# copyright notice, this list of conditions and the following |
|
11 |
# disclaimer. |
|
12 |
# |
|
13 |
# 2. Redistributions in binary form must reproduce the above |
|
14 |
# copyright notice, this list of conditions and the following |
|
15 |
# disclaimer in the documentation and/or other materials |
|
16 |
# provided with the distribution. |
|
17 |
# |
|
18 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
|
19 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
|
20 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
|
21 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
|
22 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
23 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
24 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
|
25 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
|
26 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
|
27 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
|
28 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
|
29 |
# POSSIBILITY OF SUCH DAMAGE. |
|
30 |
# |
|
31 |
# The views and conclusions contained in the software and |
|
32 |
# documentation are those of the authors and should not be |
|
33 |
# interpreted as representing official policies, either expressed |
|
34 |
# or implied, of GRNET S.A. |
|
35 |
|
|
36 |
from sqlalchemy import create_engine |
|
37 |
from sqlalchemy import Table, Column, String, MetaData |
|
38 |
from sqlalchemy.sql import select |
|
39 |
|
|
40 |
from django.conf import settings |
|
41 |
|
|
42 |
from pithos.backends.modular import ModularBackend |
|
43 |
|
|
44 |
|
|
45 |
class Migration(object): |
|
46 |
def __init__(self, db): |
|
47 |
self.engine = create_engine(db) |
|
48 |
self.metadata = MetaData(self.engine) |
|
49 |
#self.engine.echo = True |
|
50 |
self.conn = self.engine.connect() |
|
51 |
|
|
52 |
options = getattr(settings, 'BACKEND', None)[1] |
|
53 |
self.backend = ModularBackend(*options) |
|
54 |
|
|
55 |
def execute(self): |
|
56 |
pass |
|
57 |
|
|
58 |
|
|
59 |
class Cache(): |
|
60 |
def __init__(self, db): |
|
61 |
self.engine = create_engine(db) |
|
62 |
metadata = MetaData(self.engine) |
|
63 |
|
|
64 |
columns = [] |
|
65 |
columns.append(Column('path', String(2048), primary_key=True)) |
|
66 |
columns.append(Column('hash', String(255))) |
|
67 |
self.files = Table('files', metadata, *columns) |
|
68 |
self.conn = self.engine.connect() |
|
69 |
self.engine.echo = True |
|
70 |
metadata.create_all(self.engine) |
|
71 |
|
|
72 |
def put(self, path, hash): |
|
73 |
# Insert or replace. |
|
74 |
s = self.files.delete().where(self.files.c.path == path) |
|
75 |
r = self.conn.execute(s) |
|
76 |
r.close() |
|
77 |
s = self.files.insert() |
|
78 |
r = self.conn.execute(s, {'path': path, 'hash': hash}) |
|
79 |
r.close() |
|
80 |
|
|
81 |
def get(self, path): |
|
82 |
s = select([self.files.c.hash], self.files.c.path == path) |
|
83 |
r = self.conn.execute(s) |
|
84 |
l = r.fetchone() |
|
85 |
r.close() |
|
86 |
if not l: |
|
87 |
return l |
|
88 |
return l[0] |
b/contrib/stats-calculator.sql | ||
---|---|---|
1 |
# Top level |
|
2 |
create temporary table tmp_stats as select 0 as "level", 0 as "node", 0 as "parent", count(serial) as "population", sum(size) as "bytes", max(mtime) as "mtime", cluster, false as "final" from versions group by cluster; |
|
3 |
|
|
4 |
# Account level |
|
5 |
insert into tmp_stats select 1 as "level", n.node, n.parent, count(v.serial) as "population", sum(v.size) as "bytes", max(v.mtime) as "mtime", cluster, false as "final" from versions v, nodes n where n.node=v.node and n.parent=0 and n.node!=0 group by node, cluster; |
|
6 |
create temporary table tmp_nodes select distinct node, level from tmp_stats where level=1; |
|
7 |
|
|
8 |
# Container level |
|
9 |
insert into tmp_stats select 2 as "level", n.node, n.parent, count(v.serial) as "population", sum(v.size) as "bytes", max(v.mtime) as "mtime", cluster, false as "final" from versions v, nodes n where n.node=v.node and n.parent in (select node from tmp_nodes where level=1) group by node, cluster; |
|
10 |
insert into tmp_nodes select distinct node, level from tmp_stats where level=2; |
|
11 |
|
|
12 |
# Object level |
|
13 |
insert into tmp_stats select 3 as "level", n.node, n.parent, count(v.serial) as "population", sum(v.size) as "bytes", max(v.mtime) as "mtime", cluster, false as "final" from versions v, nodes n where n.node=v.node and n.parent in (select node from tmp_nodes where level=2) group by node, cluster; |
|
14 |
insert into tmp_nodes select distinct node, level from tmp_stats where level=3; |
|
15 |
|
|
16 |
# Update containers |
|
17 |
create table tmp_sums as select parent as "node", sum(population) as "population", sum(bytes) as "bytes", max(mtime) as "mtime", cluster from tmp_stats where level=3 group by parent, cluster; |
|
18 |
insert into tmp_stats select 2 as "level", n.node, n.parent, t.population, t.bytes, t.mtime, t.cluster, true as "final" from tmp_sums t, nodes n where n.node=t.node; |
|
19 |
drop table tmp_sums; |
|
20 |
|
|
21 |
# Update accounts |
|
22 |
create table tmp_sums as select parent as "node", sum(bytes) as "bytes", max(mtime) as "mtime", cluster from tmp_stats where level=2 group by parent, cluster; |
|
23 |
create table tmp_population as select parent as "node", sum(population) as "population", cluster from tmp_stats where level=2 and final=false group by parent, cluster; |
|
24 |
insert into tmp_stats select 1 as "level", t.node, 0 as "parent", IFNULL(p.population, 0) as "population", t.bytes, t.mtime, t.cluster, true as "final" from tmp_sums t left join tmp_population p on p.node=t.node and p.cluster=t.cluster; |
|
25 |
drop table tmp_sums; |
|
26 |
drop table tmp_population; |
|
27 |
|
|
28 |
# Update top level |
|
29 |
create table tmp_sums as select parent as "node", sum(bytes) as "bytes", max(mtime) as "mtime", cluster from tmp_stats where level=1 group by parent, cluster; |
|
30 |
create table tmp_population as select parent as "node", sum(population) as "population", cluster from tmp_stats where level=1 and final=false group by parent, cluster; |
|
31 |
insert into tmp_stats select 0 as "level", t.node, 0 as "parent", IFNULL(p.population, 0) as "population", t.bytes, t.mtime, t.cluster, true as "final" from tmp_sums t left join tmp_population p on p.node=t.node and p.cluster=t.cluster; |
|
32 |
drop table tmp_sums; |
|
33 |
drop table tmp_population; |
|
34 |
|
|
35 |
# Clean up |
|
36 |
drop table tmp_nodes; |
|
37 |
delete from tmp_stats where final=false; |
/dev/null | ||
---|---|---|
1 |
import os |
|
2 |
import sys |
|
3 |
os.environ['DJANGO_SETTINGS_MODULE'] = 'synnefo.settings' |
|
4 |
|
|
5 |
from django.conf import settings |
|
6 |
from astakos.im.models import AstakosUser |
|
7 |
|
|
8 |
def duplicate_users(): |
|
9 |
for u in AstakosUser.objects.filter(): |
|
10 |
if AstakosUser.objects.filter(email__iexact=u.email).count() > 1: |
|
11 |
print AstakosUser.objects.filter(email__iexact=u.email).values('pk', |
|
12 |
'email', |
|
13 |
'is_active') |
|
14 |
|
|
15 |
if len(sys.argv) == 2: |
|
16 |
pk = int(sys.argv[1]) |
|
17 |
user = AstakosUser.objects.get(pk=pk) |
|
18 |
if AstakosUser.objects.filter(email__iexact=user.email).count() == 1: |
|
19 |
print "No duplicate emails found for user %s" % (user) |
|
20 |
exit() |
|
21 |
user = AstakosUser.objects.get(pk=pk) |
|
22 |
print "Deleting user %r" % (user) |
|
23 |
user.delete() |
|
24 |
exit() |
|
25 |
else: |
|
26 |
duplicate_users() |
|
27 |
|
/dev/null | ||
---|---|---|
1 |
#!/usr/bin/env python |
|
2 |
|
|
3 |
# Copyright 2011-2012 GRNET S.A. All rights reserved. |
|
4 |
# |
|
5 |
# Redistribution and use in source and binary forms, with or |
|
6 |
# without modification, are permitted provided that the following |
|
7 |
# conditions are met: |
|
8 |
# |
|
9 |
# 1. Redistributions of source code must retain the above |
|
10 |
# copyright notice, this list of conditions and the following |
|
11 |
# disclaimer. |
|
12 |
# |
|
13 |
# 2. Redistributions in binary form must reproduce the above |
|
14 |
# copyright notice, this list of conditions and the following |
|
15 |
# disclaimer in the documentation and/or other materials |
|
16 |
# provided with the distribution. |
|
17 |
# |
|
18 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
|
19 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
|
20 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
|
21 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
|
22 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
23 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
24 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
|
25 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
|
26 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
|
27 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
|
28 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
|
29 |
# POSSIBILITY OF SUCH DAMAGE. |
|
30 |
# |
|
31 |
# The views and conclusions contained in the software and |
|
32 |
# documentation are those of the authors and should not be |
|
33 |
# interpreted as representing official policies, either expressed |
|
34 |
# or implied, of GRNET S.A. |
|
35 |
|
|
36 |
from binascii import hexlify |
|
37 |
|
|
38 |
from sqlalchemy import Table |
|
39 |
from sqlalchemy.sql import select |
|
40 |
|
|
41 |
from pithos import settings |
|
42 |
from pithos.backends.modular import ModularBackend |
|
43 |
|
|
44 |
from pithos.tools.lib.hashmap import HashMap |
|
45 |
|
|
46 |
from migrate import Migration, Cache |
|
47 |
|
|
48 |
import os |
|
49 |
|
|
50 |
class DataMigration(Migration): |
|
51 |
def __init__(self, pithosdb, db): |
|
52 |
Migration.__init__(self, pithosdb) |
|
53 |
self.cache = Cache(db) |
|
54 |
|
|
55 |
def retrieve_files(self): |
|
56 |
# Loop for all available files. |
|
57 |
filebody = Table('filebody', self.metadata, autoload=True) |
|
58 |
s = select([filebody.c.storedfilepath]) |
|
59 |
rp = self.conn.execute(s) |
|
60 |
path = rp.fetchone() |
|
61 |
while path: |
|
62 |
yield path |
|
63 |
path = rp.fetchone() |
|
64 |
rp.close() |
|
65 |
|
|
66 |
def execute(self): |
|
67 |
blocksize = self.backend.block_size |
|
68 |
blockhash = self.backend.hash_algorithm |
|
69 |
|
|
70 |
for (path,) in self.retrieve_files(): |
|
71 |
map = HashMap(blocksize, blockhash) |
|
72 |
try: |
|
73 |
map.load(open(path)) |
|
74 |
except Exception, e: |
|
75 |
print e |
|
76 |
continue |
|
77 |
hash = hexlify(map.hash()) |
|
78 |
|
|
79 |
if hash != self.cache.get(path): |
|
80 |
missing = self.backend.blocker.block_ping(map) # XXX Backend hack... |
|
81 |
status = '[>] ' + path |
|
82 |
if missing: |
|
83 |
status += ' - %d block(s) missing' % len(missing) |
|
84 |
with open(path) as fp: |
|
85 |
for h in missing: |
|
86 |
offset = map.index(h) * blocksize |
|
87 |
fp.seek(offset) |
|
88 |
block = fp.read(blocksize) |
|
89 |
self.backend.put_block(block) |
|
90 |
else: |
|
91 |
status += ' - no blocks missing' |
|
92 |
self.cache.put(path, hash) |
|
93 |
else: |
|
94 |
status = '[-] ' + path |
|
95 |
print status |
|
96 |
|
|
97 |
if __name__ == "__main__": |
|
98 |
pithosdb = 'postgresql://gss@127.0.0.1/pithos' |
|
99 |
db = 'sqlite:///migrate.db' |
|
100 |
|
|
101 |
dt = DataMigration(pithosdb, db) |
|
102 |
dt.execute() |
/dev/null | ||
---|---|---|
1 |
#!/usr/bin/env python |
|
2 |
|
|
3 |
# Copyright 2011-2012 GRNET S.A. All rights reserved. |
|
4 |
# |
|
5 |
# Redistribution and use in source and binary forms, with or |
|
6 |
# without modification, are permitted provided that the following |
|
7 |
# conditions are met: |
|
8 |
# |
|
9 |
# 1. Redistributions of source code must retain the above |
|
10 |
# copyright notice, this list of conditions and the following |
|
11 |
# disclaimer. |
|
12 |
# |
|
13 |
# 2. Redistributions in binary form must reproduce the above |
|
14 |
# copyright notice, this list of conditions and the following |
|
15 |
# disclaimer in the documentation and/or other materials |
|
16 |
# provided with the distribution. |
|
17 |
# |
|
18 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
|
19 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
|
20 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
|
21 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
|
22 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
23 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
24 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
|
25 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
|
26 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
|
27 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
|
28 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
|
29 |
# POSSIBILITY OF SUCH DAMAGE. |
|
30 |
# |
|
31 |
# The views and conclusions contained in the software and |
|
32 |
# documentation are those of the authors and should not be |
|
33 |
# interpreted as representing official policies, either expressed |
|
34 |
# or implied, of GRNET S.A. |
|
35 |
|
|
36 |
from sqlalchemy import Table |
|
37 |
from sqlalchemy.sql import select, and_ |
|
38 |
|
|
39 |
from binascii import hexlify |
|
40 |
|
|
41 |
from pithos.backends.lib.hashfiler import Blocker |
|
42 |
from pithos.backends.lib.sqlalchemy import Node |
|
43 |
|
|
44 |
from django.conf import settings |
|
45 |
|
|
46 |
from pithos.backends.modular import CLUSTER_NORMAL, CLUSTER_HISTORY, CLUSTER_DELETED |
|
47 |
from pithos.backends.lib.sqlalchemy.node import Node, ROOTNODE |
|
48 |
|
|
49 |
from pithos.tools.lib.transfer import upload |
|
50 |
from pithos.tools.lib.hashmap import HashMap |
|
51 |
from pithos.tools.lib.client import Fault |
|
52 |
|
|
53 |
from migrate import Migration, Cache |
|
54 |
|
|
55 |
from calendar import timegm |
|
56 |
from decimal import Decimal |
|
57 |
from collections import defaultdict |
|
58 |
|
|
59 |
import json |
|
60 |
import os |
|
61 |
import sys |
|
62 |
import hashlib |
|
63 |
import mimetypes |
|
64 |
import time |
|
65 |
import datetime |
|
66 |
|
|
67 |
(ID, CREATIONDATE, MODIFICATIONDATE, DELETED, ICON, NAME, VERSION, CREATEDBY_ID, MODIFIEDBY_ID, OWNER_ID, PARENT_ID, READFORALL, SHARED, USER) = range(14) |
|
68 |
|
|
69 |
class ObjectMigration(Migration): |
|
70 |
def __init__(self, old_db, db, f): |
|
71 |
Migration.__init__(self, old_db) |
|
72 |
self.cache = Cache(db) |
|
73 |
|
|
74 |
def create_node(self, username, container, object): |
|
75 |
node = self.backend.node.node_lookup(object) |
|
76 |
if not node: |
|
77 |
parent_path = '%s/%s' %(username, container) |
|
78 |
parent_node = self.backend.node.node_lookup(parent_path) |
|
79 |
if not parent_node: |
|
80 |
raise Exception('Missing node') |
|
81 |
node = self.backend.node.node_create(parent_node, object) |
|
82 |
return node |
|
83 |
|
|
84 |
def create_history(self, header_id, node_id, deleted=False): |
|
85 |
i = 0 |
|
86 |
map = HashMap(self.backend.block_size, self.backend.hash_algorithm) |
|
87 |
v = [] |
|
88 |
stored_versions = self.backend.node.node_get_versions(node_id, ['mtime']) |
|
89 |
stored_versions_mtime = [datetime.datetime.utcfromtimestamp(elem[0]) for elem in stored_versions] |
|
90 |
for t, rowcount in self.retrieve_node_versions(header_id): |
|
91 |
size, modyfied_by, filepath, mimetype, mdate = t |
|
92 |
if mdate in stored_versions_mtime: |
|
93 |
continue |
|
94 |
cluster = CLUSTER_HISTORY if i < rowcount - 1 else CLUSTER_NORMAL |
|
95 |
cluster = cluster if not deleted else CLUSTER_DELETED |
|
96 |
hash = self.cache.get(filepath) |
|
97 |
if hash == None: |
|
98 |
raise Exception("Missing hash") |
|
99 |
args = node_id, hash, size, modyfied_by, cluster, mimetype, mdate |
|
100 |
v.append(self.create_version(*args)) |
|
101 |
i += 1 |
|
102 |
return v |
|
103 |
|
|
104 |
def create_version(self, node_id, hash, size, modyfied_by, cluster, mimetype, mdate): |
|
105 |
args = (node_id, hash, size, None, modyfied_by, cluster) |
|
106 |
serial = self.backend.node.version_create(*args)[0] |
|
107 |
meta = {'hash':hash, |
|
108 |
'content-type':mimetype} |
|
109 |
self.backend.node.attribute_set(serial, ((k, v) for k, v in meta.iteritems())) |
|
110 |
timestamp = timegm(mdate.timetuple()) |
|
111 |
microseconds = mdate.time().microsecond |
|
112 |
values = timestamp, microseconds, serial |
|
113 |
f.write('update versions set mtime=\'%10d.%6d\' where serial=%s;' %values) |
|
114 |
return serial |
|
115 |
|
|
116 |
def create_tags(self, header_id, node_id, vserials): |
|
117 |
tags = self.retrieve_tags(header_id) |
|
118 |
if not tags: |
|
119 |
return |
|
120 |
for v in vserials: |
|
121 |
self.backend.node.attribute_set(v, (('X-Object-Meta-Tag', tags),)) |
|
122 |
|
|
123 |
def create_permissions(self, fid, path, owner, is_folder=True): |
|
124 |
fpath, fpermissions = self.backend.permissions.access_inherit(path) |
|
125 |
permissions = self.retrieve_permissions(fid, is_folder) |
|
126 |
if not fpermissions: |
|
127 |
keys = ('read', 'write') |
|
128 |
for k in keys: |
|
129 |
if owner in permissions[k]: |
|
130 |
permissions[k].remove(owner) |
|
131 |
self.backend.permissions.access_set(path, permissions) |
|
132 |
else: |
|
133 |
keys = ('read', 'write') |
|
134 |
common_p = {} |
|
135 |
for k in keys: |
|
136 |
if owner in permissions[k]: |
|
137 |
permissions[k].remove(owner) |
|
138 |
common = set(fpermissions[k]).intersection(set(permissions[k])) |
|
139 |
common_p[k] = list(common) |
|
140 |
#keep only the common permissions |
|
141 |
#trade off for securing access only to explicitly authorized users |
|
142 |
self.backend.permissions.access_set(fpath, common_p) |
|
143 |
|
|
144 |
def create_objects(self): |
|
145 |
for t in self.retrieve_current_nodes(): |
|
146 |
username, headerid, folderid, filename, deleted, filepath, mimetype, public, owner_id = t |
|
147 |
containers = ['pithos', 'trash'] |
|
148 |
|
|
149 |
for c in containers: |
|
150 |
#create container if it does not exist |
|
151 |
try: |
|
152 |
self.backend._lookup_container(username, c) |
|
153 |
except NameError, e: |
|
154 |
self.backend.put_container(username, username, c) |
|
155 |
|
|
156 |
container = 'pithos' if not deleted else 'trash' |
|
157 |
path = self.build_path(folderid) |
|
158 |
#create node |
|
159 |
object = '%s/%s' %(username, container) |
|
160 |
object = '%s/%s/%s' %(object, path, filename) if path else '%s/%s' %(object, filename) |
|
161 |
args = username, container, object |
|
162 |
nodeid = self.create_node(*args) |
|
163 |
#create node history |
|
164 |
vserials = self.create_history(headerid, nodeid, deleted) |
|
165 |
#set object tags |
|
166 |
self.create_tags(headerid, nodeid, vserials) |
|
167 |
#set object's publicity |
|
168 |
if public: |
|
169 |
self.backend.permissions.public_set(object) |
|
170 |
#set object's permissions |
|
171 |
self.create_permissions(headerid, object, username, is_folder=False) |
|
172 |
|
|
173 |
def build_path(self, child_id): |
|
174 |
folder = Table('folder', self.metadata, autoload=True) |
|
175 |
user = Table('gss_user', self.metadata, autoload=True) |
|
176 |
j = folder.join(user, folder.c.owner_id == user.c.id) |
|
177 |
s = select([folder, user.c.username], from_obj=j) |
|
178 |
s = s.where(folder.c.id == child_id) |
|
179 |
s.order_by(folder.c.modificationdate) |
|
180 |
rp = self.conn.execute(s) |
|
181 |
t = rp.fetchone() |
|
182 |
md5 = hashlib.md5() |
|
183 |
hash = md5.hexdigest().lower() |
|
184 |
size = 0 |
|
185 |
if not t[PARENT_ID]: |
|
186 |
return '' |
|
187 |
else: |
|
188 |
container_path = t[USER] |
|
189 |
container_path += '/trash' if t[DELETED] else '/pithos' |
|
190 |
parent_node = self.backend.node.node_lookup(container_path) |
|
191 |
if not parent_node: |
|
192 |
raise Exception('Missing node:', container_path) |
|
193 |
parent_path = self.build_path(t[PARENT_ID]) |
|
194 |
path = '%s/%s/%s' %(container_path, parent_path, t[NAME]) if parent_path else '%s/%s' %(container_path, t[NAME]) |
|
195 |
node = self.backend.node.node_lookup(path) |
|
196 |
if not node: |
|
197 |
node = self.backend.node.node_create(parent_node, path) |
|
198 |
if not node: |
|
199 |
raise Exception('Unable to create node:', path) |
|
200 |
|
|
201 |
#create versions |
|
202 |
v = self.create_version(node, hash, size, t[USER], CLUSTER_NORMAL, 'application/directory', t[CREATIONDATE]) |
|
203 |
if t[CREATIONDATE] != t[MODIFICATIONDATE]: |
|
204 |
self.backend.node.version_recluster(v, CLUSTER_HISTORY) |
|
205 |
self.create_version(node, hash, size, t[USER], CLUSTER_NORMAL, 'application/directory', t[MODIFICATIONDATE]) |
|
206 |
|
|
207 |
#set permissions |
|
208 |
self.create_permissions(t[ID], path, t[USER], is_folder=True) |
|
209 |
return '%s/%s' %(parent_path, t[NAME]) if parent_path else t[NAME] |
|
210 |
|
|
211 |
def retrieve_current_nodes(self): |
|
212 |
fileheader = Table('fileheader', self.metadata, autoload=True) |
|
213 |
filebody = Table('filebody', self.metadata, autoload=True) |
|
214 |
folder = Table('folder', self.metadata, autoload=True) |
|
215 |
gss_user = Table('gss_user', self.metadata, autoload=True) |
|
216 |
j = filebody.join(fileheader, filebody.c.id == fileheader.c.currentbody_id) |
|
217 |
j = j.join(folder, fileheader.c.folder_id == folder.c.id) |
|
218 |
j = j.join(gss_user, fileheader.c.owner_id == gss_user.c.id) |
|
219 |
s = select([gss_user.c.username, fileheader.c.id, fileheader.c.folder_id, |
|
220 |
fileheader.c.name, fileheader.c.deleted, |
|
221 |
filebody.c.storedfilepath, filebody.c.mimetype, |
|
222 |
fileheader.c.readforall, fileheader.c.owner_id], from_obj=j) |
|
223 |
rp = self.conn.execute(s) |
|
224 |
object = rp.fetchone() |
|
225 |
while object: |
|
226 |
yield object |
|
227 |
object = rp.fetchone() |
|
228 |
rp.close() |
|
229 |
|
|
230 |
def retrieve_node_versions(self, header_id): |
|
231 |
filebody = Table('filebody', self.metadata, autoload=True) |
|
232 |
gss_user = Table('gss_user', self.metadata, autoload=True) |
|
233 |
j = filebody.join(gss_user, filebody.c.modifiedby_id == gss_user.c.id) |
|
234 |
s = select([filebody.c.filesize, gss_user.c.username, |
|
235 |
filebody.c.storedfilepath, filebody.c.mimetype, |
|
236 |
filebody.c.modificationdate], from_obj=j) |
|
237 |
s = s.where(filebody.c.header_id == header_id) |
|
238 |
s = s.order_by(filebody.c.version) |
|
239 |
rp = self.conn.execute(s) |
|
240 |
version = rp.fetchone() |
|
241 |
while version: |
|
242 |
yield version, rp.rowcount |
|
243 |
version = rp.fetchone() |
|
244 |
rp.close() |
|
245 |
|
|
246 |
def retrieve_tags(self, header_id): |
|
247 |
filetag = Table('filetag', self.metadata, autoload=True) |
|
248 |
s = select([filetag.c.tag], filetag.c.fileid == header_id) |
|
249 |
rp = self.conn.execute(s) |
|
250 |
tags = rp.fetchall() if rp.returns_rows else [] |
|
251 |
tags = [elem[0] for elem in tags] |
|
252 |
rp.close() |
|
253 |
return ','.join(tags) if tags else '' |
|
254 |
|
|
255 |
def retrieve_permissions(self, id, is_folder=True): |
|
256 |
permissions = {} |
|
257 |
if is_folder: |
|
258 |
ftable = Table('folder_permission', self.metadata, autoload=True) |
|
259 |
else: |
|
260 |
ftable = Table('fileheader_permission', self.metadata, autoload=True) |
|
261 |
permission = Table('permission', self.metadata, autoload=True) |
|
262 |
group = Table('gss_group', self.metadata, autoload=True) |
|
263 |
user = Table('gss_user', self.metadata, autoload=True) |
|
264 |
j = ftable.join(permission, ftable.c.permissions_id == permission.c.id) |
|
265 |
j1 = j.join(group, group.c.id == permission.c.group_id) |
|
266 |
j2 = j.join(user, user.c.id == permission.c.user_id) |
|
267 |
|
|
268 |
permissions = defaultdict(list) |
|
269 |
|
|
270 |
def _get_permissions(self, action='read', get_groups=True): |
|
271 |
if get_groups: |
|
272 |
col, j = group.c.name, j1 |
|
273 |
cond2 = permission.c.group_id != None |
|
274 |
else: |
|
275 |
col, j = user.c.username, j2 |
|
276 |
cond2 = permission.c.user_id != None |
|
277 |
s = select([col], from_obj=j) |
|
278 |
if is_folder: |
|
279 |
s = s.where(ftable.c.folder_id == id) |
|
280 |
else: |
|
281 |
s = s.where(ftable.c.fileheader_id == id) |
|
282 |
if action == 'read': |
|
283 |
cond1 = permission.c.read == True |
|
284 |
else: |
|
285 |
cond1 = permission.c.write == True |
|
286 |
s = s.where(and_(cond1, cond2)) |
|
287 |
print '>', s, s.compile().params |
|
288 |
rp = self.conn.execute(s) |
|
289 |
p = permissions[action].extend([e[0] for e in rp.fetchall()]) |
|
290 |
rp.close() |
|
291 |
return p |
|
292 |
|
|
293 |
#get object read groups |
|
294 |
_get_permissions(self, action='read', get_groups=True) |
|
295 |
|
|
296 |
#get object read users |
|
297 |
_get_permissions(self, action='read', get_groups=False) |
|
298 |
|
|
299 |
#get object write groups |
|
300 |
_get_permissions(self, action='write', get_groups=True) |
|
301 |
|
|
302 |
#get object write groups |
|
303 |
_get_permissions(self, action='write', get_groups=False) |
|
304 |
|
|
305 |
return permissions |
|
306 |
|
|
307 |
if __name__ == "__main__": |
|
308 |
old_db = '' |
|
309 |
db = '' |
|
310 |
|
|
311 |
f = open('fixdates.sql', 'w') |
|
312 |
ot = ObjectMigration(old_db, db, f) |
|
313 |
ot.create_objects() |
|
314 |
f.close() |
|
315 |
|
|
316 |
|
/dev/null | ||
---|---|---|
1 |
#!/usr/bin/env python |
|
2 |
|
|
3 |
# Copyright 2011-2012 GRNET S.A. All rights reserved. |
|
4 |
# |
|
5 |
# Redistribution and use in source and binary forms, with or |
|
6 |
# without modification, are permitted provided that the following |
|
7 |
# conditions are met: |
|
8 |
# |
|
9 |
# 1. Redistributions of source code must retain the above |
|
10 |
# copyright notice, this list of conditions and the following |
|
11 |
# disclaimer. |
|
12 |
# |
|
13 |
# 2. Redistributions in binary form must reproduce the above |
|
14 |
# copyright notice, this list of conditions and the following |
|
15 |
# disclaimer in the documentation and/or other materials |
|
16 |
# provided with the distribution. |
|
17 |
# |
|
18 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
|
19 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
|
20 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
|
21 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
|
22 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
23 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
24 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
|
25 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
|
26 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
|
27 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
|
28 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
|
29 |
# POSSIBILITY OF SUCH DAMAGE. |
|
30 |
# |
|
31 |
# The views and conclusions contained in the software and |
|
32 |
# documentation are those of the authors and should not be |
|
33 |
# interpreted as representing official policies, either expressed |
|
34 |
# or implied, of GRNET S.A. |
|
35 |
|
|
36 |
from sqlalchemy import Table |
|
37 |
from sqlalchemy.sql import select |
|
38 |
|
|
39 |
from pithos.im.models import User |
|
40 |
|
|
41 |
from migrate import Migration |
|
42 |
|
|
43 |
import base64 |
|
44 |
|
|
45 |
class UserMigration(Migration): |
|
46 |
def __init__(self, db): |
|
47 |
Migration.__init__(self, db) |
|
48 |
self.gss_users = Table('gss_user', self.metadata, autoload=True) |
|
49 |
|
|
50 |
def execute(self): |
|
51 |
for u in self.retrieve_users(): |
|
52 |
user = User() |
|
53 |
user.pk = u['id'] |
|
54 |
user.uniq = u['username'] |
|
55 |
user.realname = u['name'] |
|
56 |
user.affiliation = u['homeorganization'] if u['homeorganization'] else '' |
|
57 |
user.auth_token = base64.b64encode(u['authtoken']) |
|
58 |
user.auth_token_created = u['creationdate'] |
|
59 |
user.auth_token_expires = u['authtokenexpirydate'] |
|
60 |
user.created = u['creationdate'] |
|
61 |
user.updated = u['modificationdate'] |
|
62 |
user.email = u['email'] |
|
63 |
user.active = 'ACTIVE' if u['active'] else 'SUSPENDED' |
|
64 |
print '#', user |
|
65 |
user.save(update_timestamps=False) |
|
66 |
|
|
67 |
#create user groups |
|
68 |
for (owner, group, members) in self.retrieve_groups(u['username']): |
|
69 |
self.backend.permissions.group_addmany(owner, group, members) |
|
70 |
|
|
71 |
|
|
72 |
def retrieve_users(self): |
|
73 |
s = self.gss_users.select() |
|
74 |
rp = self.conn.execute(s) |
|
75 |
user = rp.fetchone() |
|
76 |
while user: |
|
77 |
yield user |
|
78 |
user = rp.fetchone() |
|
79 |
rp.close() |
|
80 |
|
|
81 |
def retrieve_groups(self, owner): |
|
82 |
gss_group = Table('gss_group', self.metadata, autoload=True) |
|
83 |
gss_user = Table('gss_user', self.metadata, autoload=True) |
|
84 |
group_user = Table('gss_group_gss_user', self.metadata, autoload=True) |
|
85 |
j1 = gss_group.join(gss_user, gss_group.c.owner_id == gss_user.c.id) |
|
86 |
j2 = group_user.join(gss_user, group_user.c.members_id == gss_user.c.id) |
|
87 |
s = select([gss_group.c.id, gss_group.c.name, gss_user.c.username], from_obj=j1) |
|
88 |
s = s.where(gss_user.c.username == owner) |
|
89 |
rp = self.conn.execute(s) |
|
90 |
gr = rp.fetchone() |
|
91 |
while gr: |
|
92 |
id, group, owner = gr |
|
93 |
s = select([gss_user.c.username], from_obj=j2) |
|
94 |
s = s.where(group_user.c.groupsmember_id == id) |
|
95 |
rp2 = self.conn.execute(s) |
|
96 |
members = rp2.fetchall() |
|
97 |
rp2.close() |
|
98 |
yield owner, group, (m[0] for m in members) |
|
99 |
gr = rp.fetchone() |
|
100 |
rp.close() |
|
101 |
|
|
102 |
if __name__ == "__main__": |
|
103 |
db = '' |
|
104 |
m = UserMigration(db) |
|
105 |
m.execute() |
/dev/null | ||
---|---|---|
1 |
#!/usr/bin/env python |
|
2 |
|
|
3 |
# Copyright 2011-2012 GRNET S.A. All rights reserved. |
|
4 |
# |
|
5 |
# Redistribution and use in source and binary forms, with or |
|
6 |
# without modification, are permitted provided that the following |
|
7 |
# conditions are met: |
|
8 |
# |
|
9 |
# 1. Redistributions of source code must retain the above |
|
10 |
# copyright notice, this list of conditions and the following |
|
11 |
# disclaimer. |
|
12 |
# |
|
13 |
# 2. Redistributions in binary form must reproduce the above |
|
14 |
# copyright notice, this list of conditions and the following |
|
15 |
# disclaimer in the documentation and/or other materials |
|
16 |
# provided with the distribution. |
|
17 |
# |
|
18 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
|
19 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
|
20 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
|
21 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
|
22 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
23 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
24 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
|
25 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
|
26 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
|
27 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
|
28 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
|
29 |
# POSSIBILITY OF SUCH DAMAGE. |
|
30 |
# |
|
31 |
# The views and conclusions contained in the software and |
|
32 |
# documentation are those of the authors and should not be |
|
33 |
# interpreted as representing official policies, either expressed |
|
34 |
# or implied, of GRNET S.A. |
|
35 |
|
|
36 |
from sqlalchemy import create_engine |
|
37 |
from sqlalchemy import Table, Column, String, MetaData |
|
38 |
from sqlalchemy.sql import select |
|
39 |
|
|
40 |
from django.conf import settings |
|
41 |
|
|
42 |
from pithos.backends.modular import ModularBackend |
|
43 |
|
|
44 |
|
|
45 |
class Migration(object): |
|
46 |
def __init__(self, db): |
|
47 |
self.engine = create_engine(db) |
|
48 |
self.metadata = MetaData(self.engine) |
|
49 |
#self.engine.echo = True |
|
50 |
self.conn = self.engine.connect() |
|
51 |
|
|
52 |
options = getattr(settings, 'BACKEND', None)[1] |
|
53 |
self.backend = ModularBackend(*options) |
|
54 |
|
|
55 |
def execute(self): |
|
56 |
pass |
|
57 |
|
|
58 |
|
|
59 |
class Cache(): |
|
60 |
def __init__(self, db): |
|
61 |
self.engine = create_engine(db) |
|
62 |
metadata = MetaData(self.engine) |
|
63 |
|
|
64 |
columns = [] |
|
65 |
columns.append(Column('path', String(2048), primary_key=True)) |
|
66 |
columns.append(Column('hash', String(255))) |
|
67 |
self.files = Table('files', metadata, *columns) |
|
68 |
self.conn = self.engine.connect() |
|
69 |
self.engine.echo = True |
|
70 |
metadata.create_all(self.engine) |
|
71 |
|
|
72 |
def put(self, path, hash): |
|
73 |
# Insert or replace. |
|
74 |
s = self.files.delete().where(self.files.c.path == path) |
|
75 |
r = self.conn.execute(s) |
|
76 |
r.close() |
|
77 |
s = self.files.insert() |
|
78 |
r = self.conn.execute(s, {'path': path, 'hash': hash}) |
|
79 |
r.close() |
|
80 |
|
|
81 |
def get(self, path): |
|
82 |
s = select([self.files.c.hash], self.files.c.path == path) |
|
83 |
r = self.conn.execute(s) |
|
84 |
l = r.fetchone() |
|
85 |
r.close() |
|
86 |
if not l: |
|
87 |
return l |
|
88 |
return l[0] |
/dev/null | ||
---|---|---|
1 |
# Top level |
|
2 |
create temporary table tmp_stats as select 0 as "level", 0 as "node", 0 as "parent", count(serial) as "population", sum(size) as "bytes", max(mtime) as "mtime", cluster, false as "final" from versions group by cluster; |
|
3 |
|
|
4 |
# Account level |
|
5 |
insert into tmp_stats select 1 as "level", n.node, n.parent, count(v.serial) as "population", sum(v.size) as "bytes", max(v.mtime) as "mtime", cluster, false as "final" from versions v, nodes n where n.node=v.node and n.parent=0 and n.node!=0 group by node, cluster; |
|
6 |
create temporary table tmp_nodes select distinct node, level from tmp_stats where level=1; |
|
7 |
|
|
8 |
# Container level |
|
9 |
insert into tmp_stats select 2 as "level", n.node, n.parent, count(v.serial) as "population", sum(v.size) as "bytes", max(v.mtime) as "mtime", cluster, false as "final" from versions v, nodes n where n.node=v.node and n.parent in (select node from tmp_nodes where level=1) group by node, cluster; |
|
10 |
insert into tmp_nodes select distinct node, level from tmp_stats where level=2; |
|
11 |
|
|
12 |
# Object level |
|
13 |
insert into tmp_stats select 3 as "level", n.node, n.parent, count(v.serial) as "population", sum(v.size) as "bytes", max(v.mtime) as "mtime", cluster, false as "final" from versions v, nodes n where n.node=v.node and n.parent in (select node from tmp_nodes where level=2) group by node, cluster; |
|
14 |
insert into tmp_nodes select distinct node, level from tmp_stats where level=3; |
|
15 |
|
|
16 |
# Update containers |
|
17 |
create table tmp_sums as select parent as "node", sum(population) as "population", sum(bytes) as "bytes", max(mtime) as "mtime", cluster from tmp_stats where level=3 group by parent, cluster; |
|
18 |
insert into tmp_stats select 2 as "level", n.node, n.parent, t.population, t.bytes, t.mtime, t.cluster, true as "final" from tmp_sums t, nodes n where n.node=t.node; |
|
19 |
drop table tmp_sums; |
|
20 |
|
|
21 |
# Update accounts |
|
22 |
create table tmp_sums as select parent as "node", sum(bytes) as "bytes", max(mtime) as "mtime", cluster from tmp_stats where level=2 group by parent, cluster; |
|
23 |
create table tmp_population as select parent as "node", sum(population) as "population", cluster from tmp_stats where level=2 and final=false group by parent, cluster; |
|
24 |
insert into tmp_stats select 1 as "level", t.node, 0 as "parent", IFNULL(p.population, 0) as "population", t.bytes, t.mtime, t.cluster, true as "final" from tmp_sums t left join tmp_population p on p.node=t.node and p.cluster=t.cluster; |
|
25 |
drop table tmp_sums; |
|
26 |
drop table tmp_population; |
|
27 |
|
|
28 |
# Update top level |
|
29 |
create table tmp_sums as select parent as "node", sum(bytes) as "bytes", max(mtime) as "mtime", cluster from tmp_stats where level=1 group by parent, cluster; |
|
30 |
create table tmp_population as select parent as "node", sum(population) as "population", cluster from tmp_stats where level=1 and final=false group by parent, cluster; |
|
31 |
insert into tmp_stats select 0 as "level", t.node, 0 as "parent", IFNULL(p.population, 0) as "population", t.bytes, t.mtime, t.cluster, true as "final" from tmp_sums t left join tmp_population p on p.node=t.node and p.cluster=t.cluster; |
|
32 |
drop table tmp_sums; |
|
33 |
drop table tmp_population; |
|
34 |
|
|
35 |
# Clean up |
|
36 |
drop table tmp_nodes; |
|
37 |
delete from tmp_stats where final=false; |
Also available in: Unified diff