root / tools / migrate_db @ 2db16f05
History | View | Annotate | Download (8.8 kB)
1 |
#!/usr/bin/env python |
---|---|
2 |
|
3 |
# Copyright 2011 GRNET S.A. All rights reserved. |
4 |
# |
5 |
# Redistribution and use in source and binary forms, with or |
6 |
# without modification, are permitted provided that the following |
7 |
# conditions are met: |
8 |
# |
9 |
# 1. Redistributions of source code must retain the above |
10 |
# copyright notice, this list of conditions and the following |
11 |
# disclaimer. |
12 |
# |
13 |
# 2. Redistributions in binary form must reproduce the above |
14 |
# copyright notice, this list of conditions and the following |
15 |
# disclaimer in the documentation and/or other materials |
16 |
# provided with the distribution. |
17 |
# |
18 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
19 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
20 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
21 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
22 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
23 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
24 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
25 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
26 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
27 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
28 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
29 |
# POSSIBILITY OF SUCH DAMAGE. |
30 |
# |
31 |
# The views and conclusions contained in the software and |
32 |
# documentation are those of the authors and should not be |
33 |
# interpreted as representing official policies, either expressed |
34 |
# or implied, of GRNET S.A. |
35 |
|
36 |
from sqlalchemy import Table |
37 |
from sqlalchemy.sql import select |
38 |
|
39 |
from binascii import hexlify |
40 |
|
41 |
from pithos.backends.lib.hashfiler import Blocker |
42 |
from pithos.backends.lib.sqlalchemy import Node |
43 |
from pithos.aai.models import PithosUser |
44 |
|
45 |
from django.conf import settings |
46 |
|
47 |
from pithos.backends.modular import CLUSTER_NORMAL, CLUSTER_HISTORY, CLUSTER_DELETED |
48 |
from pithos.backends.lib.sqlalchemy.node import Node |
49 |
from pithos.backends.lib.sqlalchemy.dbwrapper import DBWrapper |
50 |
|
51 |
from lib.transfer import upload |
52 |
from lib.hashmap import HashMap, file_read_iterator |
53 |
from lib.client import Fault |
54 |
from lib.migrate import Migration |
55 |
|
56 |
import json |
57 |
import os |
58 |
import sys |
59 |
import hashlib |
60 |
import mimetypes |
61 |
|
62 |
class ObjectMigration(Migration): |
63 |
def __init__(self, old_db): |
64 |
Migration.__init__(self, old_db) |
65 |
self.wrapper = ClientWrapper(self.backend) |
66 |
params = {'wrapper': DBWrapper(self.backend.db)} |
67 |
self.node = Node(**params) |
68 |
|
69 |
def create_default_containers(self): |
70 |
users = PithosUser.objects.all() |
71 |
for u in users: |
72 |
print '#', u.uniq |
73 |
try: |
74 |
self.wrapper.create_container('pithos', u.uniq) |
75 |
self.wrapper.create_container('trash', u.uniq) |
76 |
except NameError, e: |
77 |
pass |
78 |
|
79 |
def get_path(self, child_id): |
80 |
folderTable = Table('folder', self.metadata, autoload=True) |
81 |
s = select([folderTable.c.parent_id, folderTable.c.name]) |
82 |
s = s.where(folderTable.c.id == child_id) |
83 |
rp = self.conn.execute(s) |
84 |
parent_id, foldername = rp.fetchone() |
85 |
if not parent_id: |
86 |
return '' |
87 |
else: |
88 |
return '%s/%s' %(self.get_path(parent_id), foldername) |
89 |
|
90 |
def create_object(self, username, container, object, filepath, mimetype): |
91 |
obj = '' |
92 |
path = '/'.join(object.split('/')[:-1]) |
93 |
name = object.split('/')[-1] |
94 |
#create directory markers |
95 |
for f in path.split('/'): |
96 |
obj = '%s/%s' %(obj, f) if obj else f |
97 |
try: |
98 |
self.wrapper.create_directory_marker('pithos', obj, username) |
99 |
except NameError, e: |
100 |
pass |
101 |
self.wrapper.set_account(username) |
102 |
|
103 |
prefix = '%s/' %path if path else '' |
104 |
print '#', filepath, container, prefix, name, mimetype |
105 |
return upload(self.wrapper, filepath, container, prefix, name, mimetype) |
106 |
|
107 |
def create_history(self, user, header_id, node_id, deleted=False): |
108 |
filebody = Table('filebody', self.metadata, autoload=True) |
109 |
gss_user = Table('gss_user', self.metadata, autoload=True) |
110 |
j = filebody.join(gss_user, filebody.c.modifiedby_id == gss_user.c.id) |
111 |
s = select([filebody.c.filesize, gss_user.c.username], from_obj=j) |
112 |
s = s.where(filebody.c.header_id == header_id) |
113 |
s = s.order_by(filebody.c.version) |
114 |
rp = self.conn.execute(s) |
115 |
versions = rp.fetchall() |
116 |
print '#', len(versions) |
117 |
rp.close() |
118 |
i = 0 |
119 |
for size, modyfied_by in versions: |
120 |
cluster = CLUSTER_HISTORY if i < len(versions) - 1 else CLUSTER_NORMAL |
121 |
cluster = cluster if not deleted else CLUSTER_DELETED |
122 |
args = (node_id, size, None, modyfied_by, cluster) |
123 |
self.node.version_create(*args) |
124 |
i += 1 |
125 |
|
126 |
def create_objects(self): |
127 |
fileheader = Table('fileheader', self.metadata, autoload=True) |
128 |
filebody = Table('filebody', self.metadata, autoload=True) |
129 |
folder = Table('folder', self.metadata, autoload=True) |
130 |
gss_user = Table('gss_user', self.metadata, autoload=True) |
131 |
j = filebody.join(fileheader, filebody.c.id == fileheader.c.currentbody_id) |
132 |
j = j.join(folder, fileheader.c.folder_id == folder.c.id) |
133 |
j = j.join(gss_user, fileheader.c.owner_id == gss_user.c.id) |
134 |
s = select([gss_user.c.username, fileheader.c.id, fileheader.c.folder_id, |
135 |
fileheader.c.name, fileheader.c.deleted, filebody.c.storedfilepath, |
136 |
filebody.c.mimetype], from_obj=j) |
137 |
rp = self.conn.execute(s) |
138 |
objects = rp.fetchall() |
139 |
for username, headerid, folderid, filename, deleted, filepath, mimetype in objects: |
140 |
path = self.get_path(folderid)[1:] |
141 |
container = 'pithos' if not deleted else 'trash' |
142 |
object = '%s/%s' %(path, filename) |
143 |
#filepath = '/Users/butters/Downloads/torvalds-linux-0f86267' |
144 |
vserial = self.create_object(username, container, object, filepath, mimetype) |
145 |
nodeid = self.node.version_get_properties(vserial, keys=('node',))[0] |
146 |
self.create_history(username, headerid, nodeid, deleted) |
147 |
self.node.version_remove(vserial) |
148 |
#self.set_metadata() |
149 |
#self.set_public() |
150 |
#self.statistics() |
151 |
#self.set_permissions() |
152 |
|
153 |
def handle_deleted(self): |
154 |
pass |
155 |
|
156 |
def upload_dir(self, dir, prefix, user, container): |
157 |
for f in os.listdir(dir): |
158 |
fullpath = '%s/%s' %(dir, f) |
159 |
if os.path.isfile(fullpath): |
160 |
type = mimetypes.guess_type(fullpath)[0] |
161 |
name = '/'.join(fullpath.split(prefix)[1:]) |
162 |
print '@', user, container, name, fullpath, type |
163 |
self.create_object(user, container, name, fullpath, type) |
164 |
else: self.upload_dir(fullpath, prefix, user, container) |
165 |
|
166 |
class ClientWrapper(object): |
167 |
"""Wraps client methods used by transfer.upload() |
168 |
to ModularBackend methods""" |
169 |
|
170 |
def __init__(self, backend): |
171 |
self.backend = backend |
172 |
self.block_size = self.backend.block_size |
173 |
self.block_hash = self.backend.hash_algorithm |
174 |
|
175 |
def set_account(self, account): |
176 |
self.account = account |
177 |
|
178 |
def create_container(self, container, account=None, **meta): |
179 |
self.backend.put_container(account, account, container, meta) |
180 |
|
181 |
def create_directory_marker(self, container, object, account=None): |
182 |
md5 = hashlib.md5() |
183 |
meta = {'Content-Type':'application/directory', |
184 |
'hash': md5.hexdigest().lower()} |
185 |
self.backend.update_object_hashmap(account, account, container, object, 0, [], meta) |
186 |
|
187 |
def create_object_by_hashmap(self, container, object, map, mimetype=None): |
188 |
hashmap = HashMap(self.block_size, self.block_hash) |
189 |
for h in map['hashes']: |
190 |
hashmap.append(h) |
191 |
meta = {'hash':hexlify(hashmap.hash())} |
192 |
if mimetype: |
193 |
meta['content-type'] = mimetype |
194 |
size = map['bytes'] |
195 |
try: |
196 |
args = [self.account, self.account, container, object, size, map['hashes'], meta] |
197 |
return self.backend.update_object_hashmap(*args) |
198 |
except IndexError, ie: |
199 |
fault = Fault(ie.data, 409) |
200 |
raise fault |
201 |
|
202 |
def update_container_data(self, container, f): |
203 |
#just put the blocks |
204 |
for block in file_read_iterator(f, self.block_size): |
205 |
self.backend.put_block(block) |
206 |
|
207 |
def retrieve_container_metadata(self, container): |
208 |
return {'x-container-block-size':self.block_size, |
209 |
'x-container-block-hash':self.block_hash} |
210 |
|
211 |
if __name__ == "__main__": |
212 |
old_db = '' |
213 |
|
214 |
ot = ObjectMigration(old_db) |
215 |
#ot.create_default_containers() |
216 |
#ot.create_objects() |
217 |
|
218 |
p = '' |
219 |
ot.upload_dir(p, p, 'chstath', 'linux') |
220 |
|
221 |
|