root / tools / migrate_db @ 98137a34
History | View | Annotate | Download (6.6 kB)
1 |
#!/usr/bin/env python |
---|---|
2 |
|
3 |
# Copyright 2011 GRNET S.A. All rights reserved. |
4 |
# |
5 |
# Redistribution and use in source and binary forms, with or |
6 |
# without modification, are permitted provided that the following |
7 |
# conditions are met: |
8 |
# |
9 |
# 1. Redistributions of source code must retain the above |
10 |
# copyright notice, this list of conditions and the following |
11 |
# disclaimer. |
12 |
# |
13 |
# 2. Redistributions in binary form must reproduce the above |
14 |
# copyright notice, this list of conditions and the following |
15 |
# disclaimer in the documentation and/or other materials |
16 |
# provided with the distribution. |
17 |
# |
18 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
19 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
20 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
21 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
22 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
23 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
24 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
25 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
26 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
27 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
28 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
29 |
# POSSIBILITY OF SUCH DAMAGE. |
30 |
# |
31 |
# The views and conclusions contained in the software and |
32 |
# documentation are those of the authors and should not be |
33 |
# interpreted as representing official policies, either expressed |
34 |
# or implied, of GRNET S.A. |
35 |
|
36 |
from sqlalchemy import Table |
37 |
from sqlalchemy.sql import select |
38 |
|
39 |
from binascii import hexlify |
40 |
|
41 |
from pithos.backends.lib.hashfiler import Blocker |
42 |
from pithos.aai.models import PithosUser |
43 |
|
44 |
from django.conf import settings |
45 |
|
46 |
from pithos.backends.modular import ModularBackend |
47 |
|
48 |
from lib.transfer import upload |
49 |
from lib.hashmap import HashMap |
50 |
from lib.client import Fault |
51 |
from lib.migrate import Migration |
52 |
|
53 |
import json |
54 |
import os |
55 |
import sys |
56 |
import hashlib |
57 |
|
58 |
class ObjectMigration(DataMigration): |
59 |
def __init__(self, db, path, block_size, hash_algorithm): |
60 |
DataMigration.__init__(self, db, path, block_size, hash_algorithm) |
61 |
self.wrapper = ClientWrapper() |
62 |
|
63 |
def create_default_containers(self): |
64 |
users = PithosUser.objects.all() |
65 |
for u in users: |
66 |
print '#', u.uniq |
67 |
try: |
68 |
self.wrapper.create_container('pithos', u.uniq) |
69 |
self.wrapper.create_container('trash', u.uniq) |
70 |
except NameError, e: |
71 |
pass |
72 |
|
73 |
def get_path(self, child_id): |
74 |
folderTable = Table('folder', self.metadata, autoload=True) |
75 |
s = select([folderTable.c.parent_id, folderTable.c.name]) |
76 |
s = s.where(folderTable.c.id == child_id) |
77 |
rp = self.conn.execute(s) |
78 |
parent_id, foldername = rp.fetchone() |
79 |
if not parent_id: |
80 |
return '' |
81 |
else: |
82 |
return '%s/%s' %(self.get_path(parent_id), foldername) |
83 |
|
84 |
def create_objects(self): |
85 |
fileheader = Table('fileheader', self.metadata, autoload=True) |
86 |
filebody = Table('filebody', self.metadata, autoload=True) |
87 |
folder = Table('folder', self.metadata, autoload=True) |
88 |
gss_user = Table('gss_user', self.metadata, autoload=True) |
89 |
j = filebody.join(fileheader, filebody.c.header_id == fileheader.c.id) |
90 |
j = j.join(folder, fileheader.c.folder_id == folder.c.id) |
91 |
j = j.join(gss_user, fileheader.c.owner_id == gss_user.c.id) |
92 |
s = select([gss_user.c.username, fileheader.c.folder_id, fileheader.c.name, |
93 |
filebody.c.storedfilepath], from_obj=j) |
94 |
rp = self.conn.execute(s) |
95 |
objects = rp.fetchall() |
96 |
for username, folderid, filename, filepath in objects: |
97 |
path = self.get_path(folderid)[1:] |
98 |
obj = '' |
99 |
#create directory markers |
100 |
for f in path.split('/'): |
101 |
obj = '%s/%s' %(obj, f) if obj else f |
102 |
try: |
103 |
self.wrapper.create_directory_marker('pithos', obj, username) |
104 |
except NameError, e: |
105 |
pass |
106 |
self.wrapper.set_account(username) |
107 |
|
108 |
print '#', username, path, filename |
109 |
prefix = '%s/' %path if path else '' |
110 |
upload(self.wrapper, filepath, 'pithos', prefix, filename) |
111 |
|
112 |
class ClientWrapper(object): |
113 |
"""Wraps client methods used by transfer.upload() |
114 |
to ModularBackend methods""" |
115 |
|
116 |
def __init__(self): |
117 |
options = getattr(settings, 'BACKEND', None)[1] |
118 |
self.backend = ModularBackend(*options) |
119 |
self.block_size = self.backend.block_size |
120 |
self.block_hash = self.backend.hash_algorithm |
121 |
|
122 |
def set_account(self, account): |
123 |
self.account = account |
124 |
|
125 |
def create_container(self, container, account=None, **meta): |
126 |
self.backend.put_container(account, account, container, meta) |
127 |
|
128 |
def create_directory_marker(self, container, object, account=None): |
129 |
md5 = hashlib.md5() |
130 |
meta = {'Content-Type':'application/directory', |
131 |
'hash': md5.hexdigest().lower()} |
132 |
self.backend.update_object_hashmap(account, account, container, object, 0, [], meta) |
133 |
|
134 |
def create_object_by_hashmap(self, container, object, map): |
135 |
hashmap = HashMap(self.block_size, self.block_hash) |
136 |
for hash in map['hashes']: |
137 |
hashmap.append(hash) |
138 |
meta = {'hash':hexlify(hashmap.hash())} |
139 |
size = map['bytes'] |
140 |
try: |
141 |
args = [self.account, self.account, container, object, size, map['hashes'], meta] |
142 |
self.backend.update_object_hashmap(*args) |
143 |
except IndexError, ie: |
144 |
fault = Fault(ie.data, 409) |
145 |
raise fault |
146 |
|
147 |
def create_object(self, container, object, f): |
148 |
hashmap = HashMap(self.block_size, self.block_hash) |
149 |
hashmap.load(f) |
150 |
map = [hexlify(x) for x in hashmap] |
151 |
meta = {'hash':hashmap.hash()} |
152 |
size = hashmap.size |
153 |
self.backend.update_object_hashmap(self.account, self.account, container, object, size, hashmap, meta) |
154 |
|
155 |
def retrieve_container_metadata(self, container): |
156 |
return {'x-container-block-size':self.block_size, |
157 |
'x-container-block-hash':self.block_hash} |
158 |
|
159 |
if __name__ == "__main__": |
160 |
db = '' |
161 |
|
162 |
basepath = options = getattr(settings, 'PROJECT_PATH', None) |
163 |
params = {'db':db, |
164 |
'path':os.path.join(basepath, 'data/pithos/'), |
165 |
'block_size':(4 * 1024 * 1024), |
166 |
'hash_algorithm':'sha256'} |
167 |
|
168 |
ot = ObjectMigration(**params) |
169 |
ot.create_default_containers() |
170 |
ot.create_objects() |
171 |
|
172 |
|