3 # Copyright 2011 GRNET S.A. All rights reserved.
5 # Redistribution and use in source and binary forms, with or
6 # without modification, are permitted provided that the following
9 # 1. Redistributions of source code must retain the above
10 # copyright notice, this list of conditions and the following
13 # 2. Redistributions in binary form must reproduce the above
14 # copyright notice, this list of conditions and the following
15 # disclaimer in the documentation and/or other materials
16 # provided with the distribution.
18 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 # POSSIBILITY OF SUCH DAMAGE.
31 # The views and conclusions contained in the software and
32 # documentation are those of the authors and should not be
33 # interpreted as representing official policies, either expressed
34 # or implied, of GRNET S.A.
36 from sqlalchemy import Table
37 from sqlalchemy.sql import select
39 from binascii import hexlify
41 from pithos.backends.lib.hashfiler import Blocker
42 from pithos.backends.lib.sqlalchemy import Node
43 from pithos.aai.models import PithosUser
45 from django.conf import settings
47 from pithos.backends.modular import CLUSTER_NORMAL, CLUSTER_HISTORY, CLUSTER_DELETED
48 from pithos.backends.lib.sqlalchemy.node import Node
49 from pithos.backends.lib.sqlalchemy.dbwrapper import DBWrapper
51 from lib.transfer import upload
52 from lib.hashmap import HashMap, file_read_iterator
53 from lib.client import Fault
54 from lib.migrate import Migration
62 class ObjectMigration(Migration):
63 def __init__(self, old_db):
64 Migration.__init__(self, old_db)
65 self.wrapper = ClientWrapper(self.backend)
66 params = {'wrapper': DBWrapper(self.backend.db)}
67 self.node = Node(**params)
69 def create_default_containers(self):
70 users = PithosUser.objects.all()
74 self.wrapper.create_container('pithos', u.uniq)
75 self.wrapper.create_container('trash', u.uniq)
79 def get_path(self, child_id):
80 folderTable = Table('folder', self.metadata, autoload=True)
81 s = select([folderTable.c.parent_id, folderTable.c.name])
82 s = s.where(folderTable.c.id == child_id)
83 rp = self.conn.execute(s)
84 parent_id, foldername = rp.fetchone()
88 return '%s/%s' %(self.get_path(parent_id), foldername)
90 def create_object(self, username, container, object, filepath, mimetype):
92 path = '/'.join(object.split('/')[:-1])
93 name = object.split('/')[-1]
94 #create directory markers
95 for f in path.split('/'):
96 obj = '%s/%s' %(obj, f) if obj else f
98 self.wrapper.create_directory_marker('pithos', obj, username)
101 self.wrapper.set_account(username)
103 prefix = '%s/' %path if path else ''
104 print '#', filepath, container, prefix, name, mimetype
105 return upload(self.wrapper, filepath, container, prefix, name, mimetype)
107 def create_history(self, user, header_id, node_id, deleted=False):
108 filebody = Table('filebody', self.metadata, autoload=True)
109 gss_user = Table('gss_user', self.metadata, autoload=True)
110 j = filebody.join(gss_user, filebody.c.modifiedby_id == gss_user.c.id)
111 s = select([filebody.c.filesize, gss_user.c.username], from_obj=j)
112 s = s.where(filebody.c.header_id == header_id)
113 s = s.order_by(filebody.c.version)
114 rp = self.conn.execute(s)
115 versions = rp.fetchall()
116 print '#', len(versions)
119 for size, modyfied_by in versions:
120 cluster = CLUSTER_HISTORY if i < len(versions) - 1 else CLUSTER_NORMAL
121 cluster = cluster if not deleted else CLUSTER_DELETED
122 args = (node_id, size, None, modyfied_by, cluster)
123 self.node.version_create(*args)
126 def create_objects(self):
127 fileheader = Table('fileheader', self.metadata, autoload=True)
128 filebody = Table('filebody', self.metadata, autoload=True)
129 folder = Table('folder', self.metadata, autoload=True)
130 gss_user = Table('gss_user', self.metadata, autoload=True)
131 j = filebody.join(fileheader, filebody.c.id == fileheader.c.currentbody_id)
132 j = j.join(folder, fileheader.c.folder_id == folder.c.id)
133 j = j.join(gss_user, fileheader.c.owner_id == gss_user.c.id)
134 s = select([gss_user.c.username, fileheader.c.id, fileheader.c.folder_id,
135 fileheader.c.name, fileheader.c.deleted, filebody.c.storedfilepath,
136 filebody.c.mimetype], from_obj=j)
137 rp = self.conn.execute(s)
138 objects = rp.fetchall()
139 for username, headerid, folderid, filename, deleted, filepath, mimetype in objects:
140 path = self.get_path(folderid)[1:]
141 container = 'pithos' if not deleted else 'trash'
142 object = '%s/%s' %(path, filename)
143 #filepath = '/Users/butters/Downloads/torvalds-linux-0f86267'
144 vserial = self.create_object(username, container, object, filepath, mimetype)
145 nodeid = self.node.version_get_properties(vserial, keys=('node',))[0]
146 self.create_history(username, headerid, nodeid, deleted)
147 self.node.version_remove(vserial)
151 #self.set_permissions()
153 def handle_deleted(self):
156 def upload_dir(self, dir, prefix, user, container):
157 for f in os.listdir(dir):
158 fullpath = '%s/%s' %(dir, f)
159 if os.path.isfile(fullpath):
160 type = mimetypes.guess_type(fullpath)[0]
161 name = '/'.join(fullpath.split(prefix)[1:])
162 print '@', user, container, name, fullpath, type
163 self.create_object(user, container, name, fullpath, type)
164 else: self.upload_dir(fullpath, prefix, user, container)
166 class ClientWrapper(object):
167 """Wraps client methods used by transfer.upload()
168 to ModularBackend methods"""
170 def __init__(self, backend):
171 self.backend = backend
172 self.block_size = self.backend.block_size
173 self.block_hash = self.backend.hash_algorithm
175 def set_account(self, account):
176 self.account = account
178 def create_container(self, container, account=None, **meta):
179 self.backend.put_container(account, account, container, meta)
181 def create_directory_marker(self, container, object, account=None):
183 meta = {'Content-Type':'application/directory',
184 'hash': md5.hexdigest().lower()}
185 self.backend.update_object_hashmap(account, account, container, object, 0, [], meta)
187 def create_object_by_hashmap(self, container, object, map, mimetype=None):
188 hashmap = HashMap(self.block_size, self.block_hash)
189 for h in map['hashes']:
191 meta = {'hash':hexlify(hashmap.hash())}
193 meta['content-type'] = mimetype
196 args = [self.account, self.account, container, object, size, map['hashes'], meta]
197 return self.backend.update_object_hashmap(*args)
198 except IndexError, ie:
199 fault = Fault(ie.data, 409)
202 def update_container_data(self, container, f):
204 for block in file_read_iterator(f, self.block_size):
205 self.backend.put_block(block)
207 def retrieve_container_metadata(self, container):
208 return {'x-container-block-size':self.block_size,
209 'x-container-block-hash':self.block_hash}
211 if __name__ == "__main__":
214 ot = ObjectMigration(old_db)
215 #ot.create_default_containers()
219 ot.upload_dir(p, p, 'chstath', 'linux')