3 # Copyright 2011 GRNET S.A. All rights reserved.
5 # Redistribution and use in source and binary forms, with or
6 # without modification, are permitted provided that the following
9 # 1. Redistributions of source code must retain the above
10 # copyright notice, this list of conditions and the following
13 # 2. Redistributions in binary form must reproduce the above
14 # copyright notice, this list of conditions and the following
15 # disclaimer in the documentation and/or other materials
16 # provided with the distribution.
18 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 # POSSIBILITY OF SUCH DAMAGE.
31 # The views and conclusions contained in the software and
32 # documentation are those of the authors and should not be
33 # interpreted as representing official policies, either expressed
34 # or implied, of GRNET S.A.
36 from sqlalchemy import create_engine
37 from sqlalchemy import Table, MetaData
38 from sqlalchemy.sql import select
40 from pithos.api.util import hashmap_hash, get_container_headers
41 from pithos.backends.lib.hashfiler import Blocker, Mapper
42 from pithos.aai.models import PithosUser
44 from django.conf import settings
46 from pithos.backends.modular import ModularBackend
52 class Migration(object):
53 def __init__(self, db):
54 self.engine = create_engine(db)
55 self.metadata = MetaData(self.engine)
56 #self.engine.echo = True
57 self.conn = self.engine.connect()
62 class UserMigration(Migration):
63 def __init__(self, db):
64 Migration.__init__(self, db)
65 self.gss_users = Table('gss_user', self.metadata, autoload=True)
68 s = self.gss_users.select()
69 users = self.conn.execute(s).fetchall()
74 user.uniq = u['username']
75 user.realname = u['name']
77 user.affiliation = u['homeorganization'] if u['homeorganization'] else ''
78 user.auth_token = base64.b64encode(u['authtoken'])
79 user.auth_token_created = u['creationdate']
80 user.auth_token_expires = u['authtokenexpirydate']
81 user.created = u['creationdate']
82 user.updated = u['modificationdate']
84 user.save(update_timestamps=False)
86 class DataMigration(Migration):
87 def __init__(self, db, path, block_size, hash_algorithm):
88 Migration.__init__(self, db)
89 params = {'blocksize': block_size,
90 'blockpath': os.path.join(path + '/blocks'),
91 'hashtype': hash_algorithm}
92 self.blocker = Blocker(**params)
94 params = {'mappath': os.path.join(path + '/maps'),
95 'namelen': self.blocker.hashlen}
96 self.mapper = Mapper(**params)
99 filebody = Table('filebody', self.metadata, autoload=True)
100 s = select([filebody.c.id, filebody.c.storedfilepath])
101 rp = self.conn.execute(s)
109 hashlist = self.blocker.block_stor_file(open(path))[1]
110 self.mapper.map_stor(id, hashlist)
113 class ObjectMigration(DataMigration):
114 def __init__(self, db, path, block_size, hash_algorithm):
115 DataMigration.__init__(self, db, path, block_size, hash_algorithm)
116 options = getattr(settings, 'BACKEND', None)[1]
117 self.backend = ModularBackend(*options)
119 def create_default_containers(self):
120 users = PithosUser.objects.all()
123 self.backend.put_container(u.uniq, u.uniq, 'pithos', {})
124 self.backend.put_container(u.uniq, u.uniq, 'trash', {})
128 def create_directory_markers(self, parent_id=None, path=None):
129 folderTable = Table('folder', self.metadata, autoload=True)
130 userTable = Table('gss_user', self.metadata, autoload=True)
131 s = select([folderTable.c.id, folderTable.c.name, userTable.c.username])
132 s = s.where(folderTable.c.parent_id == parent_id)
133 s = s.where(folderTable.c.owner_id == userTable.c.id)
134 rp = self.conn.execute(s)
140 id, name, uuniq = t[0], t[1], t[2]
141 #print id, name, uuniq
143 obj = '%s/%s' %(path, name) if path else name
145 self.backend.update_object_hashmap(uuniq, uuniq, 'pithos', obj, 0, [])
148 self.create_directory_markers(id, path=obj)
153 filebody = Table('filebody', self.metadata, autoload=True)
154 s = select([filebody.c.id])
155 rp = self.conn.execute(s)
161 hashlist = self.mapper.map_retr(id)
162 #hashmap = d['hashes']
163 #size = int(d['bytes'])
164 #meta.update({'hash': hashmap_hash(request, hashmap)})
165 #version_id = backend.update_object_hashmap(request.user, v_account,
166 # v_container, v_object,
170 if __name__ == "__main__":
172 t = UserMigration(db)
175 basepath = options = getattr(settings, 'PROJECT_PATH', None)
177 'path':os.path.join(basepath, 'data/pithos/'),
178 'block_size':(4 * 1024 * 1024),
179 'hash_algorithm':'sha256'}
180 dt = DataMigration(**params)
183 ot = ObjectMigration(**params)
184 ot.create_default_containers()
185 ot.create_directory_markers()