root / contrib / migrate-data @ 6ab0b1b5
History | View | Annotate | Download (3.6 kB)
1 |
#!/usr/bin/env python |
---|---|
2 |
|
3 |
# Copyright 2011-2012 GRNET S.A. All rights reserved. |
4 |
# |
5 |
# Redistribution and use in source and binary forms, with or |
6 |
# without modification, are permitted provided that the following |
7 |
# conditions are met: |
8 |
# |
9 |
# 1. Redistributions of source code must retain the above |
10 |
# copyright notice, this list of conditions and the following |
11 |
# disclaimer. |
12 |
# |
13 |
# 2. Redistributions in binary form must reproduce the above |
14 |
# copyright notice, this list of conditions and the following |
15 |
# disclaimer in the documentation and/or other materials |
16 |
# provided with the distribution. |
17 |
# |
18 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
19 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
20 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
21 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
22 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
23 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
24 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
25 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
26 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
27 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
28 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
29 |
# POSSIBILITY OF SUCH DAMAGE. |
30 |
# |
31 |
# The views and conclusions contained in the software and |
32 |
# documentation are those of the authors and should not be |
33 |
# interpreted as representing official policies, either expressed |
34 |
# or implied, of GRNET S.A. |
35 |
|
36 |
from binascii import hexlify |
37 |
|
38 |
from sqlalchemy import Table |
39 |
from sqlalchemy.sql import select |
40 |
|
41 |
from pithos import settings |
42 |
from pithos.backends.modular import ModularBackend |
43 |
|
44 |
from pithos.tools.lib.hashmap import HashMap |
45 |
|
46 |
from migrate import Migration, Cache |
47 |
|
48 |
import os |
49 |
|
50 |
class DataMigration(Migration): |
51 |
def __init__(self, pithosdb, db): |
52 |
Migration.__init__(self, pithosdb) |
53 |
self.cache = Cache(db) |
54 |
|
55 |
def retrieve_files(self): |
56 |
# Loop for all available files. |
57 |
filebody = Table('filebody', self.metadata, autoload=True) |
58 |
s = select([filebody.c.storedfilepath]) |
59 |
rp = self.conn.execute(s) |
60 |
path = rp.fetchone() |
61 |
while path: |
62 |
yield path |
63 |
path = rp.fetchone() |
64 |
rp.close() |
65 |
|
66 |
def execute(self): |
67 |
blocksize = self.backend.block_size |
68 |
blockhash = self.backend.hash_algorithm |
69 |
|
70 |
for (path,) in self.retrieve_files(): |
71 |
map = HashMap(blocksize, blockhash) |
72 |
try: |
73 |
map.load(open(path)) |
74 |
except Exception, e: |
75 |
print e |
76 |
continue |
77 |
hash = hexlify(map.hash()) |
78 |
|
79 |
if hash != self.cache.get(path): |
80 |
missing = self.backend.blocker.block_ping(map) # XXX Backend hack... |
81 |
status = '[>] ' + path |
82 |
if missing: |
83 |
status += ' - %d block(s) missing' % len(missing) |
84 |
with open(path) as fp: |
85 |
for h in missing: |
86 |
offset = map.index(h) * blocksize |
87 |
fp.seek(offset) |
88 |
block = fp.read(blocksize) |
89 |
self.backend.put_block(block) |
90 |
else: |
91 |
status += ' - no blocks missing' |
92 |
self.cache.put(path, hash) |
93 |
else: |
94 |
status = '[-] ' + path |
95 |
print status |
96 |
|
97 |
if __name__ == "__main__": |
98 |
pithosdb = 'postgresql://gss@127.0.0.1/pithos' |
99 |
db = 'sqlite:///migrate.db' |
100 |
|
101 |
dt = DataMigration(pithosdb, db) |
102 |
dt.execute() |