root / tools / migrate-data @ f6c0005f
History | View | Annotate | Download (3.6 kB)
1 |
#!/usr/bin/env python |
---|---|
2 |
|
3 |
# Copyright 2011 GRNET S.A. All rights reserved. |
4 |
# |
5 |
# Redistribution and use in source and binary forms, with or |
6 |
# without modification, are permitted provided that the following |
7 |
# conditions are met: |
8 |
# |
9 |
# 1. Redistributions of source code must retain the above |
10 |
# copyright notice, this list of conditions and the following |
11 |
# disclaimer. |
12 |
# |
13 |
# 2. Redistributions in binary form must reproduce the above |
14 |
# copyright notice, this list of conditions and the following |
15 |
# disclaimer in the documentation and/or other materials |
16 |
# provided with the distribution. |
17 |
# |
18 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
19 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
20 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
21 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
22 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
23 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
24 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
25 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
26 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
27 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
28 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
29 |
# POSSIBILITY OF SUCH DAMAGE. |
30 |
# |
31 |
# The views and conclusions contained in the software and |
32 |
# documentation are those of the authors and should not be |
33 |
# interpreted as representing official policies, either expressed |
34 |
# or implied, of GRNET S.A. |
35 |
|
36 |
from binascii import hexlify |
37 |
|
38 |
from sqlalchemy import Table |
39 |
from sqlalchemy.sql import select |
40 |
|
41 |
from pithos import settings |
42 |
from pithos.backends.modular import ModularBackend |
43 |
|
44 |
from lib.hashmap import HashMap |
45 |
from lib.migrate import Migration, Cache |
46 |
|
47 |
import os |
48 |
|
49 |
class DataMigration(Migration): |
50 |
def __init__(self, pithosdb, db): |
51 |
Migration.__init__(self, pithosdb) |
52 |
self.cache = Cache(db) |
53 |
|
54 |
def retrieve_files(self): |
55 |
# Loop for all available files. |
56 |
filebody = Table('filebody', self.metadata, autoload=True) |
57 |
s = select([filebody.c.storedfilepath]) |
58 |
rp = self.conn.execute(s) |
59 |
path = rp.fetchone() |
60 |
while path: |
61 |
yield path |
62 |
path = rp.fetchone() |
63 |
rp.close() |
64 |
|
65 |
def execute(self): |
66 |
blocksize = self.backend.block_size |
67 |
blockhash = self.backend.hash_algorithm |
68 |
|
69 |
for (path,) in self.retrieve_files(): |
70 |
map = HashMap(blocksize, blockhash) |
71 |
try: |
72 |
map.load(open(path)) |
73 |
except Exception, e: |
74 |
print e |
75 |
continue |
76 |
hash = hexlify(map.hash()) |
77 |
|
78 |
if hash != self.cache.get(path): |
79 |
missing = self.backend.blocker.block_ping(map) # XXX Backend hack... |
80 |
status = '[>] ' + path |
81 |
if missing: |
82 |
status += ' - %d block(s) missing' % len(missing) |
83 |
with open(path) as fp: |
84 |
for h in missing: |
85 |
offset = map.index(h) * blocksize |
86 |
fp.seek(offset) |
87 |
block = fp.read(blocksize) |
88 |
self.backend.put_block(block) |
89 |
else: |
90 |
status += ' - no blocks missing' |
91 |
self.cache.put(path, hash) |
92 |
else: |
93 |
status = '[-] ' + path |
94 |
print status |
95 |
|
96 |
if __name__ == "__main__": |
97 |
pithosdb = '' |
98 |
db = 'sqlite:///migrate.db' |
99 |
|
100 |
dt = DataMigration(pithosdb, db) |
101 |
dt.execute() |