root / contrib / migrate-data @ faad3c72
History | View | Annotate | Download (3.6 kB)
1 | 77aa7567 | Antony Chazapis | #!/usr/bin/env python |
---|---|---|---|
2 | 77aa7567 | Antony Chazapis | |
3 | 2e662088 | Antony Chazapis | # Copyright 2011-2012 GRNET S.A. All rights reserved. |
4 | 77aa7567 | Antony Chazapis | # |
5 | 77aa7567 | Antony Chazapis | # Redistribution and use in source and binary forms, with or |
6 | 77aa7567 | Antony Chazapis | # without modification, are permitted provided that the following |
7 | 77aa7567 | Antony Chazapis | # conditions are met: |
8 | 77aa7567 | Antony Chazapis | # |
9 | 77aa7567 | Antony Chazapis | # 1. Redistributions of source code must retain the above |
10 | 77aa7567 | Antony Chazapis | # copyright notice, this list of conditions and the following |
11 | 77aa7567 | Antony Chazapis | # disclaimer. |
12 | 77aa7567 | Antony Chazapis | # |
13 | 77aa7567 | Antony Chazapis | # 2. Redistributions in binary form must reproduce the above |
14 | 77aa7567 | Antony Chazapis | # copyright notice, this list of conditions and the following |
15 | 77aa7567 | Antony Chazapis | # disclaimer in the documentation and/or other materials |
16 | 77aa7567 | Antony Chazapis | # provided with the distribution. |
17 | 77aa7567 | Antony Chazapis | # |
18 | 77aa7567 | Antony Chazapis | # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
19 | 77aa7567 | Antony Chazapis | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
20 | 77aa7567 | Antony Chazapis | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
21 | 77aa7567 | Antony Chazapis | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
22 | 77aa7567 | Antony Chazapis | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
23 | 77aa7567 | Antony Chazapis | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
24 | 77aa7567 | Antony Chazapis | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
25 | 77aa7567 | Antony Chazapis | # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
26 | 77aa7567 | Antony Chazapis | # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
27 | 77aa7567 | Antony Chazapis | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
28 | 77aa7567 | Antony Chazapis | # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
29 | 77aa7567 | Antony Chazapis | # POSSIBILITY OF SUCH DAMAGE. |
30 | 77aa7567 | Antony Chazapis | # |
31 | 77aa7567 | Antony Chazapis | # The views and conclusions contained in the software and |
32 | 77aa7567 | Antony Chazapis | # documentation are those of the authors and should not be |
33 | 77aa7567 | Antony Chazapis | # interpreted as representing official policies, either expressed |
34 | 77aa7567 | Antony Chazapis | # or implied, of GRNET S.A. |
35 | 77aa7567 | Antony Chazapis | |
36 | 77aa7567 | Antony Chazapis | from binascii import hexlify |
37 | 77aa7567 | Antony Chazapis | |
38 | f6c0005f | Sofia Papagiannaki | from sqlalchemy import Table |
39 | 77aa7567 | Antony Chazapis | from sqlalchemy.sql import select |
40 | 77aa7567 | Antony Chazapis | |
41 | 77aa7567 | Antony Chazapis | from pithos import settings |
42 | 77aa7567 | Antony Chazapis | from pithos.backends.modular import ModularBackend |
43 | 77aa7567 | Antony Chazapis | |
44 | 6e147ecc | Antony Chazapis | from pithos.tools.lib.hashmap import HashMap |
45 | 5a96180b | Antony Chazapis | |
46 | 5a96180b | Antony Chazapis | from migrate import Migration, Cache |
47 | 77aa7567 | Antony Chazapis | |
48 | f6c0005f | Sofia Papagiannaki | import os |
49 | f6c0005f | Sofia Papagiannaki | |
50 | 77aa7567 | Antony Chazapis | class DataMigration(Migration): |
51 | 2db16f05 | Sofia Papagiannaki | def __init__(self, pithosdb, db): |
52 | 2db16f05 | Sofia Papagiannaki | Migration.__init__(self, pithosdb) |
53 | f6c0005f | Sofia Papagiannaki | self.cache = Cache(db) |
54 | 77aa7567 | Antony Chazapis | |
55 | f6c0005f | Sofia Papagiannaki | def retrieve_files(self): |
56 | 77aa7567 | Antony Chazapis | # Loop for all available files. |
57 | 98137a34 | Sofia Papagiannaki | filebody = Table('filebody', self.metadata, autoload=True) |
58 | 98137a34 | Sofia Papagiannaki | s = select([filebody.c.storedfilepath]) |
59 | 98137a34 | Sofia Papagiannaki | rp = self.conn.execute(s) |
60 | f6c0005f | Sofia Papagiannaki | path = rp.fetchone() |
61 | f6c0005f | Sofia Papagiannaki | while path: |
62 | f6c0005f | Sofia Papagiannaki | yield path |
63 | f6c0005f | Sofia Papagiannaki | path = rp.fetchone() |
64 | 98137a34 | Sofia Papagiannaki | rp.close() |
65 | f6c0005f | Sofia Papagiannaki | |
66 | f6c0005f | Sofia Papagiannaki | def execute(self): |
67 | f6c0005f | Sofia Papagiannaki | blocksize = self.backend.block_size |
68 | f6c0005f | Sofia Papagiannaki | blockhash = self.backend.hash_algorithm |
69 | 98137a34 | Sofia Papagiannaki | |
70 | f6c0005f | Sofia Papagiannaki | for (path,) in self.retrieve_files(): |
71 | 77aa7567 | Antony Chazapis | map = HashMap(blocksize, blockhash) |
72 | f6c0005f | Sofia Papagiannaki | try: |
73 | f6c0005f | Sofia Papagiannaki | map.load(open(path)) |
74 | f6c0005f | Sofia Papagiannaki | except Exception, e: |
75 | f6c0005f | Sofia Papagiannaki | print e |
76 | f6c0005f | Sofia Papagiannaki | continue |
77 | 77aa7567 | Antony Chazapis | hash = hexlify(map.hash()) |
78 | 77aa7567 | Antony Chazapis | |
79 | f6c0005f | Sofia Papagiannaki | if hash != self.cache.get(path): |
80 | 77aa7567 | Antony Chazapis | missing = self.backend.blocker.block_ping(map) # XXX Backend hack... |
81 | 77aa7567 | Antony Chazapis | status = '[>] ' + path |
82 | 77aa7567 | Antony Chazapis | if missing: |
83 | 77aa7567 | Antony Chazapis | status += ' - %d block(s) missing' % len(missing) |
84 | 77aa7567 | Antony Chazapis | with open(path) as fp: |
85 | 77aa7567 | Antony Chazapis | for h in missing: |
86 | 77aa7567 | Antony Chazapis | offset = map.index(h) * blocksize |
87 | 77aa7567 | Antony Chazapis | fp.seek(offset) |
88 | 77aa7567 | Antony Chazapis | block = fp.read(blocksize) |
89 | 77aa7567 | Antony Chazapis | self.backend.put_block(block) |
90 | 77aa7567 | Antony Chazapis | else: |
91 | 77aa7567 | Antony Chazapis | status += ' - no blocks missing' |
92 | f6c0005f | Sofia Papagiannaki | self.cache.put(path, hash) |
93 | 77aa7567 | Antony Chazapis | else: |
94 | 77aa7567 | Antony Chazapis | status = '[-] ' + path |
95 | 77aa7567 | Antony Chazapis | print status |
96 | 345fffa0 | Antony Chazapis | |
97 | 77aa7567 | Antony Chazapis | if __name__ == "__main__": |
98 | 800a1bce | Sofia Papagiannaki | pithosdb = 'postgresql://gss@127.0.0.1/pithos' |
99 | 77aa7567 | Antony Chazapis | db = 'sqlite:///migrate.db' |
100 | 77aa7567 | Antony Chazapis | |
101 | 2db16f05 | Sofia Papagiannaki | dt = DataMigration(pithosdb, db) |
102 | 77aa7567 | Antony Chazapis | dt.execute() |