Statistics
| Branch: | Tag: | Revision:

root / contrib / migrate-data @ 95059648

History | View | Annotate | Download (3.6 kB)

1 77aa7567 Antony Chazapis
#!/usr/bin/env python
2 77aa7567 Antony Chazapis
3 2e662088 Antony Chazapis
# Copyright 2011-2012 GRNET S.A. All rights reserved.
4 77aa7567 Antony Chazapis
# 
5 77aa7567 Antony Chazapis
# Redistribution and use in source and binary forms, with or
6 77aa7567 Antony Chazapis
# without modification, are permitted provided that the following
7 77aa7567 Antony Chazapis
# conditions are met:
8 77aa7567 Antony Chazapis
# 
9 77aa7567 Antony Chazapis
#   1. Redistributions of source code must retain the above
10 77aa7567 Antony Chazapis
#      copyright notice, this list of conditions and the following
11 77aa7567 Antony Chazapis
#      disclaimer.
12 77aa7567 Antony Chazapis
# 
13 77aa7567 Antony Chazapis
#   2. Redistributions in binary form must reproduce the above
14 77aa7567 Antony Chazapis
#      copyright notice, this list of conditions and the following
15 77aa7567 Antony Chazapis
#      disclaimer in the documentation and/or other materials
16 77aa7567 Antony Chazapis
#      provided with the distribution.
17 77aa7567 Antony Chazapis
# 
18 77aa7567 Antony Chazapis
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19 77aa7567 Antony Chazapis
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 77aa7567 Antony Chazapis
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 77aa7567 Antony Chazapis
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22 77aa7567 Antony Chazapis
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 77aa7567 Antony Chazapis
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 77aa7567 Antony Chazapis
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25 77aa7567 Antony Chazapis
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 77aa7567 Antony Chazapis
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 77aa7567 Antony Chazapis
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 77aa7567 Antony Chazapis
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 77aa7567 Antony Chazapis
# POSSIBILITY OF SUCH DAMAGE.
30 77aa7567 Antony Chazapis
# 
31 77aa7567 Antony Chazapis
# The views and conclusions contained in the software and
32 77aa7567 Antony Chazapis
# documentation are those of the authors and should not be
33 77aa7567 Antony Chazapis
# interpreted as representing official policies, either expressed
34 77aa7567 Antony Chazapis
# or implied, of GRNET S.A.
35 77aa7567 Antony Chazapis
36 77aa7567 Antony Chazapis
from binascii import hexlify
37 77aa7567 Antony Chazapis
38 f6c0005f Sofia Papagiannaki
from sqlalchemy import Table
39 77aa7567 Antony Chazapis
from sqlalchemy.sql import select
40 77aa7567 Antony Chazapis
41 77aa7567 Antony Chazapis
from pithos import settings
42 77aa7567 Antony Chazapis
from pithos.backends.modular import ModularBackend
43 77aa7567 Antony Chazapis
44 6e147ecc Antony Chazapis
from pithos.tools.lib.hashmap import HashMap
45 5a96180b Antony Chazapis
46 5a96180b Antony Chazapis
from migrate import Migration, Cache
47 77aa7567 Antony Chazapis
48 f6c0005f Sofia Papagiannaki
import os
49 f6c0005f Sofia Papagiannaki
    
50 77aa7567 Antony Chazapis
class DataMigration(Migration):
51 2db16f05 Sofia Papagiannaki
    def __init__(self, pithosdb, db):
52 2db16f05 Sofia Papagiannaki
        Migration.__init__(self,  pithosdb)
53 f6c0005f Sofia Papagiannaki
        self.cache = Cache(db)
54 77aa7567 Antony Chazapis
    
55 f6c0005f Sofia Papagiannaki
    def retrieve_files(self):
56 77aa7567 Antony Chazapis
        # Loop for all available files.
57 98137a34 Sofia Papagiannaki
        filebody = Table('filebody', self.metadata, autoload=True)
58 98137a34 Sofia Papagiannaki
        s = select([filebody.c.storedfilepath])
59 98137a34 Sofia Papagiannaki
        rp = self.conn.execute(s)
60 f6c0005f Sofia Papagiannaki
        path = rp.fetchone()
61 f6c0005f Sofia Papagiannaki
        while path:
62 f6c0005f Sofia Papagiannaki
            yield path
63 f6c0005f Sofia Papagiannaki
            path = rp.fetchone()
64 98137a34 Sofia Papagiannaki
        rp.close()
65 f6c0005f Sofia Papagiannaki
    
66 f6c0005f Sofia Papagiannaki
    def execute(self):
67 f6c0005f Sofia Papagiannaki
        blocksize = self.backend.block_size
68 f6c0005f Sofia Papagiannaki
        blockhash = self.backend.hash_algorithm
69 98137a34 Sofia Papagiannaki
        
70 f6c0005f Sofia Papagiannaki
        for (path,) in self.retrieve_files():
71 77aa7567 Antony Chazapis
            map = HashMap(blocksize, blockhash)
72 f6c0005f Sofia Papagiannaki
            try:
73 f6c0005f Sofia Papagiannaki
                map.load(open(path))
74 f6c0005f Sofia Papagiannaki
            except Exception, e:
75 f6c0005f Sofia Papagiannaki
                print e
76 f6c0005f Sofia Papagiannaki
                continue
77 77aa7567 Antony Chazapis
            hash = hexlify(map.hash())
78 77aa7567 Antony Chazapis
            
79 f6c0005f Sofia Papagiannaki
            if hash != self.cache.get(path):
80 77aa7567 Antony Chazapis
                missing = self.backend.blocker.block_ping(map) # XXX Backend hack...
81 77aa7567 Antony Chazapis
                status = '[>] ' + path
82 77aa7567 Antony Chazapis
                if missing:
83 77aa7567 Antony Chazapis
                    status += ' - %d block(s) missing' % len(missing)
84 77aa7567 Antony Chazapis
                    with open(path) as fp:
85 77aa7567 Antony Chazapis
                        for h in missing:
86 77aa7567 Antony Chazapis
                            offset = map.index(h) * blocksize
87 77aa7567 Antony Chazapis
                            fp.seek(offset)
88 77aa7567 Antony Chazapis
                            block = fp.read(blocksize)
89 77aa7567 Antony Chazapis
                            self.backend.put_block(block)
90 77aa7567 Antony Chazapis
                else:
91 77aa7567 Antony Chazapis
                    status += ' - no blocks missing'
92 f6c0005f Sofia Papagiannaki
                self.cache.put(path, hash)
93 77aa7567 Antony Chazapis
            else:
94 77aa7567 Antony Chazapis
                status = '[-] ' + path
95 77aa7567 Antony Chazapis
            print status
96 345fffa0 Antony Chazapis
97 77aa7567 Antony Chazapis
if __name__ == "__main__":
98 800a1bce Sofia Papagiannaki
    pithosdb = 'postgresql://gss@127.0.0.1/pithos'
99 77aa7567 Antony Chazapis
    db = 'sqlite:///migrate.db'
100 77aa7567 Antony Chazapis
    
101 2db16f05 Sofia Papagiannaki
    dt = DataMigration(pithosdb, db)
102 77aa7567 Antony Chazapis
    dt.execute()