Statistics
| Branch: | Tag: | Revision:

root / contrib / migrate-data @ 363a5cdb

History | View | Annotate | Download (3.6 kB)

1
#!/usr/bin/env python
2

    
3
# Copyright 2011-2012 GRNET S.A. All rights reserved.
4
# 
5
# Redistribution and use in source and binary forms, with or
6
# without modification, are permitted provided that the following
7
# conditions are met:
8
# 
9
#   1. Redistributions of source code must retain the above
10
#      copyright notice, this list of conditions and the following
11
#      disclaimer.
12
# 
13
#   2. Redistributions in binary form must reproduce the above
14
#      copyright notice, this list of conditions and the following
15
#      disclaimer in the documentation and/or other materials
16
#      provided with the distribution.
17
# 
18
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29
# POSSIBILITY OF SUCH DAMAGE.
30
# 
31
# The views and conclusions contained in the software and
32
# documentation are those of the authors and should not be
33
# interpreted as representing official policies, either expressed
34
# or implied, of GRNET S.A.
35

    
36
from binascii import hexlify
37

    
38
from sqlalchemy import Table
39
from sqlalchemy.sql import select
40

    
41
from pithos import settings
42
from pithos.backends.modular import ModularBackend
43

    
44
from pithos.tools.lib.hashmap import HashMap
45

    
46
from migrate import Migration, Cache
47

    
48
import os
49
    
50
class DataMigration(Migration):
51
    def __init__(self, pithosdb, db):
52
        Migration.__init__(self,  pithosdb)
53
        self.cache = Cache(db)
54
    
55
    def retrieve_files(self):
56
        # Loop for all available files.
57
        filebody = Table('filebody', self.metadata, autoload=True)
58
        s = select([filebody.c.storedfilepath])
59
        rp = self.conn.execute(s)
60
        path = rp.fetchone()
61
        while path:
62
            yield path
63
            path = rp.fetchone()
64
        rp.close()
65
    
66
    def execute(self):
67
        blocksize = self.backend.block_size
68
        blockhash = self.backend.hash_algorithm
69
        
70
        for (path,) in self.retrieve_files():
71
            map = HashMap(blocksize, blockhash)
72
            try:
73
                map.load(open(path))
74
            except Exception, e:
75
                print e
76
                continue
77
            hash = hexlify(map.hash())
78
            
79
            if hash != self.cache.get(path):
80
                missing = self.backend.blocker.block_ping(map) # XXX Backend hack...
81
                status = '[>] ' + path
82
                if missing:
83
                    status += ' - %d block(s) missing' % len(missing)
84
                    with open(path) as fp:
85
                        for h in missing:
86
                            offset = map.index(h) * blocksize
87
                            fp.seek(offset)
88
                            block = fp.read(blocksize)
89
                            self.backend.put_block(block)
90
                else:
91
                    status += ' - no blocks missing'
92
                self.cache.put(path, hash)
93
            else:
94
                status = '[-] ' + path
95
            print status
96

    
97
if __name__ == "__main__":
98
    pithosdb = 'postgresql://gss@127.0.0.1/pithos'
99
    db = 'sqlite:///migrate.db'
100
    
101
    dt = DataMigration(pithosdb, db)
102
    dt.execute()