Synnefo

#!/usr/bin/env python

# Copyright 2011 GRNET S.A. All rights reserved.

# 

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

# 

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

# 

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

# 

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

# 

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

from sqlalchemy import Table

from sqlalchemy.sql import select

from binascii import hexlify

from pithos.backends.lib.hashfiler import Blocker

from pithos.backends.lib.sqlalchemy import Node

from django.conf import settings

from pithos.backends.modular import CLUSTER_NORMAL, CLUSTER_HISTORY, CLUSTER_DELETED

from pithos.backends.lib.sqlalchemy.node import Node, ROOTNODE

from lib.transfer import upload

from lib.hashmap import HashMap, file_read_iterator

from lib.client import Fault

from lib.migrate import Migration, Cache

from calendar import timegm

from decimal import Decimal

from collections import defaultdict

import json

import os

import sys

import hashlib

import mimetypes

import time

import datetime

(ID, CREATIONDATE, MODIFICATIONDATE, DELETED, ICON, NAME, VERSION, CREATEDBY_ID, MODIFIEDBY_ID, OWNER_ID, PARENT_ID, READFORALL, SHARED, USER) = range(14)

class ObjectMigration(Migration):

    def __init__(self, old_db, db, f):

        Migration.__init__(self, old_db)

        self.cache = Cache(db)

    def create_node(self, username, container, object):

        node = self.backend.node.node_lookup(object)

        if not node:

            parent_path = '%s/%s' %(username, container)

            parent_node = self.backend.node.node_lookup(parent_path)

            if not parent_node:

                raise Exception('Missing node')

            node = self.backend.node.node_create(parent_node, object)

        return node

    def create_history(self, header_id, node_id, deleted=False):

        i = 0

        map = HashMap(self.backend.block_size, self.backend.hash_algorithm)

        v = []

        stored_versions = self.backend.node.node_get_versions(node_id, ['mtime'])

        stored_versions_mtime = [datetime.datetime.utcfromtimestamp(elem[0]) for elem in stored_versions]

        for t, rowcount  in self.retrieve_node_versions(header_id):

            size, modyfied_by, filepath, mimetype, mdate = t

            if mdate in stored_versions_mtime:

                continue

            cluster = CLUSTER_HISTORY if i < rowcount - 1 else CLUSTER_NORMAL

            cluster = cluster if not deleted else CLUSTER_DELETED

            hash = self.cache.get(filepath)

            if hash == None:

                raise Exception("Missing hash")

            args = node_id, hash, size, modyfied_by, cluster, mimetype, mdate

            v.append(self.create_version(*args))

            i += 1

        return v

    def create_version(self, node_id, hash, size, modyfied_by, cluster, mimetype, mdate):

        args = (node_id, hash, size, None, modyfied_by, cluster)

        serial = self.backend.node.version_create(*args)[0]

        meta = {'hash':hash,

                'content-type':mimetype}

        self.backend.node.attribute_set(serial, ((k, v) for k, v in meta.iteritems()))

        timestamp = timegm(mdate.timetuple())

        microseconds = mdate.time().microsecond

        values = timestamp, microseconds, serial

        f.write('update versions set mtime=\'%10d.%6d\' where serial=%s;' %values)

        return serial

    def create_tags(self, header_id, node_id, vserials):

        tags = self.retrieve_tags(header_id)

        if not tags:

            return

        for v in vserials:

            self.backend.node.attribute_set(v, (('tag', tags),))

    def create_permissions(self, fid, path, owner):

        fpath, permissions = self.backend.permissions.access_inherit(path)

        if not permissions:

            permissions = self.retrieve_permissions(fid)

            keys = ('read', 'write')

            for k in keys:

                if owner in permissions[k]:

                    permissions[k].remove(owner)

            self.backend.permissions.access_set(path, permissions)

    def create_objects(self):

        for t in self.retrieve_current_nodes():

            username, headerid, folderid, filename, deleted, filepath, mimetype, public, owner_id = t

            containers = ['pithos', 'trash']

            for c in containers:

                #create container if it does not exist

                try:

                    self.backend._lookup_container(username, c)

                except NameError, e:

                    self.backend.put_container(username, username, c) 

            container = 'pithos' if not deleted else 'trash'

            path = self.build_path(folderid)

            #create node

            object = '%s/%s/%s/%s' %(username, container, path, filename)

            args = username, container, object, filepath, mimetype

            nodeid = self.create_node(*args)

            #create node history 

            vserials = self.create_history(headerid, nodeid, deleted)

            #set object tags

            self.create_tags(headerid, nodeid, vserials)

            #set object's publicity

            if public:

                self.backend.permissions.public_set(object)

            #set object's permissions

            self.create_permissions(headerid, object, username)

    def build_path(self, child_id):

        folder = Table('folder', self.metadata, autoload=True)

        user = Table('gss_user', self.metadata, autoload=True)

        j = folder.join(user, folder.c.owner_id == user.c.id)

        s = select([folder, user.c.username], from_obj=j)

        s = s.where(folder.c.id == child_id)

        s.order_by(folder.c.modificationdate)

        rp = self.conn.execute(s)

        t = rp.fetchone()

        md5 = hashlib.md5()

        hash = md5.hexdigest().lower()

        size = 0

        if not t[PARENT_ID]:

            return ''

        else:

            container_path = t[USER]

            container_path += '/trash' if t[DELETED] else '/pithos'

            parent_node = self.backend.node.node_lookup(container_path)

            if not parent_node:

                raise Exception('Missing node:', container_path)

            parent_path = self.build_path(t[PARENT_ID])

            path = '%s/%s/%s' %(container_path, parent_path, t[NAME]) if parent_path else '%s/%s' %(container_path, t[NAME])

            node = self.backend.node.node_lookup(path)

            if not node:

                node = self.backend.node.node_create(parent_node, path)

                if not node:

                    raise Exception('Unable to create node:', path)

                #create versions

                v = self.create_version(node, hash, size, t[USER], CLUSTER_NORMAL, 'application/directory', t[CREATIONDATE])

                if t[CREATIONDATE] != t[MODIFICATIONDATE]:

                    self.backend.node.version_recluster(v, CLUSTER_HISTORY)

                    self.create_version(node, hash, size, t[USER], CLUSTER_NORMAL, 'application/directory', t[MODIFICATIONDATE])

                #set permissions

                self.create_permissions(t[ID], path, t[USER])

            return '%s/%s' %(parent_path, t[NAME]) if parent_path else t[NAME]

    def retrieve_current_nodes(self):

        fileheader = Table('fileheader', self.metadata, autoload=True)

        filebody = Table('filebody', self.metadata, autoload=True)

        folder = Table('folder', self.metadata, autoload=True)

        gss_user = Table('gss_user', self.metadata, autoload=True)

        j = filebody.join(fileheader, filebody.c.id == fileheader.c.currentbody_id)

        j = j.join(folder, fileheader.c.folder_id == folder.c.id)

        j = j.join(gss_user, fileheader.c.owner_id == gss_user.c.id)

        s = select([gss_user.c.username,  fileheader.c.id, fileheader.c.folder_id,

                    fileheader.c.name,  fileheader.c.deleted,

                    filebody.c.storedfilepath, filebody.c.mimetype,

                    fileheader.c.readforall, fileheader.c.owner_id], from_obj=j)

        rp = self.conn.execute(s)

        object = rp.fetchone()

        while object:

            yield object

            object = rp.fetchone()

        rp.close()

    def retrieve_node_versions(self, header_id):

        filebody = Table('filebody', self.metadata, autoload=True)

        gss_user = Table('gss_user', self.metadata, autoload=True)

        j = filebody.join(gss_user, filebody.c.modifiedby_id == gss_user.c.id)

        s = select([filebody.c.filesize, gss_user.c.username,

                    filebody.c.storedfilepath, filebody.c.mimetype,

                    filebody.c.modificationdate], from_obj=j)

        s = s.where(filebody.c.header_id == header_id)

        s = s.order_by(filebody.c.version)

        rp = self.conn.execute(s)

        version = rp.fetchone()

        while version:

            yield version, rp.rowcount

            version = rp.fetchone()

        rp.close()

    def retrieve_tags(self, header_id):

        filetag = Table('filetag', self.metadata, autoload=True)

        s = select([filetag.c.tag], filetag.c.fileid == header_id)

        rp = self.conn.execute(s)

        tags = rp.fetchall() if rp.returns_rows else []

        tags = [elem[0] for elem in tags]

        rp.close()

        return ','.join(tags) if tags else ''

    def retrieve_permissions(self, id, is_folder=True):

        permissions = {}

        if is_folder:

            ftable = Table('folder_permission', self.metadata, autoload=True)

        else:

            ftable = Table('fileheader_permission', self.metadata, autoload=True)

        permission = Table('permission', self.metadata, autoload=True)

        group = Table('gss_group', self.metadata, autoload=True)

        user = Table('gss_user', self.metadata, autoload=True)

        j = ftable.join(permission, ftable.c.permissions_id == permission.c.id)

        j1 = j.join(group, group.c.id == permission.c.group_id)

        j2 = j.join(user, user.c.id == permission.c.user_id)

        permissions = defaultdict(list)

        #get folder read groups

        s = select([group.c.name], from_obj=j1)

        if is_folder:

            s = s.where(ftable.c.folder_id == id)

        else:

            s = s.where(ftable.c.fileheader_id == id)

        s = s.where(permission.c.read == True)

        s = s.where(permission.c.group_id != None)

        rp = self.conn.execute(s)

        permissions['read'].extend([e[0] for e in rp.fetchall()])

        #get folder read users

        s = select([user.c.username], from_obj=j2)

        if is_folder:

            s = s.where(ftable.c.folder_id == id)

        else:

            s = s.where(ftable.c.fileheader_id == id)

        s = s.where(permission.c.read == True)

        s = s.where(permission.c.user_id != None)

        rp = self.conn.execute(s)

        permissions['read'].extend([e[0] for e in rp.fetchall()])

        #get folder write groups

        s = select([group.c.name], from_obj=j1)

        if is_folder:

            s = s.where(ftable.c.folder_id == id)

        else:

            s = s.where(ftable.c.fileheader_id == id)

        s = s.where(permission.c.write == True)

        s = s.where(permission.c.group_id != None)

        rp = self.conn.execute(s)

        permissions['write'].extend([e[0] for e in rp.fetchall()])

        #get folder write groups

        s = select([user.c.username], from_obj=j2)

        if is_folder:

            s = s.where(ftable.c.folder_id == id)

        else:

            s = s.where(ftable.c.fileheader_id == id)

        s = s.where(permission.c.write == True)

        s = s.where(permission.c.user_id != None)

        rp = self.conn.execute(s)

        permissions['write'].extend([e[0] for e in rp.fetchall()])

        rp.close()

        return permissions

if __name__ == "__main__":

    old_db = ''

    db = ''

    f = open('fixdates.sql', 'w')

    ot = ObjectMigration(old_db, db, f)

    ot.create_objects()

    f.close()