Synnefo » Pithos

#!/usr/bin/env python

# Copyright 2011-2012 GRNET S.A. All rights reserved.

# 

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

# 

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

# 

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

# 

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

# 

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

from sqlalchemy import Table

from sqlalchemy.sql import select, and_

from binascii import hexlify

from pithos.backends.lib.hashfiler import Blocker

from pithos.backends.lib.sqlalchemy import Node

from django.conf import settings

from pithos.backends.modular import CLUSTER_NORMAL, CLUSTER_HISTORY, CLUSTER_DELETED

from pithos.backends.lib.sqlalchemy.node import Node, ROOTNODE

from pithos.lib.transfer import upload

from pithos.lib.hashmap import HashMap

from pithos.lib.client import Fault

from migrate import Migration, Cache

from calendar import timegm

from decimal import Decimal

from collections import defaultdict

import json

import os

import sys

import hashlib

import mimetypes

import time

import datetime

(ID, CREATIONDATE, MODIFICATIONDATE, DELETED, ICON, NAME, VERSION, CREATEDBY_ID, MODIFIEDBY_ID, OWNER_ID, PARENT_ID, READFORALL, SHARED, USER) = range(14)

class ObjectMigration(Migration):

    def __init__(self, old_db, db, f):

        Migration.__init__(self, old_db)

        self.cache = Cache(db)

    def create_node(self, username, container, object):

        node = self.backend.node.node_lookup(object)

        if not node:

            parent_path = '%s/%s' %(username, container)

            parent_node = self.backend.node.node_lookup(parent_path)

            if not parent_node:

                raise Exception('Missing node')

            node = self.backend.node.node_create(parent_node, object)

        return node

    def create_history(self, header_id, node_id, deleted=False):

        i = 0

        map = HashMap(self.backend.block_size, self.backend.hash_algorithm)

        v = []

        stored_versions = self.backend.node.node_get_versions(node_id, ['mtime'])

        stored_versions_mtime = [datetime.datetime.utcfromtimestamp(elem[0]) for elem in stored_versions]

        for t, rowcount  in self.retrieve_node_versions(header_id):

            size, modyfied_by, filepath, mimetype, mdate = t

            if mdate in stored_versions_mtime:

                continue

            cluster = CLUSTER_HISTORY if i < rowcount - 1 else CLUSTER_NORMAL

            cluster = cluster if not deleted else CLUSTER_DELETED

            hash = self.cache.get(filepath)

            if hash == None:

                raise Exception("Missing hash")

            args = node_id, hash, size, modyfied_by, cluster, mimetype, mdate

            v.append(self.create_version(*args))

            i += 1

        return v

    def create_version(self, node_id, hash, size, modyfied_by, cluster, mimetype, mdate):

        args = (node_id, hash, size, None, modyfied_by, cluster)

        serial = self.backend.node.version_create(*args)[0]

        meta = {'hash':hash,

                'content-type':mimetype}

        self.backend.node.attribute_set(serial, ((k, v) for k, v in meta.iteritems()))

        timestamp = timegm(mdate.timetuple())

        microseconds = mdate.time().microsecond

        values = timestamp, microseconds, serial

        f.write('update versions set mtime=\'%10d.%6d\' where serial=%s;' %values)

        return serial

    def create_tags(self, header_id, node_id, vserials):

        tags = self.retrieve_tags(header_id)

        if not tags:

            return

        for v in vserials:

            self.backend.node.attribute_set(v, (('X-Object-Meta-Tag', tags),))

    def create_permissions(self, fid, path, owner, is_folder=True):

        fpath, fpermissions = self.backend.permissions.access_inherit(path)

        permissions = self.retrieve_permissions(fid, is_folder)

        if not fpermissions:

            keys = ('read', 'write')

            for k in keys:

                if owner in permissions[k]:

                    permissions[k].remove(owner)

            self.backend.permissions.access_set(path, permissions)

        else:

            keys = ('read', 'write')

            common_p = {}

            for k in keys:

                if owner in permissions[k]:

                    permissions[k].remove(owner)

                common = set(fpermissions[k]).intersection(set(permissions[k]))

                common_p[k] = list(common)

            #keep only the common permissions

            #trade off for securing access only to explicitly authorized users

            self.backend.permissions.access_set(fpath, common_p)

    def create_objects(self):

        for t in self.retrieve_current_nodes():

            username, headerid, folderid, filename, deleted, filepath, mimetype, public, owner_id = t

            containers = ['pithos', 'trash']

            for c in containers:

                #create container if it does not exist

                try:

                    self.backend._lookup_container(username, c)

                except NameError, e:

                    self.backend.put_container(username, username, c) 

            container = 'pithos' if not deleted else 'trash'

            path = self.build_path(folderid)

            #create node

            object = '%s/%s' %(username, container)

            object = '%s/%s/%s' %(object, path, filename) if path else '%s/%s' %(object, filename)

            args = username, container, object

            nodeid = self.create_node(*args)

            #create node history 

            vserials = self.create_history(headerid, nodeid, deleted)

            #set object tags

            self.create_tags(headerid, nodeid, vserials)

            #set object's publicity

            if public:

                self.backend.permissions.public_set(object)

            #set object's permissions

            self.create_permissions(headerid, object, username, is_folder=False)

    def build_path(self, child_id):

        folder = Table('folder', self.metadata, autoload=True)

        user = Table('gss_user', self.metadata, autoload=True)

        j = folder.join(user, folder.c.owner_id == user.c.id)

        s = select([folder, user.c.username], from_obj=j)

        s = s.where(folder.c.id == child_id)

        s.order_by(folder.c.modificationdate)

        rp = self.conn.execute(s)

        t = rp.fetchone()

        md5 = hashlib.md5()

        hash = md5.hexdigest().lower()

        size = 0

        if not t[PARENT_ID]:

            return ''

        else:

            container_path = t[USER]

            container_path += '/trash' if t[DELETED] else '/pithos'

            parent_node = self.backend.node.node_lookup(container_path)

            if not parent_node:

                raise Exception('Missing node:', container_path)

            parent_path = self.build_path(t[PARENT_ID])

            path = '%s/%s/%s' %(container_path, parent_path, t[NAME]) if parent_path else '%s/%s' %(container_path, t[NAME])

            node = self.backend.node.node_lookup(path)

            if not node:

                node = self.backend.node.node_create(parent_node, path)

                if not node:

                    raise Exception('Unable to create node:', path)

                #create versions

                v = self.create_version(node, hash, size, t[USER], CLUSTER_NORMAL, 'application/directory', t[CREATIONDATE])

                if t[CREATIONDATE] != t[MODIFICATIONDATE]:

                    self.backend.node.version_recluster(v, CLUSTER_HISTORY)

                    self.create_version(node, hash, size, t[USER], CLUSTER_NORMAL, 'application/directory', t[MODIFICATIONDATE])

                #set permissions

                self.create_permissions(t[ID], path, t[USER], is_folder=True)

            return '%s/%s' %(parent_path, t[NAME]) if parent_path else t[NAME]

    def retrieve_current_nodes(self):

        fileheader = Table('fileheader', self.metadata, autoload=True)

        filebody = Table('filebody', self.metadata, autoload=True)

        folder = Table('folder', self.metadata, autoload=True)

        gss_user = Table('gss_user', self.metadata, autoload=True)

        j = filebody.join(fileheader, filebody.c.id == fileheader.c.currentbody_id)

        j = j.join(folder, fileheader.c.folder_id == folder.c.id)

        j = j.join(gss_user, fileheader.c.owner_id == gss_user.c.id)

        s = select([gss_user.c.username,  fileheader.c.id, fileheader.c.folder_id,

                    fileheader.c.name,  fileheader.c.deleted,

                    filebody.c.storedfilepath, filebody.c.mimetype,

                    fileheader.c.readforall, fileheader.c.owner_id], from_obj=j)

        rp = self.conn.execute(s)

        object = rp.fetchone()

        while object:

            yield object

            object = rp.fetchone()

        rp.close()

    def retrieve_node_versions(self, header_id):

        filebody = Table('filebody', self.metadata, autoload=True)

        gss_user = Table('gss_user', self.metadata, autoload=True)

        j = filebody.join(gss_user, filebody.c.modifiedby_id == gss_user.c.id)

        s = select([filebody.c.filesize, gss_user.c.username,

                    filebody.c.storedfilepath, filebody.c.mimetype,

                    filebody.c.modificationdate], from_obj=j)

        s = s.where(filebody.c.header_id == header_id)

        s = s.order_by(filebody.c.version)

        rp = self.conn.execute(s)

        version = rp.fetchone()

        while version:

            yield version, rp.rowcount

            version = rp.fetchone()

        rp.close()

    def retrieve_tags(self, header_id):

        filetag = Table('filetag', self.metadata, autoload=True)

        s = select([filetag.c.tag], filetag.c.fileid == header_id)

        rp = self.conn.execute(s)

        tags = rp.fetchall() if rp.returns_rows else []

        tags = [elem[0] for elem in tags]

        rp.close()

        return ','.join(tags) if tags else ''

    def retrieve_permissions(self, id, is_folder=True):

        permissions = {}

        if is_folder:

            ftable = Table('folder_permission', self.metadata, autoload=True)

        else:

            ftable = Table('fileheader_permission', self.metadata, autoload=True)

        permission = Table('permission', self.metadata, autoload=True)

        group = Table('gss_group', self.metadata, autoload=True)

        user = Table('gss_user', self.metadata, autoload=True)

        j = ftable.join(permission, ftable.c.permissions_id == permission.c.id)

        j1 = j.join(group, group.c.id == permission.c.group_id)

        j2 = j.join(user, user.c.id == permission.c.user_id)

        permissions = defaultdict(list)

        def _get_permissions(self, action='read', get_groups=True):

            if get_groups:

                col, j = group.c.name, j1

                cond2 = permission.c.group_id != None

            else:

                col, j = user.c.username, j2

                cond2 = permission.c.user_id != None

            s = select([col], from_obj=j)

            if is_folder:

                s = s.where(ftable.c.folder_id == id)

            else:

                s = s.where(ftable.c.fileheader_id == id)

            if action == 'read':

                cond1 = permission.c.read == True

            else:

                cond1 = permission.c.write == True

            s = s.where(and_(cond1, cond2))

            print '>', s, s.compile().params

            rp = self.conn.execute(s)

            p = permissions[action].extend([e[0] for e in rp.fetchall()])

            rp.close()

            return p

        #get object read groups

        _get_permissions(self, action='read', get_groups=True)

        #get object read users

        _get_permissions(self, action='read', get_groups=False)

        #get object write groups

        _get_permissions(self, action='write', get_groups=True)

        #get object write groups

        _get_permissions(self, action='write', get_groups=False)

        return permissions

if __name__ == "__main__":

    old_db = ''

    db = ''

    f = open('fixdates.sql', 'w')

    ot = ObjectMigration(old_db, db, f)

    ot.create_objects()

    f.close()