Synnefo

include pithos/settings.d/*

prune other

Using the tools

---------------

In the pithos/tools directory you will find the following utilities:

    pithos-sh       Pithos shell

    pithos-sync     Pithos synchronization client

    pithos-fs       Pithos FUSE implementation

    pithos-test     Pithos server tests

Also, the pithos/lib directory contains a python library that can be used

to access Pithos and manage stored objects. All tools use the included lib.

Connection options can be set via environmental variables:

    PITHOS_USER             Login user

    PITHOS_AUTH             Login token

    PITHOS_SERVER           Pithos server (default: plus.pithos.grnet.gr)

    PITHOS_API              Pithos server path (default: v1)

    PITHOS_SYNC_CONTAINER   Container to sync with (default: pithos)

Store

~~~~~

.. automodule:: pithos.backends.lib.hashfiler.store

.. automodule:: pithos.lib.client

#!/usr/bin/env python

import sys

if len(sys.argv) != 4:

	print "Usage: %s <inviter token> <invitee name> <invitee email>" % (sys.argv[0],)

	sys.exit(-1)

import httplib2

http = httplib2.Http(disable_ssl_certificate_validation=True)

url = 'https://pithos.dev.grnet.gr/im/invite'

import urllib

params = urllib.urlencode({

	'uniq': sys.argv[3],

	'realname': sys.argv[2]

})

response, content = http.request(url, 'POST', params,

	headers={'Content-type': 'application/x-www-form-urlencoded', 'X-Auth-Token': sys.argv[1]}

)

if response['status'] == '200':

	print 'OK'

	sys.exit(0)

else:

	print response, content

	sys.exit(-1)

#!/usr/bin/env python

# Copyright 2011 GRNET S.A. All rights reserved.

# 

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

# 

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

# 

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

# 

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

# 

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

from binascii import hexlify

from sqlalchemy import Table

from sqlalchemy.sql import select

from pithos import settings

from pithos.backends.modular import ModularBackend

from pithos.lib.hashmap import HashMap

from migrate import Migration, Cache

import os

class DataMigration(Migration):

    def __init__(self, pithosdb, db):

        Migration.__init__(self,  pithosdb)

        self.cache = Cache(db)

    def retrieve_files(self):

        # Loop for all available files.

        filebody = Table('filebody', self.metadata, autoload=True)

        s = select([filebody.c.storedfilepath])

        rp = self.conn.execute(s)

        path = rp.fetchone()

        while path:

            yield path

            path = rp.fetchone()

        rp.close()

    def execute(self):

        blocksize = self.backend.block_size

        blockhash = self.backend.hash_algorithm

        for (path,) in self.retrieve_files():

            map = HashMap(blocksize, blockhash)

            try:

                map.load(open(path))

            except Exception, e:

                print e

                continue

            hash = hexlify(map.hash())

            if hash != self.cache.get(path):

                missing = self.backend.blocker.block_ping(map) # XXX Backend hack...

                status = '[>] ' + path

                if missing:

                    status += ' - %d block(s) missing' % len(missing)

                    with open(path) as fp:

                        for h in missing:

                            offset = map.index(h) * blocksize

                            fp.seek(offset)

                            block = fp.read(blocksize)

                            self.backend.put_block(block)

                else:

                    status += ' - no blocks missing'

                self.cache.put(path, hash)

            else:

                status = '[-] ' + path

            print status

if __name__ == "__main__":

    pithosdb = 'postgresql://gss@127.0.0.1/pithos'

    db = 'sqlite:///migrate.db'

    dt = DataMigration(pithosdb, db)

    dt.execute()

#!/usr/bin/env python

# Copyright 2011 GRNET S.A. All rights reserved.

# 

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

# 

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

# 

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

# 

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

# 

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

from sqlalchemy import Table

from sqlalchemy.sql import select, and_

from binascii import hexlify

from pithos.backends.lib.hashfiler import Blocker

from pithos.backends.lib.sqlalchemy import Node

from django.conf import settings

from pithos.backends.modular import CLUSTER_NORMAL, CLUSTER_HISTORY, CLUSTER_DELETED

from pithos.backends.lib.sqlalchemy.node import Node, ROOTNODE

from pithos.lib.transfer import upload

from pithos.lib.hashmap import HashMap

from pithos.lib.client import Fault

from migrate import Migration, Cache

from calendar import timegm

from decimal import Decimal

from collections import defaultdict

import json

import os

import sys

import hashlib

import mimetypes

import time

import datetime

(ID, CREATIONDATE, MODIFICATIONDATE, DELETED, ICON, NAME, VERSION, CREATEDBY_ID, MODIFIEDBY_ID, OWNER_ID, PARENT_ID, READFORALL, SHARED, USER) = range(14)

class ObjectMigration(Migration):

    def __init__(self, old_db, db, f):

        Migration.__init__(self, old_db)

        self.cache = Cache(db)

    def create_node(self, username, container, object):

        node = self.backend.node.node_lookup(object)

        if not node:

            parent_path = '%s/%s' %(username, container)

            parent_node = self.backend.node.node_lookup(parent_path)

            if not parent_node:

                raise Exception('Missing node')

            node = self.backend.node.node_create(parent_node, object)

        return node

    def create_history(self, header_id, node_id, deleted=False):

        i = 0

        map = HashMap(self.backend.block_size, self.backend.hash_algorithm)

        v = []

        stored_versions = self.backend.node.node_get_versions(node_id, ['mtime'])

        stored_versions_mtime = [datetime.datetime.utcfromtimestamp(elem[0]) for elem in stored_versions]

        for t, rowcount  in self.retrieve_node_versions(header_id):

            size, modyfied_by, filepath, mimetype, mdate = t

            if mdate in stored_versions_mtime:

                continue

            cluster = CLUSTER_HISTORY if i < rowcount - 1 else CLUSTER_NORMAL

            cluster = cluster if not deleted else CLUSTER_DELETED

            hash = self.cache.get(filepath)

            if hash == None:

                raise Exception("Missing hash")

            args = node_id, hash, size, modyfied_by, cluster, mimetype, mdate

            v.append(self.create_version(*args))

            i += 1

        return v

    def create_version(self, node_id, hash, size, modyfied_by, cluster, mimetype, mdate):

        args = (node_id, hash, size, None, modyfied_by, cluster)

        serial = self.backend.node.version_create(*args)[0]

        meta = {'hash':hash,

                'content-type':mimetype}

        self.backend.node.attribute_set(serial, ((k, v) for k, v in meta.iteritems()))

        timestamp = timegm(mdate.timetuple())

        microseconds = mdate.time().microsecond

        values = timestamp, microseconds, serial

        f.write('update versions set mtime=\'%10d.%6d\' where serial=%s;' %values)

        return serial

    def create_tags(self, header_id, node_id, vserials):

        tags = self.retrieve_tags(header_id)

        if not tags:

            return

        for v in vserials:

            self.backend.node.attribute_set(v, (('X-Object-Meta-Tag', tags),))

    def create_permissions(self, fid, path, owner, is_folder=True):

        fpath, fpermissions = self.backend.permissions.access_inherit(path)

        permissions = self.retrieve_permissions(fid, is_folder)

        if not fpermissions:

            keys = ('read', 'write')

            for k in keys:

                if owner in permissions[k]:

                    permissions[k].remove(owner)

            self.backend.permissions.access_set(path, permissions)

        else:

            keys = ('read', 'write')

            common_p = {}

            for k in keys:

                if owner in permissions[k]:

                    permissions[k].remove(owner)

                common = set(fpermissions[k]).intersection(set(permissions[k]))

                common_p[k] = list(common)

            #keep only the common permissions

            #trade off for securing access only to explicitly authorized users

            self.backend.permissions.access_set(fpath, common_p)

    def create_objects(self):

        for t in self.retrieve_current_nodes():

            username, headerid, folderid, filename, deleted, filepath, mimetype, public, owner_id = t

            containers = ['pithos', 'trash']

            for c in containers:

                #create container if it does not exist

                try:

                    self.backend._lookup_container(username, c)

                except NameError, e:

                    self.backend.put_container(username, username, c) 

            container = 'pithos' if not deleted else 'trash'

            path = self.build_path(folderid)

            #create node

            object = '%s/%s' %(username, container)

            object = '%s/%s/%s' %(object, path, filename) if path else '%s/%s' %(object, filename)

            args = username, container, object

            nodeid = self.create_node(*args)

            #create node history 

            vserials = self.create_history(headerid, nodeid, deleted)

            #set object tags

            self.create_tags(headerid, nodeid, vserials)

            #set object's publicity

            if public:

                self.backend.permissions.public_set(object)

            #set object's permissions

            self.create_permissions(headerid, object, username, is_folder=False)

    def build_path(self, child_id):

        folder = Table('folder', self.metadata, autoload=True)

        user = Table('gss_user', self.metadata, autoload=True)

        j = folder.join(user, folder.c.owner_id == user.c.id)

        s = select([folder, user.c.username], from_obj=j)

        s = s.where(folder.c.id == child_id)

        s.order_by(folder.c.modificationdate)

        rp = self.conn.execute(s)

        t = rp.fetchone()

        md5 = hashlib.md5()

        hash = md5.hexdigest().lower()

        size = 0

        if not t[PARENT_ID]:

            return ''

        else:

            container_path = t[USER]

            container_path += '/trash' if t[DELETED] else '/pithos'

            parent_node = self.backend.node.node_lookup(container_path)

            if not parent_node:

                raise Exception('Missing node:', container_path)

            parent_path = self.build_path(t[PARENT_ID])

            path = '%s/%s/%s' %(container_path, parent_path, t[NAME]) if parent_path else '%s/%s' %(container_path, t[NAME])

            node = self.backend.node.node_lookup(path)

            if not node:

                node = self.backend.node.node_create(parent_node, path)

                if not node:

                    raise Exception('Unable to create node:', path)

                #create versions

                v = self.create_version(node, hash, size, t[USER], CLUSTER_NORMAL, 'application/directory', t[CREATIONDATE])

                if t[CREATIONDATE] != t[MODIFICATIONDATE]:

                    self.backend.node.version_recluster(v, CLUSTER_HISTORY)

                    self.create_version(node, hash, size, t[USER], CLUSTER_NORMAL, 'application/directory', t[MODIFICATIONDATE])

                #set permissions

                self.create_permissions(t[ID], path, t[USER], is_folder=True)

            return '%s/%s' %(parent_path, t[NAME]) if parent_path else t[NAME]

    def retrieve_current_nodes(self):

        fileheader = Table('fileheader', self.metadata, autoload=True)

        filebody = Table('filebody', self.metadata, autoload=True)

        folder = Table('folder', self.metadata, autoload=True)

        gss_user = Table('gss_user', self.metadata, autoload=True)

        j = filebody.join(fileheader, filebody.c.id == fileheader.c.currentbody_id)

        j = j.join(folder, fileheader.c.folder_id == folder.c.id)

        j = j.join(gss_user, fileheader.c.owner_id == gss_user.c.id)

        s = select([gss_user.c.username,  fileheader.c.id, fileheader.c.folder_id,

                    fileheader.c.name,  fileheader.c.deleted,

                    filebody.c.storedfilepath, filebody.c.mimetype,

                    fileheader.c.readforall, fileheader.c.owner_id], from_obj=j)

        rp = self.conn.execute(s)

        object = rp.fetchone()

        while object:

            yield object

            object = rp.fetchone()

        rp.close()

    def retrieve_node_versions(self, header_id):

        filebody = Table('filebody', self.metadata, autoload=True)

        gss_user = Table('gss_user', self.metadata, autoload=True)

        j = filebody.join(gss_user, filebody.c.modifiedby_id == gss_user.c.id)

        s = select([filebody.c.filesize, gss_user.c.username,

                    filebody.c.storedfilepath, filebody.c.mimetype,

                    filebody.c.modificationdate], from_obj=j)

        s = s.where(filebody.c.header_id == header_id)

        s = s.order_by(filebody.c.version)

        rp = self.conn.execute(s)

        version = rp.fetchone()

        while version:

            yield version, rp.rowcount

            version = rp.fetchone()

        rp.close()

    def retrieve_tags(self, header_id):

        filetag = Table('filetag', self.metadata, autoload=True)

        s = select([filetag.c.tag], filetag.c.fileid == header_id)

        rp = self.conn.execute(s)

        tags = rp.fetchall() if rp.returns_rows else []

        tags = [elem[0] for elem in tags]

        rp.close()

        return ','.join(tags) if tags else ''

    def retrieve_permissions(self, id, is_folder=True):

        permissions = {}

        if is_folder:

            ftable = Table('folder_permission', self.metadata, autoload=True)

        else:

            ftable = Table('fileheader_permission', self.metadata, autoload=True)

        permission = Table('permission', self.metadata, autoload=True)

        group = Table('gss_group', self.metadata, autoload=True)

        user = Table('gss_user', self.metadata, autoload=True)

        j = ftable.join(permission, ftable.c.permissions_id == permission.c.id)

        j1 = j.join(group, group.c.id == permission.c.group_id)

        j2 = j.join(user, user.c.id == permission.c.user_id)

        permissions = defaultdict(list)

        def _get_permissions(self, action='read', get_groups=True):

            if get_groups:

                col, j = group.c.name, j1

                cond2 = permission.c.group_id != None

            else:

                col, j = user.c.username, j2

                cond2 = permission.c.user_id != None

            s = select([col], from_obj=j)

            if is_folder:

                s = s.where(ftable.c.folder_id == id)

            else:

                s = s.where(ftable.c.fileheader_id == id)

            if action == 'read':

                cond1 = permission.c.read == True

            else:

                cond1 = permission.c.write == True

            s = s.where(and_(cond1, cond2))

            print '>', s, s.compile().params

            rp = self.conn.execute(s)

            p = permissions[action].extend([e[0] for e in rp.fetchall()])

            rp.close()

            return p

        #get object read groups

        _get_permissions(self, action='read', get_groups=True)

        #get object read users

        _get_permissions(self, action='read', get_groups=False)

        #get object write groups

        _get_permissions(self, action='write', get_groups=True)

        #get object write groups

        _get_permissions(self, action='write', get_groups=False)

        return permissions

if __name__ == "__main__":

    old_db = ''

    db = ''

    f = open('fixdates.sql', 'w')

    ot = ObjectMigration(old_db, db, f)

    ot.create_objects()

    f.close()

#!/usr/bin/env python

# Copyright 2011 GRNET S.A. All rights reserved.

# 

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

# 

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

# 

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

# 

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

# 

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

from sqlalchemy import Table

from sqlalchemy.sql import select

from pithos.im.models import User

from migrate import Migration

import base64

class UserMigration(Migration):

    def __init__(self, db):

        Migration.__init__(self, db)

        self.gss_users = Table('gss_user', self.metadata, autoload=True)

    def execute(self):

        for u in self.retrieve_users():

            user = User()

            user.pk = u['id']

            user.uniq = u['username']

            user.realname = u['name']

            user.affiliation = u['homeorganization'] if u['homeorganization'] else ''

            user.auth_token = base64.b64encode(u['authtoken'])

            user.auth_token_created = u['creationdate']

            user.auth_token_expires = u['authtokenexpirydate']

            user.created = u['creationdate']

            user.updated = u['modificationdate']

            user.email = u['email']

            user.active = 'ACTIVE' if u['active'] else 'SUSPENDED'

            print '#', user

            user.save(update_timestamps=False)

            #create user groups

            for (owner, group, members) in self.retrieve_groups(u['username']):

                self.backend.permissions.group_addmany(owner, group, members)

    def retrieve_users(self):

        s = self.gss_users.select()

        rp = self.conn.execute(s)

        user = rp.fetchone()

        while user:

            yield user

            user = rp.fetchone()

        rp.close()

    def retrieve_groups(self, owner):

        gss_group = Table('gss_group', self.metadata, autoload=True)

        gss_user = Table('gss_user', self.metadata, autoload=True)

        group_user = Table('gss_group_gss_user', self.metadata, autoload=True)

        j1 = gss_group.join(gss_user, gss_group.c.owner_id == gss_user.c.id)

        j2 = group_user.join(gss_user, group_user.c.members_id == gss_user.c.id)

        s = select([gss_group.c.id, gss_group.c.name, gss_user.c.username], from_obj=j1)

        s = s.where(gss_user.c.username == owner)

        rp = self.conn.execute(s)

        gr = rp.fetchone()

        while gr:

            id, group, owner = gr

            s = select([gss_user.c.username], from_obj=j2)

            s = s.where(group_user.c.groupsmember_id == id)

            rp2 = self.conn.execute(s)

            members = rp2.fetchall()

            rp2.close()

            yield owner, group, (m[0] for m in members)

            gr = rp.fetchone()

        rp.close()

if __name__ == "__main__":

    db = ''

    m = UserMigration(db)

    m.execute()

#!/usr/bin/env python

# Copyright 2011 GRNET S.A. All rights reserved.

# 

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

# 

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

# 

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

# 

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

# 

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

from sqlalchemy import create_engine

from sqlalchemy import Table, Column, String, MetaData

from sqlalchemy.sql import select

from django.conf import settings

from pithos.backends.modular import ModularBackend

class Migration(object):

    def __init__(self, db):

        self.engine = create_engine(db)

        self.metadata = MetaData(self.engine)

        #self.engine.echo = True

        self.conn = self.engine.connect()

        options = getattr(settings, 'BACKEND', None)[1]

        self.backend = ModularBackend(*options)

    def execute(self):

        pass

class Cache():

    def __init__(self, db):

        self.engine = create_engine(db)

        metadata = MetaData(self.engine)

        columns=[]

        columns.append(Column('path', String(2048), primary_key=True))

        columns.append(Column('hash', String(255)))

        self.files = Table('files', metadata, *columns)

        self.conn = self.engine.connect()

        self.engine.echo = True

        metadata.create_all(self.engine)

    def put(self, path, hash):

        # Insert or replace.

        s = self.files.delete().where(self.files.c.path==path)

        r = self.conn.execute(s)

        r.close()

        s = self.files.insert()

        r = self.conn.execute(s, {'path': path, 'hash': hash})

        r.close()

    def get(self, path):

        s = select([self.files.c.hash], self.files.c.path == path)

        r = self.conn.execute(s)

        l = r.fetchone()

        r.close()

        if not l:

            return l

        return l[0]

# Copyright (c) Django Software Foundation and individual contributors.

# All rights reserved.

# 

# Redistribution and use in source and binary forms, with or without modification,

# are permitted provided that the following conditions are met:

# 

#     1. Redistributions of source code must retain the above copyright notice, 

#        this list of conditions and the following disclaimer.

#     

#     2. Redistributions in binary form must reproduce the above copyright 

#        notice, this list of conditions and the following disclaimer in the

#        documentation and/or other materials provided with the distribution.

# 

#     3. Neither the name of Django nor the names of its contributors may be used

#        to endorse or promote products derived from this software without

#        specific prior written permission.

# 

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND

# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR

# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES

# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;

# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON

# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS

# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

VERSION = (0, 8, 2, 'alpha', 0)

def get_version():

    version = '%s.%s' % (VERSION[0], VERSION[1])

    if VERSION[2]:

        version = '%s.%s' % (version, VERSION[2])

    if VERSION[3:] == ('alpha', 0):

        version = '%s pre-alpha' % version

    else:

        if VERSION[3] != 'final':

            version = '%s %s %s' % (version, VERSION[3], VERSION[4])

    return version

from pithos.lib.compat import parse_http_date_safe, parse_http_date

from pithos.lib.hashmap import HashMap

# Copyright 2011 GRNET S.A. All rights reserved.

# 

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

# 

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

# 

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

# 

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

# 

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

from httplib import HTTPConnection, HTTP

from sys import stdin

from xml.dom import minidom

from StringIO import StringIO

from urllib import quote, unquote

import json

import types

import socket

import urllib

import datetime

ERROR_CODES = {304:'Not Modified',

               400:'Bad Request',

               401:'Unauthorized',

               403:'Forbidden',

               404:'Not Found',

               409:'Conflict',

               411:'Length Required',

               412:'Precondition Failed',

               413:'Request Entity Too Large',

               416:'Range Not Satisfiable',

               422:'Unprocessable Entity',

               503:'Service Unavailable',

               }

class Fault(Exception):

    def __init__(self, data='', status=None):

        if data == '' and status in ERROR_CODES.keys():

            data = ERROR_CODES[status]

        Exception.__init__(self, data)

        self.data = data

        self.status = status

class Client(object):

    def __init__(self, host, token, account, api='v1', verbose=False, debug=False):

        """`host` can also include a port, e.g '127.0.0.1:8000'."""

        self.host = host

        self.account = account

        self.api = api

        self.verbose = verbose or debug

        self.debug = debug

        self.token = token

    def _req(self, method, path, body=None, headers={}, format='text', params={}):

        slash = '/' if self.api else ''

        full_path = '%s%s%s?format=%s' % (slash, self.api, quote(path), format)

        for k,v in params.items():

            if v:

                full_path = '%s&%s=%s' %(full_path, quote(k), quote(str(v)))

            else:

                full_path = '%s&%s=' %(full_path, k)

        conn = HTTPConnection(self.host)

        kwargs = {}

        for k,v in headers.items():

            headers.pop(k)

            k = k.replace('_', '-')

            headers[quote(k)] = quote(v, safe='/=,:@ *"') if type(v) == types.StringType else v

        kwargs['headers'] = headers

        kwargs['headers']['X-Auth-Token'] = self.token

        if body:

            kwargs['body'] = body

            kwargs['headers'].setdefault('content-type', 'application/octet-stream')

        kwargs['headers'].setdefault('content-length', len(body) if body else 0)

        #print '#', method, full_path, kwargs

        t1 = datetime.datetime.utcnow()

        conn.request(method, full_path, **kwargs)

        resp = conn.getresponse()

        t2 = datetime.datetime.utcnow()

        #print 'response time:', str(t2-t1)

        headers = resp.getheaders()

        headers = dict((unquote(h), unquote(v)) for h,v in headers)

        if self.verbose:

            print '%d %s' % (resp.status, resp.reason)

            for key, val in headers.items():

                print '%s: %s' % (key.capitalize(), val)

            print

        length = resp.getheader('content-length', None)

        data = resp.read(length)

        if self.debug:

            print data

            print

        if int(resp.status) in ERROR_CODES.keys():

            raise Fault(data, int(resp.status))

        #print '**',  resp.status, headers, data, '\n'

        return resp.status, headers, data

    def _chunked_transfer(self, path, method='PUT', f=stdin, headers=None,

                          blocksize=1024, params={}):

        """perfomrs a chunked request"""

        http = HTTPConnection(self.host)

        # write header

        full_path = '/%s%s?' % (self.api, path)

        for k,v in params.items():

            if v:

                full_path = '%s&%s=%s' %(full_path, k, v)

            else:

                full_path = '%s&%s=' %(full_path, k)

        full_path = urllib.quote(full_path, '?&:=/')

        http.putrequest(method, full_path)

        http.putheader('x-auth-token', self.token)

        http.putheader('content-type', 'application/octet-stream')

        http.putheader('transfer-encoding', 'chunked')

        if headers:

            for header,value in headers.items():

                http.putheader(header, value)

        http.endheaders()

        # write body

        data = ''

        while True:

            if f.closed:

                break

            block = f.read(blocksize)

            if block == '':

                break

            data = '%x\r\n%s\r\n' % (len(block), block)

            try:

                http.send(data)

            except:

                #retry

                http.send(data)

        data = '0\r\n\r\n'

        try:

            http.send(data)

        except:

            #retry

            http.send(data)

        # get response

        resp = http.getresponse()

        headers = dict(resp.getheaders())

        if self.verbose:

            print '%d %s' % (resp.status, resp.reason)

            for key, val in headers.items():

                print '%s: %s' % (key.capitalize(), val)

            print

        length = resp.getheader('Content-length', None)

        data = resp.read(length)

        if self.debug:

            print data

            print

        if int(resp.status) in ERROR_CODES.keys():

            raise Fault(data, int(resp.status))

        #print '*',  resp.status, headers, data

        return resp.status, headers, data

    def delete(self, path, format='text', params={}):

        return self._req('DELETE', path, format=format, params=params)

    def get(self, path, format='text', headers={}, params={}):

        return self._req('GET', path, headers=headers, format=format,

                        params=params)

    def head(self, path, format='text', params={}):

         return self._req('HEAD', path, format=format, params=params)

    def post(self, path, body=None, format='text', headers=None, params={}):

        return self._req('POST', path, body, headers=headers, format=format,

                        params=params)

    def put(self, path, body=None, format='text', headers=None, params={}):

        return self._req('PUT', path, body, headers=headers, format=format,

                         params=params)

    def _list(self, path, format='text', params={}, **headers):

        status, headers, data = self.get(path, format=format, headers=headers,

                                         params=params)

        if format == 'json':

            data = json.loads(data) if data else ''

        elif format == 'xml':

            data = minidom.parseString(data)

        else:

            data = data.split('\n')[:-1] if data else ''

        return data

    def _get_metadata(self, path, prefix=None, params={}):

        status, headers, data = self.head(path, params=params)

        prefixlen = len(prefix) if prefix else 0

        meta = {}

        for key, val in headers.items():

            if prefix and not key.startswith(prefix):

                continue

            elif prefix and key.startswith(prefix):

                key = key[prefixlen:]

            meta[key] = val

        return meta

    def _filter(self, l, d):

        """

        filter out from l elements having the metadata values provided

        """

        ll = l

        for elem in l:

            if type(elem) == types.DictionaryType:

                for key in d.keys():

                    k = 'x_object_meta_%s' % key

                    if k in elem.keys() and elem[k] == d[key]:

                        ll.remove(elem)

                        break

        return ll

class OOS_Client(Client):

    """Openstack Object Storage Client"""

    def _update_metadata(self, path, entity, **meta):

        """adds new and updates the values of previously set metadata"""

        ex_meta = self.retrieve_account_metadata(restricted=True)

        ex_meta.update(meta)

        headers = {}

        prefix = 'x-%s-meta-' % entity

        for k,v in ex_meta.items():

            k = '%s%s' % (prefix, k)

            headers[k] = v

        return self.post(path, headers=headers)

    def _reset_metadata(self, path, entity, **meta):

        """

        overwrites all user defined metadata

        """

        headers = {}

        prefix = 'x-%s-meta-' % entity

        for k,v in meta.items():

            k = '%s%s' % (prefix, k)

            headers[k] = v

        return self.post(path, headers=headers)

    def _delete_metadata(self, path, entity, meta=[]):

        """delete previously set metadata"""

        ex_meta = self.retrieve_account_metadata(restricted=True)

        headers = {}

        prefix = 'x-%s-meta-' % entity

        for k in ex_meta.keys():

            if k in meta:

                headers['%s%s' % (prefix, k)] = ex_meta[k]

        return self.post(path, headers=headers)

    # Storage Account Services

    def list_containers(self, format='text', limit=None,

                        marker=None, params={}, account=None, **headers):

        """lists containers"""

        account = account or self.account

        path = '/%s' % account

        params.update({'limit':limit, 'marker':marker})

        return self._list(path, format, params, **headers)

    def retrieve_account_metadata(self, restricted=False, account=None, **params):

        """returns the account metadata"""

        account = account or self.account

        path = '/%s' % account

        prefix = 'x-account-meta-' if restricted else None

        return self._get_metadata(path, prefix, params)

    def update_account_metadata(self, account=None, **meta):

        """updates the account metadata"""

        account = account or self.account

        path = '/%s' % account

        return self._update_metadata(path, 'account', **meta)

    def delete_account_metadata(self, meta=[], account=None):

        """deletes the account metadata"""

        account = account or self.account

        path = '/%s' % account

        return self._delete_metadata(path, 'account', meta)

    def reset_account_metadata(self, account=None, **meta):

        """resets account metadata"""

        account = account or self.account

        path = '/%s' % account

        return self._reset_metadata(path, 'account', **meta)

    # Storage Container Services

    def _filter_trashed(self, l):

        return self._filter(l, {'trash':'true'})

    def list_objects(self, container, format='text',

                     limit=None, marker=None, prefix=None, delimiter=None,

                     path=None, include_trashed=False, params={}, account=None,

                     **headers):

        """returns a list with the container objects"""

        account = account or self.account

        params.update({'limit':limit, 'marker':marker, 'prefix':prefix,

                       'delimiter':delimiter, 'path':path})

        l = self._list('/%s/%s' % (account, container), format, params,

                       **headers)

        #TODO support filter trashed with xml also

        if format != 'xml' and not include_trashed:

            l = self._filter_trashed(l)

        return l

    def create_container(self, container, account=None, **meta):

        """creates a container"""

        account = account or self.account

        headers = {}

        for k,v in meta.items():

            headers['x-container-meta-%s' %k.strip().upper()] = v.strip()

        status, header, data = self.put('/%s/%s' % (account, container),

                                        headers=headers)

        if status == 202:

            return False

        elif status != 201:

            raise Fault(data, int(status))

        return True

    def delete_container(self, container, params={}, account=None):

        """deletes a container"""

        account = account or self.account

        return self.delete('/%s/%s' % (account, container), params=params)

    def retrieve_container_metadata(self, container, restricted=False,

                                    account=None, **params):

        """returns the container metadata"""

        account = account or self.account

        prefix = 'x-container-meta-' if restricted else None

        return self._get_metadata('/%s/%s' % (account, container), prefix,

                                  params)

    def update_container_metadata(self, container, account=None, **meta):

        """unpdates the container metadata"""

        account = account or self.account

        return self._update_metadata('/%s/%s' % (account, container),

                                     'container', **meta)

    def delete_container_metadata(self, container, meta=[], account=None):

        """deletes the container metadata"""

        account = account or self.account

        path = '/%s/%s' % (account, container)

        return self._delete_metadata(path, 'container', meta)

    # Storage Object Services

    def request_object(self, container, object, format='text', params={},

                       account=None, **headers):

        """returns tuple containing the status, headers and data response for an object request"""

        account = account or self.account

        path = '/%s/%s/%s' % (account, container, object)

        status, headers, data = self.get(path, format, headers, params)

        return status, headers, data

    def retrieve_object(self, container, object, format='text', params={},

                        account=None, **headers):

        """returns an object's data"""

        account = account or self.account

        t = self.request_object(container, object, format, params, account,

                                **headers)

        data = t[2]

        if format == 'json':

            data = json.loads(data) if data else ''

        elif format == 'xml':

            data = minidom.parseString(data)

        return data

    def retrieve_object_hashmap(self, container, object, params={},

                        account=None, **headers):

        """returns the hashmap representing object's data"""

        args = locals().copy()

        for elem in ['self', 'container', 'object']:

            args.pop(elem)

        return self.retrieve_object(container, object, format='json', **args)

    def create_directory_marker(self, container, object, account=None):

        """creates a dierectory marker"""

        account = account or self.account

        if not object:

            raise Fault('Directory markers have to be nested in a container')

        h = {'content_type':'application/directory'}

        return self.create_zero_length_object(container, object, account=account,

                                              **h)

    def create_object(self, container, object, f=stdin, format='text', meta={},

                      params={}, etag=None, content_type=None, content_encoding=None,

                      content_disposition=None, account=None, **headers):

        """creates a zero-length object"""

        account = account or self.account

        path = '/%s/%s/%s' % (account, container, object)

        for k, v  in headers.items():

            if v == None:

                headers.pop(k)

        l = ['etag', 'content_encoding', 'content_disposition', 'content_type']

        l = [elem for elem in l if eval(elem)]

        for elem in l:

            headers.update({elem:eval(elem)})

        headers.setdefault('content-type', 'application/octet-stream')

        for k,v in meta.items():

            headers['x-object-meta-%s' %k.strip()] = v.strip()

        data = f.read() if f else None

        return self.put(path, data, format, headers=headers, params=params)

    def create_zero_length_object(self, container, object, meta={}, etag=None,

                                  content_type=None, content_encoding=None,

                                  content_disposition=None, account=None,

                                  **headers):

        account = account or self.account

        args = locals().copy()

        for elem in ['self', 'container', 'headers', 'account']:

            args.pop(elem)

        args.update(headers)

        return self.create_object(container, account=account, f=None, **args)

    def update_object(self, container, object, f=stdin,

                      offset=None, meta={}, params={}, content_length=None,

                      content_type=None, content_encoding=None,

                      content_disposition=None,  account=None, **headers):

        account = account or self.account

        path = '/%s/%s/%s' % (account, container, object)

        for k, v  in headers.items():

            if v == None:

                headers.pop(k)

        l = ['content_encoding', 'content_disposition', 'content_type',

             'content_length']

        l = [elem for elem in l if eval(elem)]

        for elem in l:

            headers.update({elem:eval(elem)})

        if 'content_range' not in headers.keys():

            if offset != None:

                headers['content_range'] = 'bytes %s-/*' % offset

            else:

                headers['content_range'] = 'bytes */*'

        for k,v in meta.items():

            headers['x-object-meta-%s' %k.strip()] = v.strip()

        data = f.read() if f else None

        return self.post(path, data, headers=headers, params=params)

    def update_object_using_chunks(self, container, object, f=stdin,

                                   blocksize=1024, offset=None, meta={},

                                   params={}, content_type=None, content_encoding=None,

                                   content_disposition=None, account=None, **headers):

        """updates an object (incremental upload)"""

        account = account or self.account

        path = '/%s/%s/%s' % (account, container, object)

        headers = headers if not headers else {}

        l = ['content_type', 'content_encoding', 'content_disposition']

        l = [elem for elem in l if eval(elem)]

        for elem in l:

            headers.update({elem:eval(elem)})

        if offset != None:

            headers['content_range'] = 'bytes %s-/*' % offset

        else:

            headers['content_range'] = 'bytes */*'

        for k,v in meta.items():

            v = v.strip()

            headers['x-object-meta-%s' %k.strip()] = v

        return self._chunked_transfer(path, 'POST', f, headers=headers,

                                      blocksize=blocksize, params=params)

    def _change_obj_location(self, src_container, src_object, dst_container,

                             dst_object, remove=False, meta={}, account=None,

                             content_type=None, **headers):

        account = account or self.account

        path = '/%s/%s/%s' % (account, dst_container, dst_object)

        headers = {} if not headers else headers

        for k, v in meta.items():

            headers['x-object-meta-%s' % k] = v

        if remove:

            headers['x-move-from'] = '/%s/%s' % (src_container, src_object)

        else:

            headers['x-copy-from'] = '/%s/%s' % (src_container, src_object)

        headers['content_length'] = 0

        if content_type:

            headers['content_type'] = content_type 

        return self.put(path, headers=headers)

    def copy_object(self, src_container, src_object, dst_container, dst_object,

                   meta={}, account=None, content_type=None, **headers):

        """copies an object"""

        account = account or self.account

        return self._change_obj_location(src_container, src_object,

                                   dst_container, dst_object, account=account,

                                   remove=False, meta=meta,

                                   content_type=content_type, **headers)

    def move_object(self, src_container, src_object, dst_container,

                             dst_object, meta={}, account=None,

                             content_type=None, **headers):

        """moves an object"""

        account = account or self.account