Synnefo » Pithos

from modular import ModularBackend

    def put_account(self, user, account):

        """Create a new account with the given name.

        Raises:

            NotAllowedError: Operation not permitted

        """

        return

# Copyright 2011 GRNET S.A. All rights reserved.

# 

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

# 

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

# 

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

# 

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

# 

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

class DBWorker(object):

    """Database connection handler."""

    def __init__(self, **params):

        self.params = params

        conn = params['connection']

        cur = params['cursor']

        self.execute = cur.execute

        self.executemany = cur.executemany

        self.fetchone = cur.fetchone

        self.fetchall = cur.fetchall

        self.cur = cur

        self.conn = conn

# Copyright 2011 GRNET S.A. All rights reserved.

# 

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

# 

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

# 

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

# 

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

# 

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

from dbworker import DBWorker

class Groups(DBWorker):

    """Groups are named collections of members, belonging to an owner."""

    def __init__(self, **params):

        DBWorker.__init__(self, **params)

        execute = self.execute

        execute(""" create table if not exists groups

                          ( owner  text,

                            name   text,

                            member text,

                            primary key (owner, name, member) ) """)

        execute(""" create index if not exists idx_groups_member

                    on groups(member) """)

    def group_names(self, owner):

        """List all group names belonging to owner."""

        q = "select distinct name from groups where owner = ?"

        self.execute(q, (owner,))

        return [r[0] for r in self.fetchall()]

    def group_list(self, owner):

        """List all (group, member) tuples belonging to owner."""

        q = "select name, member from groups where owner = ?"

        self.execute(q, (owner,))

        return self.fetchall()

    def group_add(self, owner, group, member):

        """Add a member to a group."""

        q = "insert or ignore into groups (owner, name, member) values (?, ?, ?)"

        self.execute(q, (owner, group, member))

    def group_addmany(self, owner, group, members):

        """Add members to a group."""

        q = "insert or ignore into groups (owner, name, member) values (?, ?, ?)"

        self.executemany(q, ((owner, group, member) for member in members))

    def group_remove(self, owner, group, member):

        """Remove a member from a group."""

        q = "delete from groups where owner = ? and name = ? and member = ?"

        self.execute(q, (owner, group, member))

    def group_delete(self, owner, group):

        """Delete a group."""

        q = "delete from groups where owner = ? and name = ?"

        self.execute(q, (owner, group))

    def group_destroy(self, owner):

        """Delete all groups belonging to owner."""

        q = "delete from groups where owner = ?"

        self.execute(q, (owner,))

    def group_members(self, owner, group):

        """Return the list of members of a group."""

        q = "select member from groups where owner = ? and name = ?"

        self.execute(q, (owner, group))

        return [r[0] for r in self.fetchall()]

    def group_check(self, owner, group, member):

        """Check if a member is in a group."""

        q = "select 1 from groups where owner = ? and name = ? and member = ?"

        self.execute(q, (group, member))

        return bool(self.fetchone())

    def group_parents(self, member):

        """Return all (owner, group) tuples that contain member."""

        q = "select owner, name from groups where member = ?"

        self.execute(q, (member,))

        return self.fetchall()

# Copyright 2011 GRNET S.A. All rights reserved.

# 

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

# 

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

# 

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

# 

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

# 

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

from blocker import Blocker

from mapper import Mapper

# Copyright 2011 GRNET S.A. All rights reserved.

# 

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

# 

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

# 

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

# 

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

# 

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

from os import makedirs

from os.path import isdir, realpath, exists, join

from hashlib import new as newhasher

from binascii import hexlify

from pithos.lib.hashfiler.context_file import ContextFile, file_sync_read_chunks

class Blocker(object):

    """Blocker.

       Required contstructor parameters: blocksize, blockpath, hashtype.

    """

    blocksize = None

    blockpath = None

    hashtype = None

    def __init__(self, **params):

        blocksize = params['blocksize']

        blockpath = params['blockpath']

        blockpath = realpath(blockpath)

        if not isdir(blockpath):

            if not exists(blockpath):

                makedirs(blockpath)

            else:

                raise ValueError("Variable blockpath '%s' is not a directory" % (blockpath,))

        hashtype = params['hashtype']

        try:

            hasher = newhasher(hashtype)

        except ValueError:

            msg = "Variable hashtype '%s' is not available from hashlib"

            raise ValueError(msg % (hashtype,))

        hasher.update("")

        emptyhash = hasher.digest()

        self.blocksize = blocksize

        self.blockpath = blockpath

        self.hashtype = hashtype

        self.hashlen = len(emptyhash)

        self.emptyhash = emptyhash

    def get_rear_block(self, blkhash, create=0):

        name = join(self.blockpath, hexlify(blkhash))

        return ContextFile(name, create)

    def check_rear_block(self, blkhash):

        name = join(self.blockpath, hexlify(blkhash))

        return exists(name)

    def block_hash(self, data):

        """Hash a block of data"""

        hasher = newhasher(self.hashtype)

        hasher.update(data.rstrip('\x00'))

        return hasher.digest()

    def block_ping(self, hashes):

        """Check hashes for existence and

           return those missing from block storage.

        """

        missing = []

        append = missing.append

        for i, h in enumerate(hashes):

            if not self.check_rear_block(h):

                append(i)

        return missing

    def block_retr(self, hashes):

        """Retrieve blocks from storage by their hashes."""

        blocksize = self.blocksize

        blocks = []

        append = blocks.append

        block = None

        for h in hashes:

            with self.get_rear_block(h, 0) as rbl:

                if not rbl:

                    break

                for block in rbl.sync_read_chunks(blocksize, 1, 0):

                    break # there should be just one block there

            if not block:

                break

            append(block)

        return blocks

    def block_stor(self, blocklist):

        """Store a bunch of blocks and return (hashes, missing).

           Hashes is a list of the hashes of the blocks,

           missing is a list of indices in that list indicating

           which blocks were missing from the store.

        """

        block_hash = self.block_hash

        hashlist = [block_hash(b) for b in blocklist]

        mf = None

        missing = self.block_ping(hashlist)

        for i in missing:

            with self.get_rear_block(hashlist[i], 1) as rbl:

                 rbl.sync_write(blocklist[i]) #XXX: verify?

        return hashlist, missing

    def block_delta(self, blkhash, offdata=()):

        """Construct and store a new block from a given block

           and a list of (offset, data) 'patches'. Return:

           (the hash of the new block, if the block already existed)

        """

        if not offdata:

            return None, None

        blocksize = self.blocksize

        block = self.block_retr((blkhash,))

        if not block:

            return None, None

        block = block[0]

        newblock = ''

        idx = 0

        size = 0

        trunc = 0

        for off, data in offdata:

            if not data:

                trunc = 1

                break

            newblock += block[idx:off] + data

            size += off - idx + len(data)

            if size >= blocksize:

                break

            off = size

        if not trunc:

            newblock += block[size:len(block)]

        h, a = self.block_stor((newblock,))

        return h[0], 1 if a else 0

    def block_hash_file(self, openfile):

        """Return the list of hashes (hashes map)

           for the blocks in a buffered file.

           Helper method, does not affect store.

        """

        hashes = []

        append = hashes.append

        block_hash = self.block_hash

        for block in file_sync_read_chunks(openfile, self.blocksize, 1, 0):

            append(block_hash(block))

        return hashes

    def block_stor_file(self, openfile):

        """Read blocks from buffered file object and store them. Return:

           (bytes read, list of hashes, list of hashes that were missing)

        """

        blocksize = self.blocksize

        block_stor = self.block_stor

        hashlist = []

        hextend = hashlist.extend

        storedlist = []

        sextend = storedlist.extend

        lastsize = 0

        for block in file_sync_read_chunks(openfile, blocksize, 1, 0):

            hl, sl = block_stor((block,))

            hextend(hl)

            sextend(sl)

            lastsize = len(block)

        size = (len(hashlist) -1) * blocksize + lastsize if hashlist else 0

        return size, hashlist, storedlist

# Copyright 2011 GRNET S.A. All rights reserved.

# 

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

# 

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

# 

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

# 

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

# 

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

from os import SEEK_CUR, SEEK_SET, fsync

from errno import ENOENT

_zeros = ''

def zeros(nr):

    global _zeros

    size = len(_zeros)

    if nr == size:

        return _zeros

    if nr > size:

        _zeros += '\0' * (nr - size)

        return _zeros

    if nr < size:

        _zeros = _zeros[:nr]

        return _zeros

def file_sync_write_chunks(openfile, chunksize, offset, chunks, size=None):

    """Write given chunks to the given buffered file object.

       Writes never span across chunk boundaries.

       If size is given stop after or pad until size bytes have been written.

    """

    fwrite = openfile.write

    seek = openfile.seek

    padding = 0

    try:

        seek(offset * chunksize)

    except IOError, e:

        seek = None

        for x in xrange(offset):

            fwrite(zeros(chunksize))

    cursize = offset * chunksize

    for chunk in chunks:

        if padding:

            if seek:

                seek(padding -1, SEEK_CUR)

                fwrite("\x00")

            else:

                fwrite(buffer(zeros(chunksize), 0, padding))

        if size is not None and cursize + chunksize >= size:

            chunk = chunk[:chunksize - (cursize - size)]

            fwrite(chunk)

            cursize += len(chunk)

            break

        fwrite(chunk)

        padding = chunksize - len(chunk)

    padding = size - cursize if size is not None else 0

    if padding <= 0:

        return

    q, r = divmod(padding, chunksize)

    for x in xrange(q):

        fwrite(zeros(chunksize))

    fwrite(buffer(zeros(chunksize), 0, r))

def file_sync_read_chunks(openfile, chunksize, nr, offset=0):

    """Read and yield groups of chunks from a buffered file object at offset.

       Reads never span accros chunksize boundaries.

    """

    fread = openfile.read

    remains = offset * chunksize

    seek = openfile.seek

    try:

        seek(remains)

    except IOError, e:

        seek = None

        while 1:

            s = fread(remains)

            remains -= len(s)

            if remains <= 0:

                break

    while nr:

        remains = chunksize

        chunk = ''

        while 1:

            s = fread(remains)

            if not s:

                if chunk:

                    yield chunk

                return

            chunk += s

            remains -= len(s)

            if remains <= 0:

                break

        yield chunk

        nr -= 1

class ContextFile(object):

    __slots__ = ("name", "fdesc", "create")

    def __init__(self, name, create=0):

        self.name = name

        self.fdesc = None

        self.create = create

        #self.dirty = 0

    def __enter__(self):

        name = self.name

        try:

            fdesc = open(name, 'rb+')

        except IOError, e:

            if not self.create or e.errno != ENOENT:

                raise

            fdesc = open(name, 'w+')

        self.fdesc = fdesc

        return self

    def __exit__(self, exc, arg, trace):

        fdesc = self.fdesc

        if fdesc is not None:

            #if self.dirty:

            #    fsync(fdesc.fileno())

            fdesc.close()

        return False # propagate exceptions

    def seek(self, offset, whence=SEEK_SET):

        return self.fdesc.seek(offset, whence)

    def tell(self):

        return self.fdesc.tell()

    def truncate(self, size):

        self.fdesc.truncate(size)

    def sync_write(self, data):

        #self.dirty = 1

        self.fdesc.write(data)

    def sync_write_chunks(self, chunksize, offset, chunks, size=None):

        #self.dirty = 1

        return file_sync_write_chunks(self.fdesc, chunksize, offset, chunks, size)

    def sync_read(self, size):

        read = self.fdesc.read

        data = ''

        while 1:

            s = read(size)

            if not s:

                break

            data += s

        return data

    def sync_read_chunks(self, chunksize, nr, offset=0):

        return file_sync_read_chunks(self.fdesc, chunksize, nr, offset)

# Copyright 2011 GRNET S.A. All rights reserved.

# 

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

# 

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

# 

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

# 

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

# 

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

from os.path import realpath, join, exists, isdir

from os import makedirs, unlink

from errno import ENOENT

from pithos.lib.hashfiler.context_file import ContextFile

class Mapper(object):

    """Mapper.

       Required contstructor parameters: mappath, namelen.

    """

    mappath = None

    namelen = None

    def __init__(self, **params):

        self.params = params

        self.namelen = params['namelen']

        mappath = realpath(params['mappath'])

        if not isdir(mappath):

            if not exists(mappath):

                makedirs(mappath)

            else:

                raise ValueError("Variable mappath '%s' is not a directory" % (mappath,))

        self.mappath = mappath

    def get_rear_map(self, name, create=0):

        name = join(self.mappath, hex(int(name)))

        return ContextFile(name, create)

    def delete_rear_map(self, name):

        name = join(self.mappath, hex(int(name)))

        try:

            unlink(name)

            return 1

        except OSError, e:

            if e.errno != ENOENT:

                raise

        return 0

    def map_retr(self, name, blkoff=0, nr=100000000000000):

        """Return as a list, part of the hashes map of an object

           at the given block offset.

           By default, return the whole hashes map.

        """

        namelen = self.namelen

        hashes = ()

        with self.get_rear_map(name, 0) as rmap:

            if rmap:

                hashes = list(rmap.sync_read_chunks(namelen, nr, blkoff))

        return hashes

    def map_stor(self, name, hashes=(), blkoff=0, create=1):

        """Store hashes in the given hashes map, replacing the old ones."""

        namelen = self.namelen

        with self.get_rear_map(name, 1) as rmap:

            rmap.sync_write_chunks(namelen, blkoff, hashes, None)

#     def map_copy(self, src, dst):

#         """Copy a hashes map to another one, replacing it."""

#         with self.get_rear_map(src, 0) as rmap:

#             if rmap:

#                 rmap.copy_to(dst)

    def map_remv(self, name):

        """Remove a hashes map. Returns true if the map was found and removed."""

        return self.delete_rear_map(name)

# Copyright 2011 GRNET S.A. All rights reserved.

# 

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

# 

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

# 

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

# 

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

# 

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

from xfeatures import XFeatures

from groups import Groups

from public import Public

class Permissions(XFeatures, Groups, Public):

    def __init__(self, **params):

        XFeatures.__init__(self, **params)

        Groups.__init__(self, **params)

        Public.__init__(self, **params)

    def access_grant(self, access, path, member='all', members=()):

        """Grant a member with an access to a path."""

        xfeatures = self.xfeature_list(path)

        xfl = len(xfeatures)

        if xfl > 1 or (xfl == 1 and xfeatures[0][0] != path):

            return xfeatures

        if xfl == 0:

            feature = self.alloc_serial()

            self.xfeature_bestow(path, feature)

        else:

            fpath, feature = xfeatures[0]

        if members:

            self.feature_setmany(feature, access, members)

        else:

            self.feature_set(feature, access, member)

        return ()

    def access_revoke(self, access, path, member='all', members=()):

        """Revoke access to path from members.

           Note that this will not revoke access for members

           that are indirectly granted access through group membership.

        """

        # XXX: Maybe provide a force_revoke that will kick out

        #      all groups containing the given members?

        xfeatures = self.xfeature_list(path)

        xfl = len(xfeatures)

        if xfl != 1 or xfeatures[0][0] != path:

            return xfeatures

        fpath, feature = xfeatures[0]

        if members:

            self.feature_unsetmany(feature, access, members=members)

        else:

            self.feature_unset(feature, access, member)

        # XXX: provide a meaningful return value? 

        return ()

    def access_check(self, access, path, member):

        """Return true if the member has this access to the path."""

        r = self.xfeature_inherit(path)

        if not r:

            return 0

        fpath, feature = r

        memberset = set(self.feature_get(feature, access))

        if member in memberset:

            return 1

        for group in self.group_parents(self, member):

            if group in memberset:

                return 1

        return 0

    def access_list(self, path):

        """Return the list of (access, member) pairs for the path."""

        r = self.xfeature_inherit(path)

        if not r:

            return ()

        fpath, feature = r

        return self.feature_list(feature)

    def access_list_paths(self, member):

        """Return the list of (access, path) pairs granted to member."""

        q = ("select distinct key, path from xfeatures inner join "

             "   (select distinct feature, key from xfeaturevals inner join "

             "      (select name as value from members "

             "       where member = ? union select ?) "

             "    using (value)) "

             "using (feature)")

        self.execute(q, (member, member))

        return self.fetchall()

# Copyright 2011 GRNET S.A. All rights reserved.

# 

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

# 

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

# 

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

# 

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

# 

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

from dbworker import DBWorker

class Public(DBWorker):

    """Paths can be marked as public."""

    def __init__(self, **params):

        DBWorker.__init__(self, **params)

        execute = self.execute

        execute(""" create table if not exists public

                          ( path text primary key ) """)

    def public_set(self, path):

        q = "insert or ignore into public (path) values (?)"

        self.execute(q, (path,))

    def public_unset(self, path):

        q = "delete from public where path = ?"

        self.execute(q, (path,))

    def public_check(self, path):

        q = "select 1 from public where path = ?"

        self.execute(q, (path,))

        return bool(self.fetchone())

# Copyright 2011 GRNET S.A. All rights reserved.

# 

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

# 

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

# 

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

# 

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

# 

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

from dbworker import DBWorker

class XFeatures(DBWorker):

    """XFeatures are path properties that allow non-nested

       inheritance patterns. Currently used for storing permissions.

    """

    def __init__(self, **params):

        DBWorker.__init__(self, **params)

        execute = self.execute

        execute(""" pragma foreign_keys = on """)

        execute(""" create table if not exists xfeatures

                          ( feature_id integer primary key,

                            path       text ) """)

        execute(""" create unique index if not exists idx_features_path

                    on xfeatures(path) """)

        execute(""" create table if not exists xfeaturevals

                          ( feature_id integer,

                            key        integer,

                            value      text,

                            primary key (feature_id, key, value)

                            foreign key (feature_id) references xfeatures(feature_id)

                            on delete cascade ) """)

    def xfeature_inherit(self, path):

        """Return the (path, feature) inherited by the path, or None."""

        q = ("select path, feature_id from xfeatures "

             "where path <= ? "

             "order by path desc limit 1")

        self.execute(q, (path,))

        r = self.fetchone()

        if r is not None and path.startswith(r[0]):

            return r

        return None

    def xfeature_list(self, path):

        """Return the list of the (prefix, feature) pairs matching path.

           A prefix matches path if either the prefix includes the path,

           or the path includes the prefix.

        """

        inherited = self.xfeature_inherit(path)

        if inherited:

            return [inherited]

        q = ("select path, feature_id from xfeatures "

             "where path like ? and path != ? order by path")

        self.execute(q, (path + '%', path,))

        return self.fetchall()

    def xfeature_create(self, path):

        """Create and return a feature for path.

           If the path already inherits a feature or

           bestows to paths already inheriting a feature,

           create no feature and return None.

        """

        prefixes = self.xfeature_list(path)

        pl = len(prefixes)

        if (pl > 1) or (pl == 1 and prefixes[0][0] != path):

            return None

        q = "insert into xfeatures (path) values (?)"

        id = self.execute(q, (path,)).lastrowid

        return id

    def xfeature_destroy(self, path):

        """Destroy a feature and all its key, value pairs."""

        q = "delete from xfeatures where path = ?"

        self.execute(q, (path,))

    def feature_list(self, feature):

        """Return the list of all key, value pairs

           associated with a feature.

        """

        q = "select key, value from xfeaturevals where feature = ?"

        self.execute(q, (feature,))

        return self.fetchall()

    def feature_set(self, feature, key, value):

        """Associate a key, value pair with a feature."""

        q = "insert or ignore into xfeaturevals (feature_id, key, value) values (?, ?, ?)"

        self.execute(q, (feature, key, value))

    def feature_setmany(self, feature, key, values):

        """Associate the given key, and values with a feature."""

        q = "insert or ignore into xfeaturevals (feature_id, key, value) values (?, ?, ?)"

        self.executemany(q, ((feature, key, v) for v in values))

    def feature_unset(self, feature, key, value):

        """Disassociate a key, value pair from a feature."""

        q = ("delete from xfeaturevals where "

             "feature_id = ? and key = ? and value = ?")

        self.execute(q, (feature, key, value))

    def feature_unsetmany(self, feature, key, values):

        """Disassociate the key for the values given, from a feature."""

        q = ("delete from xfeaturevals where "

             "feature_id = ? and key = ? and value = ?")

        self.executemany(q, ((feature, key, v) for v in values))

    def feature_get(self, feature, key):

        """Return the list of values for a key of a feature."""

        q = "select value from xfeaturevals where feature_id = ? and key = ?"

        self.execute(q, (feature, key))

        return [r[0] for r in self.fetchall()]

    def feature_clear(self, feature, key):

        """Delete all key, value pairs for a key of a feature."""

        q = "delete from xfeaturevals where feature_id = ? and key = ?"

        self.execute(q, (feature, key))

# Copyright 2011 GRNET S.A. All rights reserved.

# 

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

# 

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

# 

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

# 

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

# 

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

import os

import time

import sqlite3

import logging

import hashlib

import binascii

from base import NotAllowedError, BaseBackend

from lib.permissions import Permissions

from lib.hashfiler import Mapper, Blocker

logger = logging.getLogger(__name__)

def backend_method(func=None, autocommit=1):

    if func is None:

        def fn(func):

            return backend_method(func, autocommit)

        return fn

    if not autocommit:

        return func

    def fn(self, *args, **kw):

        self.con.execute('begin deferred')

        try:

            ret = func(self, *args, **kw)

            self.con.commit()

            return ret

        except:

            self.con.rollback()

            raise

    return fn

class ModularBackend(BaseBackend):

    """A modular backend.

    Uses SQLite for storage.

    """

    # TODO: Create account if not present in all functions.

    def __init__(self, db):

        self.hash_algorithm = 'sha256'

        self.block_size = 4 * 1024 * 1024 # 4MB

        self.default_policy = {'quota': 0, 'versioning': 'auto'}

        basepath = os.path.split(db)[0]

        if basepath and not os.path.exists(basepath):

            os.makedirs(basepath)

        if not os.path.isdir(basepath):

            raise RuntimeError("Cannot open database at '%s'" % (db,))

        self.con = sqlite3.connect(basepath + '/db', check_same_thread=False)

        sql = '''pragma foreign_keys = on'''

        self.con.execute(sql)

        sql = '''create table if not exists versions (

                    version_id integer primary key,

                    name text,

                    user text,

                    tstamp integer not null,

                    size integer default 0,

                    hide integer default 0)'''

        self.con.execute(sql)

        sql = '''create table if not exists metadata (

                    version_id integer,

                    key text,

                    value text,

                    primary key (version_id, key)

                    foreign key (version_id) references versions(version_id)

                    on delete cascade)'''

        self.con.execute(sql)

        sql = '''create table if not exists policy (

                    name text, key text, value text, primary key (name, key))'''

        self.con.execute(sql)

        self.con.commit()

        params = {'blocksize': self.block_size,

                  'blockpath': basepath + '/blocks',

                  'hashtype': self.hash_algorithm}

        self.blocker = Blocker(**params)

        params = {'mappath': basepath + '/maps',

                  'namelen': self.blocker.hashlen}

        self.mapper = Mapper(**params)

        params = {'connection': self.con,

                  'cursor': self.con.cursor()}

        self.permissions = Permissions(**params)

    @backend_method

    def list_accounts(self, user, marker=None, limit=10000):

        """Return a list of accounts the user can access."""

        allowed = self._allowed_accounts(user)

        start, limit = self._list_limits(allowed, marker, limit)

        return allowed[start:start + limit]

    @backend_method

    def get_account_meta(self, user, account, until=None):

        """Return a dictionary with the account metadata."""

        logger.debug("get_account_meta: %s %s", account, until)

        if user != account:

            if until or account not in self._allowed_accounts(user):

                raise NotAllowedError

        else:

            self._create_account(user, account)

        try:

            version_id, mtime = self._get_accountinfo(account, until)

        except NameError:

            # Account does not exist before until.

            version_id = None

            mtime = until

        count, bytes, tstamp = self._get_pathstats(account, until)

        if mtime > tstamp:

            tstamp = mtime

        if until is None:

            modified = tstamp

        else:

            modified = self._get_pathstats(account)[2] # Overall last modification

            if mtime > modified:

                modified = mtime

        # Proper count.

        sql = 'select count(name) from (%s) where name glob ? and not name glob ?'

        sql = sql % self._sql_until(until)

        c = self.con.execute(sql, (account + '/*', account + '/*/*'))

        row = c.fetchone()

        count = row[0]

        if user != account:

            meta = {'name': account}

        else:

            meta = self._get_metadata(account, version_id)

            meta.update({'name': account, 'count': count, 'bytes': bytes})

            if until is not None:

                meta.update({'until_timestamp': tstamp})

        meta.update({'modified': modified})

        return meta

    @backend_method

    def update_account_meta(self, user, account, meta, replace=False):

        """Update the metadata associated with the account."""

        logger.debug("update_account_meta: %s %s %s", account, meta, replace)

        if user != account:

            raise NotAllowedError

        self._put_metadata(user, account, meta, replace, False)

    @backend_method

    def get_account_groups(self, user, account):

        """Return a dictionary with the user groups defined for this account."""

        logger.debug("get_account_groups: %s", account)

        if user != account:

            if account not in self._allowed_accounts(user):

                raise NotAllowedError

            return {}

        self._create_account(user, account)

        return self._get_groups(account)

    @backend_method

    def update_account_groups(self, user, account, groups, replace=False):

        """Update the groups associated with the account."""

        logger.debug("update_account_groups: %s %s %s", account, groups, replace)

        if user != account:

            raise NotAllowedError

        self._create_account(user, account)

        self._check_groups(groups)

        self._put_groups(account, groups, replace)

    @backend_method

    def put_account(self, user, account):

        """Create a new account with the given name."""

        logger.debug("put_account: %s", account)

        if user != account:

            raise NotAllowedError

        try:

            version_id, mtime = self._get_accountinfo(account)

        except NameError:

            pass

        else:

            raise NameError('Account already exists')

        self._put_version(account, user)

    @backend_method

    def delete_account(self, user, account):

        """Delete the account with the given name."""

        logger.debug("delete_account: %s", account)

        if user != account:

            raise NotAllowedError

        count = self._get_pathstats(account)[0]

        if count > 0:

            raise IndexError('Account is not empty')

        sql = 'delete from versions where name = ?'

        self.con.execute(sql, (account,))

        self._del_groups(account)

    @backend_method

    def list_containers(self, user, account, marker=None, limit=10000, shared=False, until=None):

        """Return a list of containers existing under an account."""

        logger.debug("list_containers: %s %s %s %s", account, marker, limit, until)

        if user != account:

            if until or account not in self._allowed_accounts(user):

                raise NotAllowedError

            allowed = self._allowed_containers(user, account)

            start, limit = self._list_limits(allowed, marker, limit)

            return allowed[start:start + limit]

        else:

            if shared:

                allowed = [x.split('/', 2)[1] for x in self._shared_paths(account)]

                start, limit = self._list_limits(allowed, marker, limit)

                return allowed[start:start + limit]

        return [x[0] for x in self._list_objects(account, '', '/', marker, limit, False, [], until)]

    @backend_method

    def get_container_meta(self, user, account, container, until=None):

        """Return a dictionary with the container metadata."""

        logger.debug("get_container_meta: %s %s %s", account, container, until)

        if user != account:

            if until or container not in self._allowed_containers(user, account):

                raise NotAllowedError

        path, version_id, mtime = self._get_containerinfo(account, container, until)

        count, bytes, tstamp = self._get_pathstats(path, until)

        if mtime > tstamp:

            tstamp = mtime

        if until is None:

            modified = tstamp

        else:

            modified = self._get_pathstats(path)[2] # Overall last modification

            if mtime > modified:

                modified = mtime

        if user != account:

            meta = {'name': container, 'modified': modified}

        else:

            meta = self._get_metadata(path, version_id)

            meta.update({'name': container, 'count': count, 'bytes': bytes, 'modified': modified})

            if until is not None:

                meta.update({'until_timestamp': tstamp})

        return meta

    @backend_method

    def update_container_meta(self, user, account, container, meta, replace=False):

        """Update the metadata associated with the container."""

        logger.debug("update_container_meta: %s %s %s %s", account, container, meta, replace)

        if user != account:

            raise NotAllowedError

        path, version_id, mtime = self._get_containerinfo(account, container)

        self._put_metadata(user, path, meta, replace, False)

    @backend_method

    def get_container_policy(self, user, account, container):

        """Return a dictionary with the container policy."""

        logger.debug("get_container_policy: %s %s", account, container)

        if user != account:

            if container not in self._allowed_containers(user, account):

                raise NotAllowedError

            return {}

        path = self._get_containerinfo(account, container)[0]

        return self._get_policy(path)

    @backend_method

    def update_container_policy(self, user, account, container, policy, replace=False):

        """Update the policy associated with the account."""

        logger.debug("update_container_policy: %s %s %s %s", account, container, policy, replace)

        if user != account:

            raise NotAllowedError

        path = self._get_containerinfo(account, container)[0]

        self._check_policy(policy)

        if replace:

            for k, v in self.default_policy.iteritems():

                if k not in policy:

                    policy[k] = v

        for k, v in policy.iteritems():

            sql = 'insert or replace into policy (name, key, value) values (?, ?, ?)'

            self.con.execute(sql, (path, k, v))

    @backend_method

    def put_container(self, user, account, container, policy=None):

        """Create a new container with the given name."""

        logger.debug("put_container: %s %s %s", account, container, policy)

        if user != account:

            raise NotAllowedError

        try:

            path, version_id, mtime = self._get_containerinfo(account, container)

        except NameError:

            pass

        else:

            raise NameError('Container already exists')

        if policy:

            self._check_policy(policy)

        path = '/'.join((account, container))

        version_id = self._put_version(path, user)[0]

        for k, v in self.default_policy.iteritems():

            if k not in policy:

                policy[k] = v

        for k, v in policy.iteritems():

            sql = 'insert or replace into policy (name, key, value) values (?, ?, ?)'

            self.con.execute(sql, (path, k, v))

    @backend_method

    def delete_container(self, user, account, container, until=None):

        """Delete/purge the container with the given name."""

        logger.debug("delete_container: %s %s %s", account, container, until)

        if user != account:

            raise NotAllowedError

        path, version_id, mtime = self._get_containerinfo(account, container)

        if until is not None:

            sql = '''select version_id from versions where name like ? and tstamp <= ?

                        and version_id not in (select version_id from (%s))'''

            sql = sql % self._sql_until() # Do not delete current versions.

            c = self.con.execute(sql, (path + '/%', until))

            for v in [x[0] for x in c.fetchall()]:

                self._del_version(v)

            return

        count = self._get_pathstats(path)[0]

        if count > 0:

            raise IndexError('Container is not empty')

        sql = 'delete from versions where name = ? or name like ?' # May contain hidden items.