Synnefo

    try:

        objects = request.backend.list_object_meta(

        object_permissions = {}

        object_public = {}

        if until is None:

        try:

            for x in objects:

                s, h = \

                sizes.append(s)

                hashmaps.append(h)

        except NotAllowedError:

        else:

            if TRANSLATE_UUIDS:

                try:

                except ItemNotExists:

                    faults.ItemNotFound('Invalid source account')

    return response

def object_write_form(request, v_account, v_container, v_object):

    # Normal Response Codes: 201

    # Error Response Codes: internalServerError (500),

        # Do permissions first, as it may fail easier.

        if permissions is not None:

            try:

            except NotAllowedError:

                raise faults.Forbidden('Not allowed')

            except ItemNotExists:

    try:

        size, hashmap = \

    except NotAllowedError:

        raise faults.Forbidden('Not allowed')

    except ItemNotExists:

                        hashmap[bi] = src_hashmap[sbi]

                    else:

                        data = request.backend.get_block(src_hashmap[sbi])

                else:

                    hashmap.append(src_hashmap[sbi])

                offset += bl

                   'application/octet-stream', data, meta, permissions))

            data = get_random_data(int(random.random()))

            self.utils.create_update_object(

        self.utils.merge_account('Account1', 'account1', only_stats=False,

                                 dry=False, silent=True)

astakos_services = deepcopy(vanilla_astakos_services)

fill_endpoints(astakos_services, ASTAKOS_BASE_URL)

for path, value in CUSTOMIZE_ASTAKOS_SERVICES:

    set_path(astakos_services, path, value, createpath=True)

BASE_ASTAKOS_PROXY_PATH = BASE_ASTAKOS_PROXY_PATH.strip('/')

COOKIE_NAME = getattr(settings, 'PITHOS_ASTAKOS_COOKIE_NAME', '_pithos2_a')

BACKEND_BLOCK_UMASK = getattr(settings, 'PITHOS_BACKEND_BLOCK_UMASK', 0o022)

# Queue for billing.

# Default setting for new accounts.

BACKEND_ACCOUNT_QUOTA = getattr(

BACKEND_CONTAINER_QUOTA = getattr(

    settings, 'PITHOS_BACKEND_CONTAINER_QUOTA', 0)

BACKEND_VERSIONING = getattr(settings, 'PITHOS_BACKEND_VERSIONING', 'auto')

# Enable backend pooling

BACKEND_POOL_ENABLED = getattr(settings, 'PITHOS_BACKEND_POOL_ENABLED', True)

SERVICE_TOKEN = getattr(settings, 'PITHOS_SERVICE_TOKEN', '')

RADOS_STORAGE = getattr(settings, 'PITHOS_RADOS_STORAGE', False)

RADOS_POOL_MAPS = getattr(settings, 'PITHOS_RADOS_POOL_MAPS', 'maps')

# This enables a ui compatibility layer for the introduction of UUIDs in

# the requests on its own.

PROXY_USER_SERVICES = getattr(settings, 'PITHOS_PROXY_USER_SERVICES', True)

# Set the alphabet to use for constructing the URL of Pithos public files

    settings,

    'PITHOS_PUBLIC_URL_ALPHABET',

# The maximum number or items returned by the listing api methods

API_LIST_LIMIT = getattr(settings, 'PITHOS_API_LIST_LIMIT', 10000)

class UrlEncoder(object):

        self.alphabet = alphabet

        self.block_size = block_size

        self.mask = (1 << block_size) - 1

                                 ASTAKOSCLIENT_POOLSIZE,

                                 SERVICE_TOKEN,

                                 ASTAKOS_BASE_URL,

                                 BACKEND_POOL_ENABLED, BACKEND_POOL_SIZE,

                                 BACKEND_BLOCK_SIZE, BACKEND_HASH_ALGORITHM,

                                 RADOS_STORAGE, RADOS_POOL_BLOCKS,

            int(meta['until_timestamp']))

    for k, v in policy.iteritems():

        response[smart_str(format_header_key('X-Container-Policy-' + k),

def get_object_headers(request):

def validate_modification_preconditions(request, meta):

    if 'modified' not in meta:

        return  # TODO: Always return?

    if if_none_match is not None:

        # TODO: If this passes, must ignore If-Modified-Since header.

        if etag is not None:

                # TODO: Continue if an If-Modified-Since header is present.

                if request.method in ('HEAD', 'GET'):

                    raise faults.NotModified('Resource ETag matches')

def socket_read_iterator(request, length=0, blocksize=4096):

    Read up to 'length'. If 'length' is negative, will attempt a chunked read.

    The maximum ammount of data read is controlled by MAX_UPLOAD_SIZE.

DEFAULT_CONTAINER_VERSIONING = 'auto'

class NotAllowedError(Exception):

    pass

class BaseBackend(object):

    The following variables should be available:

        'hash_algorithm': Suggested is 'sha256'

        'block_size': Suggested is 4MB

    """

    def close(self):

    def list_accounts(self, user, marker=None, limit=10000):

        """Return a list of accounts the user can access.

            'marker': Start list from the next item after 'marker'

            'limit': Number of containers to return

        """

        return []

        """Return a dictionary with the account metadata for the domain.

        The keys returned are all user-defined, except:

            'until_timestamp': Last modification until the timestamp provided

        Raises:

            NotAllowedError: Operation not permitted

        """

        Parameters:

            'domain': Metadata domain

            'meta': Dictionary with metadata to update

            'replace': Replace instead of update

        return

    def get_account_groups(self, user, account):

        Raises:

            NotAllowedError: Operation not permitted

        The keys returned are:

            'quota': The maximum bytes allowed (default is 0 - unlimited)

        Raises:

            NotAllowedError: Operation not permitted

        """

        return

        """Return a list of container names existing under an account.

            'marker': Start list from the next item after 'marker'

            'limit': Number of containers to return

        """

        return []

        Raises:

            NotAllowedError: Operation not permitted

        """

        return []

        """Return a dictionary with the container metadata for the domain.

        The keys returned are all user-defined, except:

        """

        return {}

        """Update the metadata associated with the container for the domain.

        Parameters:

            'domain': Metadata domain

            'meta': Dictionary with metadata to update

            'replace': Replace instead of update

        The keys returned are:

            'quota': The maximum bytes allowed (default is 0 - unlimited)

        Raises:

            NotAllowedError: Operation not permitted

        """

        return {}

        """Update the policy associated with the container.

        Raises:

        """

        return

        """Create a new container with the given name.

        Raises:

            NotAllowedError: Operation not permitted

        """

        return

        """Delete/purge the container with the given name.

        Raises:

            NotAllowedError: Operation not permitted

        """

        return

            'prefix': List objects starting with 'prefix'

            'marker': Start list from the next item after 'marker'

            'virtual': If not set, the result will only include names starting

                       with 'prefix' and ending without a 'delimiter' or with

                       the first occurance of the 'delimiter' after 'prefix'.

                       up to and including the 'delimiter' if it is found

            'domain': Metadata domain for keys

        """

        return []

        Same parameters with list_objects. Returned dicts have no user-defined

        metadata and, if until is not None, a None 'modified' timestamp.

        return []

    def list_object_permissions(self, user, account, container, prefix=''):

        Raises:

            NotAllowedError: Operation not permitted

        return []

    def list_object_public(self, user, account, container, prefix=''):

        return {}

        """Return a dictionary with the object metadata for the domain.

        The keys returned are all user-defined, except:

            'modified': Last modification timestamp (overall)

            'version': The version identifier

            'version_timestamp': The version's modification timestamp

            'checksum': The MD5 sum of the object (may be empty)

        """

        return {}

        Parameters:

            'domain': Metadata domain

        """

        return {}

        """Update (set) the permissions associated with the object.

        Parameters:

        """

        return 0, []

        Parameters:

            'domain': Metadata domain

        """

        return ''

        """Update an object's checksum."""

        return

        """Copy an object's data and metadata and return the new version.

        Parameters:

            'domain': Metadata domain

            'replace_meta': Replace metadata instead of update

            'src_version': Copy from the version provided

        Raises:

            NotAllowedError: Operation not permitted

        """

        return ''

        """Move an object's data and metadata and return the new version.

        Parameters:

            'domain': Metadata domain

            'replace_meta': Replace metadata instead of update

            'permissions': New object permissions

        Raises:

            NotAllowedError: Operation not permitted

        """

        return ''

        """Delete/purge an object.

        Parameters:

        Raises:

            NotAllowedError: Operation not permitted

        return

    def list_versions(self, user, account, container, name):

        Raises:

            NotAllowedError: Operation not permitted

            IndexError: Offset or data outside block limits

        """

        return 0

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

from fileblocker import FileBlocker

def intersect(a, b):

    """ return the intersection of two lists """

    return list(set(a) & set(b))

def union(a, b):

    """ return the union of two lists """

    return list(set(a) | set(b))

            if params['blockpool']:

                from radosblocker import RadosBlocker

                self.rblocker = RadosBlocker(**params)

            pass

        self.fblocker = FileBlocker(**params)

            (_, r_missing) = self.rblocker.block_stor(blocklist)

        return (hashes, union(r_missing, f_missing))

    def block_delta(self, blkhash, offset, data):

        """Construct and store a new block from a given block

           and a data 'patch' applied at offset. Return:

        r_existed = True

        (f_hash, f_existed) = self.fblocker.block_delta(blkhash, offset, data)

        if self.rblocker:

        if not r_hash and not f_hash:

            return None, None

        if self.rblocker and not r_hash:

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

from errno import ENOENT, EROFS

    try:

        seek(offset * chunksize)

        seek = None

        for x in xrange(offset):

            fwrite(zeros(chunksize))

    seek = openfile.seek

    try:

        seek(remains)

        seek = None

        while 1:

            s = fread(remains)

    def sync_write_chunks(self, chunksize, offset, chunks, size=None):

        #self.dirty = 1

    def sync_read(self, size):

        read = self.fdesc.read

# or implied, of GRNET S.A.

from os import SEEK_CUR, SEEK_SET

_zeros = ''

        radosobject.sunc_write(zeros(chunksize))

    radosobject.sync_write(buffer(zeros(chunksize), 0, r))

def file_sync_read_chunks(radosobject, chunksize, nr, offset=0):

    """Read and yield groups of chunks from a buffered file object at offset.

       Reads never span accros chunksize boundaries.

        yield chunk

        nr -= 1

class RadosObject(object):

    __slots__ = ("name", "ioctx", "create", "offset")

        datalen = 0

        while 1:

            try:

            except ObjectNotFound:

                s = None

            if not s:

    def sync_read_chunks(self, chunksize, nr, offset=0):

        return file_sync_read_chunks(self, chunksize, nr, offset)

# Copyright 2011-2012 GRNET S.A. All rights reserved.

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

            if not exists(blockpath):

                makedirs(blockpath)

            else:

        hashtype = params['hashtype']

        try:

                if not rbl:

                    break

                for block in rbl.sync_read_chunks(blocksize, 1, 0):

            if not block:

                break

            append(self._pad(block))

        """

        block_hash = self.block_hash

        hashlist = [block_hash(b) for b in blocklist]

        for i in missing:

            with self._get_rear_block(hashlist[i], 1) as rbl:

        return hashlist, missing

        block = self.block_retr((blkhash,))

        if not block:

            return None, None

        block = block[0]

        newblock = block[:offset] + data

        if len(newblock) > blocksize:

            sextend(sl)

            lastsize = len(block)

        return size, hashlist, storedlist

# Copyright 2011-2012 GRNET S.A. All rights reserved.

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

from os.path import isdir, realpath, exists, join

from binascii import hexlify

    """Mapper.

       Required constructor parameters: mappath, namelen.

    """

    mappath = None

    namelen = None

            if not exists(mappath):

                makedirs(mappath)

            else:

        self.mappath = mappath

    def _get_rear_map(self, maphash, create=0):

            return

        with self._get_rear_map(maphash, 1) as rmap:

            rmap.sync_write_chunks(namelen, blkoff, hashes, None)

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

from filemapper import FileMapper

class Mapper(object):

    """Mapper.

       Required constructor parameters: mappath, namelen.

            if params['mappool']:

                from radosmapper import RadosMapper

                self.rmap = RadosMapper(**params)

            pass

        self.fmap = FileMapper(**params)

from context_object import RadosObject, file_sync_read_chunks

class RadosBlocker(object):

    """Blocker.

                if not rbl:

                    break

                for block in rbl.sync_read_chunks(blocksize, 1, 0):

            if not block:

                break

            append(self._pad(block))

        """

        block_hash = self.block_hash

        hashlist = [block_hash(b) for b in blocklist]

        for i in missing:

            with self._get_rear_block(hashlist[i], 1) as rbl:

        return hashlist, missing

            sextend(sl)

            lastsize = len(block)

        return size, hashlist, storedlist

from binascii import hexlify

from rados import *

class RadosMapper(object):

    """Mapper.

            return

        with self._get_rear_map(maphash, 1) as rmap:

            rmap.sync_write_chunks(namelen, blkoff, hashes, None)

# or implied, of GRNET S.A.

from dbwrapper import DBWrapper

from permissions import Permissions, READ, WRITE

from config import Config

from quotaholder_serials import QuotaholderSerial

__all__ = ["DBWrapper",

down_revision = '3dd56e750a3'

from alembic import op

from pithos.api.settings import (SERVICE_TOKEN, ASTAKOS_BASE_URL)

import sqlalchemy as sa

catalog = {}

def get_uuid(account):

    global catalog

    if account in catalog:

        return catalog[account]

inverse_catalog = {}

def get_displayname(account):

    global inverse_catalog

    if account in inverse_catalog:

    column('path', sa.String(2048))

)

    'xfeaturevals',

    column('feature_id', sa.Integer),

    column('key', sa.Integer),

    column('value', sa.String(256))

)

    'groups',

    column('owner', sa.String(256)),

    column('name', sa.String(256)),

    column('member', sa.String(256))

)

def migrate(callback):

    connection = op.get_bind()

            bar.next()

            continue

        path = sep.join([match, rest])

        connection.execute(u)

        bar.next()

    bar.finish()

    s = sa.select([v.c.muser]).distinct()

    musers = connection.execute(s).fetchall()

    bar = IncrementalBar('Migrating version modification users...',

    for muser, in musers:

        match = callback(muser)

        if not match:

            bar.next()

            continue

        path = sep.join([match, rest])

        connection.execute(u)

        bar.next()

    bar.finish()

            bar.next()

            continue

        path = sep.join([match, rest])

        connection.execute(u)

        bar.next()

    bar.finish()

            continue

        new_value = sep.join([match, group])

        u = xvals.update()

        connection.execute(u)

        bar.next()

    bar.finish()

            bar.next()

    bar.finish()

def upgrade():

    migrate(get_uuid)

def downgrade():

    migrate(get_displayname)

from pithos.backends.modular import ULTIMATE_ANSWER

from pithos.api.short_url import encode_url

def upgrade():

    op.add_column('public', sa.Column('url', sa.String(2048)))

    op.create_unique_constraint('idx_public_url', 'public', ['url'])

    s = sa.select([p.c.public_id])

    rows = conn.execute(s).fetchall()

    for r in rows:

        op.execute(s)

def downgrade():

    op.drop_constraint('idx_public_url', 'public')

    op.drop_column('public', 'url')

    u = st.update().where(st.c.population < 0).values({'population': 0})

    op.execute(u)

def downgrade():

    pass

from dbworker import DBWorker

def create_tables(engine):

    metadata = MetaData()

    columns = []

    columns.append(Column('key', String(256), primary_key=True))

    columns.append(Column('value', String(256)))

    metadata.create_all(engine)

    return metadata.sorted_tables

class Config(DBWorker):

    """Config are properties holding persistent information about system state.

    """

        if row:

            return row[0]

        return None

    def set_value(self, key, value):

        """Set a configuration entry.

        """

        r = self.conn.execute(s, key=key, value=value)

        inserted_primary_key = r.inserted_primary_key[0]

        r.close()

ESCAPE_CHAR = '@'

class DBWorker(object):

    """Database connection handler."""

        self.engine = wrapper.engine

    def escape_like(self, s, escape_char=ESCAPE_CHAR):

        else:

            #self.engine = create_engine(db, pool_size=0, max_overflow=-1)

            self.engine = create_engine(

        self.engine.echo = False

        self.engine.echo_pool = False

        self.conn = self.engine.connect()

    def group_check(self, owner, group, member):

        """Check if a member is in a group."""

        r = self.conn.execute(s)

        l = r.fetchone()

        r.close()

from sqlalchemy import (Table, Integer, BigInteger, DECIMAL, Boolean,

                        Column, String, MetaData, ForeignKey)

from sqlalchemy.exc import NoSuchTableError

from dbworker import DBWorker, ESCAPE_CHAR

                          primary_key=True))

    columns.append(Column('key', String(128), primary_key=True))

    columns.append(Column('value', String(256)))

    #create statistics table

    columns = []

    columns.append(Column('mtime', DECIMAL(precision=16, scale=6)))

    columns.append(Column('cluster', Integer, nullable=False, default=0,

                          primary_key=True, autoincrement=False))

    #create versions table

    columns = []

    def node_get_versions(self, node, keys=(), propnames=_propnames):

        """Return the properties of all versions at node.

           If keys is empty, return all properties in the order

        """

        s = select([self.versions.c.serial,

        if not keys:

            return rows

    def node_count_children(self, node):

        """Return node's child count."""

        s = select([self.nodes.c.node],

                   and_(self.nodes.c.parent == parent,

                        select([func.count(self.versions.c.serial)],

        rp = self.conn.execute(s)

        rp.close()

        if nodes:

            s = self.nodes.delete().where(self.nodes.c.node.in_(nodes))

        s = select([self.nodes.c.node],

                   and_(self.nodes.c.node == node,

                        select([func.count(self.versions.c.serial)],

        rp.close()

        if nodes:

            s = self.nodes.delete().where(self.nodes.c.node.in_(nodes))

        #insert or replace

        #TODO better upsert

        u = u.values(population=population, size=size, mtime=mtime)

        rp = self.conn.execute(u)

        rp.close()

        """

        mtime = time()

        serial = self.conn.execute(s).inserted_primary_key[0]

        self.statistics_update_ancestors(node, 1, size, mtime, cluster,

                                         update_statistics_ancestors_depth)

            return props

        return None

        """Lookup the current versions of the given nodes.

           Return a list with their properties:

        """

        if not nodes:

            return ()

        """Return a sequence of values for the properties of

           the version specified by serial and the keys, in the order given.

           If keys is empty, return all properties in the order

        """

        v = self.versions.alias()

        return hash, size

    def attribute_get(self, serial, domain, keys=()):

        """

        if keys:

    def attribute_unset_is_latest(self, node, exclude):

        u = self.attributes.update().where(and_(

            self.attributes.c.node == node,

        self.conn.execute(u)