Synnefo » ./kamaki

# Copyright 2011-2013 GRNET S.A. All rights reserved.

#

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

#

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

#

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

#

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.command

from time import localtime, strftime

from os import path, makedirs, walk

from io import StringIO

from pydoc import pager

from kamaki.cli import command

from kamaki.cli.command_tree import CommandTree

from kamaki.cli.errors import (

    raiseCLIError, CLISyntaxError, CLIBaseUrlError, CLIInvalidArgument)

from kamaki.cli.utils import (

    format_size, to_bytes, bold, get_path_size, guess_mime_type)

from kamaki.cli.argument import FlagArgument, ValueArgument, IntArgument

from kamaki.cli.argument import KeyValueArgument, DateArgument

from kamaki.cli.argument import ProgressBarArgument

from kamaki.cli.commands import _command_init, errors

from kamaki.cli.commands import addLogSettings, DontRaiseKeyError

from kamaki.cli.commands import (

    _optional_output_cmd, _optional_json, _name_filter)

from kamaki.clients.pithos import PithosClient, ClientError

from kamaki.clients.astakos import AstakosClient

pithos_cmds = CommandTree('file', 'Pithos+/Storage API commands')

_commands = [pithos_cmds]

# Argument functionality

class SharingArgument(ValueArgument):

    """Set sharing (read and/or write) groups

    .

    :value type: "read=term1,term2,... write=term1,term2,..."

    .

    :value returns: {'read':['term1', 'term2', ...],

    .   'write':['term1', 'term2', ...]}

    """

    @property

    def value(self):

        return getattr(self, '_value', self.default)

    @value.setter

    def value(self, newvalue):

        perms = {}

        try:

            permlist = newvalue.split(' ')

        except AttributeError:

            return

        for p in permlist:

            try:

                (key, val) = p.split('=')

            except ValueError as err:

                raiseCLIError(

                    err,

                    'Error in --sharing',

                    details='Incorrect format',

                    importance=1)

            if key.lower() not in ('read', 'write'):

                msg = 'Error in --sharing'

                raiseCLIError(err, msg, importance=1, details=[

                    'Invalid permission key %s' % key])

            val_list = val.split(',')

            if not key in perms:

                perms[key] = []

            for item in val_list:

                if item not in perms[key]:

                    perms[key].append(item)

        self._value = perms

class RangeArgument(ValueArgument):

    """

    :value type: string of the form <start>-<end> where <start> and <end> are

        integers

    :value returns: the input string, after type checking <start> and <end>

    """

    @property

    def value(self):

        return getattr(self, '_value', self.default)

    @value.setter

    def value(self, newvalues):

        if not newvalues:

            self._value = self.default

            return

        self._value = ''

        for newvalue in newvalues.split(','):

            self._value = ('%s,' % self._value) if self._value else ''

            start, sep, end = newvalue.partition('-')

            if sep:

                if start:

                    start, end = (int(start), int(end))

                    assert start <= end, 'Invalid range value %s' % newvalue

                    self._value += '%s-%s' % (int(start), int(end))

                else:

                    self._value += '-%s' % int(end)

            else:

                self._value += '%s' % int(start)

# Command specs

class _pithos_init(_command_init):

    """Initialize a pithos+ kamaki client"""

    @staticmethod

    def _is_dir(remote_dict):

        return 'application/directory' == remote_dict.get(

            'content_type', remote_dict.get('content-type', ''))

    @DontRaiseKeyError

    def _custom_container(self):

        return self.config.get_cloud(self.cloud, 'pithos_container')

    @DontRaiseKeyError

    def _custom_uuid(self):

        return self.config.get_cloud(self.cloud, 'pithos_uuid')

    def _set_account(self):

        self.account = self._custom_uuid()

        if self.account:

            return

        if getattr(self, 'auth_base', False):

            self.account = self.auth_base.user_term('id', self.token)

        else:

            astakos_url = self._custom_url('astakos')

            astakos_token = self._custom_token('astakos') or self.token

            if not astakos_url:

                raise CLIBaseUrlError(service='astakos')

            astakos = AstakosClient(astakos_url, astakos_token)

            self.account = astakos.user_term('id')

    @errors.generic.all

    @addLogSettings

    def _run(self):

        self.base_url = None

        if getattr(self, 'cloud', None):

            self.base_url = self._custom_url('pithos')

        else:

            self.cloud = 'default'

        self.token = self._custom_token('pithos')

        self.container = self._custom_container()

        if getattr(self, 'auth_base', False):

            self.token = self.token or self.auth_base.token

            if not self.base_url:

                pithos_endpoints = self.auth_base.get_service_endpoints(

                    self._custom_type('pithos') or 'object-store',

                    self._custom_version('pithos') or '')

                self.base_url = pithos_endpoints['publicURL']

        elif not self.base_url:

            raise CLIBaseUrlError(service='pithos')

        self._set_account()

        self.client = PithosClient(

            base_url=self.base_url,

            token=self.token,

            account=self.account,

            container=self.container)

    def main(self):

        self._run()

class _file_account_command(_pithos_init):

    """Base class for account level storage commands"""

    def __init__(self, arguments={}, auth_base=None, cloud=None):

        super(_file_account_command, self).__init__(

            arguments, auth_base, cloud)

        self['account'] = ValueArgument(

            'Set user account (not permanent)', ('-A', '--account'))

    def _run(self, custom_account=None):

        super(_file_account_command, self)._run()

        if custom_account:

            self.client.account = custom_account

        elif self['account']:

            self.client.account = self['account']

    @errors.generic.all

    def main(self):

        self._run()

class _file_container_command(_file_account_command):

    """Base class for container level storage commands"""

    container = None

    path = None

    def __init__(self, arguments={}, auth_base=None, cloud=None):

        super(_file_container_command, self).__init__(

            arguments, auth_base, cloud)

        self['container'] = ValueArgument(

            'Set container to work with (temporary)', ('-C', '--container'))

    def extract_container_and_path(

            self, container_with_path, path_is_optional=True):

        """Contains all heuristics for deciding what should be used as

        container or path. Options are:

        * user string of the form container:path

        * self.container, self.path variables set by super constructor, or

        explicitly by the caller application

        Error handling is explicit as these error cases happen only here

        """

        try:

            assert isinstance(container_with_path, str)

        except AssertionError as err:

            if self['container'] and path_is_optional:

                self.container = self['container']

                self.client.container = self['container']

                return

            raiseCLIError(err)

        user_cont, sep, userpath = container_with_path.partition(':')

        if sep:

            if not user_cont:

                raiseCLIError(CLISyntaxError(

                    'Container is missing\n',

                    details=errors.pithos.container_howto))

            alt_cont = self['container']

            if alt_cont and user_cont != alt_cont:

                raiseCLIError(CLISyntaxError(

                    'Conflict: 2 containers (%s, %s)' % (user_cont, alt_cont),

                    details=errors.pithos.container_howto)

                )

            self.container = user_cont

            if not userpath:

                raiseCLIError(CLISyntaxError(

                    'Path is missing for object in container %s' % user_cont,

                    details=errors.pithos.container_howto)

                )

            self.path = userpath

        else:

            alt_cont = self['container'] or self.client.container

            if alt_cont:

                self.container = alt_cont

                self.path = user_cont

            elif path_is_optional:

                self.container = user_cont

                self.path = None

            else:

                self.container = user_cont

                raiseCLIError(CLISyntaxError(

                    'Both container and path are required',

                    details=errors.pithos.container_howto)

                )

    @errors.generic.all

    def _run(self, container_with_path=None, path_is_optional=True):

        super(_file_container_command, self)._run()

        if self['container']:

            self.client.container = self['container']

            if container_with_path:

                self.path = container_with_path

            elif not path_is_optional:

                raise CLISyntaxError(

                    'Both container and path are required',

                    details=errors.pithos.container_howto)

        elif container_with_path:

            self.extract_container_and_path(

                container_with_path,

                path_is_optional)

            self.client.container = self.container

        self.container = self.client.container

    def main(self, container_with_path=None, path_is_optional=True):

        self._run(container_with_path, path_is_optional)

@command(pithos_cmds)

class file_list(_file_container_command, _optional_json, _name_filter):

    """List containers, object trees or objects in a directory

    Use with:

    1 no parameters : containers in current account

    2. one parameter (container) or --container : contents of container

    3. <container>:<prefix> or --container=<container> <prefix>: objects in

    .   container starting with prefix

    """

    arguments = dict(

        detail=FlagArgument('detailed output', ('-l', '--list')),

        limit=IntArgument('limit number of listed items', ('-n', '--number')),

        marker=ValueArgument('output greater that marker', '--marker'),

        delimiter=ValueArgument('show output up to delimiter', '--delimiter'),

        path=ValueArgument(

            'show output starting with prefix up to /', '--path'),

        meta=ValueArgument(

            'show output with specified meta keys', '--meta',

            default=[]),

        if_modified_since=ValueArgument(

            'show output modified since then', '--if-modified-since'),

        if_unmodified_since=ValueArgument(

            'show output not modified since then', '--if-unmodified-since'),

        until=DateArgument('show metadata until then', '--until'),

        format=ValueArgument(

            'format to parse until data (default: d/m/Y H:M:S )', '--format'),

        shared=FlagArgument('show only shared', '--shared'),

        more=FlagArgument('read long results', '--more'),

        exact_match=FlagArgument(

            'Show only objects that match exactly with path',

            '--exact-match'),

        enum=FlagArgument('Enumerate results', '--enumerate'),

        recursive=FlagArgument(

            'Recursively list containers and their contents',

            ('-R', '--recursive'))

    )

    def print_objects(self, object_list):

        for index, obj in enumerate(object_list):

            if self['exact_match'] and self.path and not (

                    obj['name'] == self.path or 'content_type' in obj):

                continue

            pretty_obj = obj.copy()

            index += 1

            empty_space = ' ' * (len(str(len(object_list))) - len(str(index)))

            if 'subdir' in obj:

                continue

            if obj['content_type'] == 'application/directory':

                isDir = True

                size = 'D'

            else:

                isDir = False

                size = format_size(obj['bytes'])

                pretty_obj['bytes'] = '%s (%s)' % (obj['bytes'], size)

            oname = obj['name'] if self['more'] else bold(obj['name'])

            prfx = (

                '%s%s. ' % (empty_space, index)) if self['enum'] else ''

            if self['detail']:

                self.writeln('%s%s' % (prfx, oname))

                self.print_dict(pretty_obj, exclude=('name'))

                self.writeln()

            else:

                oname = '%s%9s %s' % (prfx, size, oname)

                oname += '/' if isDir else u''

                self.writeln(oname)

    def print_containers(self, container_list):

        for index, container in enumerate(container_list):

            if 'bytes' in container:

                size = format_size(container['bytes'])

            prfx = ('%s. ' % (index + 1)) if self['enum'] else ''

            _cname = container['name'] if (

                self['more']) else bold(container['name'])

            cname = u'%s%s' % (prfx, _cname)

            if self['detail']:

                self.writeln(cname)

                pretty_c = container.copy()

                if 'bytes' in container:

                    pretty_c['bytes'] = '%s (%s)' % (container['bytes'], size)

                self.print_dict(pretty_c, exclude=('name'))

                self.writeln()

            else:

                if 'count' in container and 'bytes' in container:

                    self.writeln('%s (%s, %s objects)' % (

                        cname, size, container['count']))

                else:

                    self.writeln(cname)

            objects = container.get('objects', [])

            if objects:

                self.print_objects(objects)

                self.writeln('')

    def _argument_context_check(self):

        container_level_only = ('recursive', )

        object_level_only = ('delimiter', 'path', 'exact_match')

        details, mistake = [], ''

        if self.container:

            for term in container_level_only:

                if self[term]:

                    details = [

                        'This is a container-level argument',

                        'Use it without a <container> parameter']

                    mistake = self.arguments[term]

        else:

            for term in object_level_only:

                if not self['recursive'] and self[term]:

                    details = [

                        'This is an opbject-level argument',

                        'Use it with a <container> parameter',

                        'or with the -R/--recursive argument']

                    mistake = self.arguments[term]

        if mistake and details:

            raise CLIInvalidArgument(

                'Invalid use of %s argument' % '/'.join(mistake.parsed_name),

                details=details + ['Try --help for more details'])

    def _create_object_forest(self, container_list):

        try:

            for container in container_list:

                self.client.container = container['name']

                objects = self.client.container_get(

                    limit=False if self['more'] else self['limit'],

                    marker=self['marker'],

                    delimiter=self['delimiter'],

                    path=self['path'],

                    if_modified_since=self['if_modified_since'],

                    if_unmodified_since=self['if_unmodified_since'],

                    until=self['until'],

                    meta=self['meta'],

                    show_only_shared=self['shared'])

                container['objects'] = objects.json

        finally:

            self.client.container = None

    @errors.generic.all

    @errors.pithos.connection

    @errors.pithos.object_path

    @errors.pithos.container

    def _run(self):

        files, prnt = None, None

        self._argument_context_check()

        if not self.container:

            r = self.client.account_get(

                limit=False if self['more'] else self['limit'],

                marker=self['marker'],

                if_modified_since=self['if_modified_since'],

                if_unmodified_since=self['if_unmodified_since'],

                until=self['until'],

                show_only_shared=self['shared'])

            files, prnt = self._filter_by_name(r.json), self.print_containers

            if self['recursive']:

                self._create_object_forest(files)

        else:

            prefix = (self.path and not self['name']) or self['name_pref']

            r = self.client.container_get(

                limit=False if self['more'] else self['limit'],

                marker=self['marker'],

                prefix=prefix,

                delimiter=self['delimiter'],

                path=self['path'],

                if_modified_since=self['if_modified_since'],

                if_unmodified_since=self['if_unmodified_since'],

                until=self['until'],

                meta=self['meta'],

                show_only_shared=self['shared'])

            files, prnt = self._filter_by_name(r.json), self.print_objects

        if self['more']:

            outbu, self._out = self._out, StringIO()

        try:

            if self['json_output'] or self['output_format']:

                self._print(files)

            else:

                prnt(files)

        finally:

            if self['more']:

                pager(self._out.getvalue())

                self._out = outbu

    def main(self, container____path__=None):

        super(self.__class__, self)._run(container____path__)

        self._run()

@command(pithos_cmds)

class file_mkdir(_file_container_command, _optional_output_cmd):

    """Create a directory

    Kamaki hanldes directories the same way as OOS Storage and Pithos+:

    A directory  is   an  object  with  type  "application/directory"

    An object with path  dir/name can exist even if  dir does not exist

    or even if dir  is  a non  directory  object.  Users can modify dir '

    without affecting the dir/name object in any way.

    """

    @errors.generic.all

    @errors.pithos.connection

    @errors.pithos.container

    def _run(self):

        self._optional_output(self.client.create_directory(self.path))

    def main(self, container___directory):

        super(self.__class__, self)._run(

            container___directory, path_is_optional=False)

        self._run()

@command(pithos_cmds)

class file_touch(_file_container_command, _optional_output_cmd):

    """Create an empty object (file)

    If object exists, this command will reset it to 0 length

    """

    arguments = dict(

        content_type=ValueArgument(

            'Set content type (default: application/octet-stream)',

            '--content-type',

            default='application/octet-stream')

    )

    @errors.generic.all

    @errors.pithos.connection

    @errors.pithos.container

    def _run(self):

        self._optional_output(

            self.client.create_object(self.path, self['content_type']))

    def main(self, container___path):

        super(file_touch, self)._run(container___path, path_is_optional=False)

        self._run()

@command(pithos_cmds)

class file_create(_file_container_command, _optional_output_cmd):

    """Create a container"""

    arguments = dict(

        versioning=ValueArgument(

            'set container versioning (auto/none)', '--versioning'),

        limit=IntArgument('set default container limit', '--limit'),

        meta=KeyValueArgument(

            'set container metadata (can be repeated)', '--meta')

    )

    @errors.generic.all

    @errors.pithos.connection

    @errors.pithos.container

    def _run(self, container):

        try:

            self._optional_output(self.client.create_container(

                container=container,

                sizelimit=self['limit'],

                versioning=self['versioning'],

                metadata=self['meta'],

                success=(201, )))

        except ClientError as ce:

            if ce.status in (202, ):

                raiseCLIError(ce, 'Container %s alread exists' % container)

    def main(self, container=None):

        super(self.__class__, self)._run(container)

        if container and self.container != container:

            raiseCLIError('Invalid container name %s' % container, details=[

                'Did you mean "%s" ?' % self.container,

                'Use --container for names containing :'])

        self._run(container)

class _source_destination_command(_file_container_command):

    arguments = dict(

        destination_account=ValueArgument('', ('-a', '--dst-account')),

        recursive=FlagArgument('', ('-R', '--recursive')),

        prefix=FlagArgument('', '--with-prefix', default=''),

        suffix=ValueArgument('', '--with-suffix', default=''),

        add_prefix=ValueArgument('', '--add-prefix', default=''),

        add_suffix=ValueArgument('', '--add-suffix', default=''),

        prefix_replace=ValueArgument('', '--prefix-to-replace', default=''),

        suffix_replace=ValueArgument('', '--suffix-to-replace', default=''),

    )

    def __init__(self, arguments={}, auth_base=None, cloud=None):

        self.arguments.update(arguments)

        super(_source_destination_command, self).__init__(

            self.arguments, auth_base, cloud)

    def _run(self, source_container___path, path_is_optional=False):

        super(_source_destination_command, self)._run(

            source_container___path, path_is_optional)

        self.dst_client = PithosClient(

            base_url=self.client.base_url,

            token=self.client.token,

            account=self['destination_account'] or self.client.account)

    @errors.generic.all

    @errors.pithos.account

    def _dest_container_path(self, dest_container_path):

        if self['destination_container']:

            self.dst_client.container = self['destination_container']

            return (self['destination_container'], dest_container_path)

        if dest_container_path:

            dst = dest_container_path.split(':')

            if len(dst) > 1:

                try:

                    self.dst_client.container = dst[0]

                    self.dst_client.get_container_info(dst[0])

                except ClientError as err:

                    if err.status in (404, 204):

                        raiseCLIError(

                            'Destination container %s not found' % dst[0])

                    raise

                else:

                    self.dst_client.container = dst[0]

                return (dst[0], dst[1])

            return(None, dst[0])

        raiseCLIError('No destination container:path provided')

    def _get_all(self, prefix):

        return self.client.container_get(prefix=prefix).json

    def _get_src_objects(self, src_path, source_version=None):

        """Get a list of the source objects to be called

        :param src_path: (str) source path

        :returns: (method, params) a method that returns a list when called

        or (object) if it is a single object

        """

        if src_path and src_path[-1] == '/':

            src_path = src_path[:-1]

        if self['prefix']:

            return (self._get_all, dict(prefix=src_path))

        try:

            srcobj = self.client.get_object_info(

                src_path, version=source_version)

        except ClientError as srcerr:

            if srcerr.status == 404:

                raiseCLIError(

                    'Source object %s not in source container %s' % (

                        src_path, self.client.container),

                    details=['Hint: --with-prefix to match multiple objects'])

            elif srcerr.status not in (204,):

                raise

            return (self.client.list_objects, {})

        if self._is_dir(srcobj):

            if not self['recursive']:

                raiseCLIError(

                    'Object %s of cont. %s is a dir' % (

                        src_path, self.client.container),

                    details=['Use --recursive to access directories'])

            return (self._get_all, dict(prefix=src_path))

        srcobj['name'] = src_path

        return srcobj

    def src_dst_pairs(self, dst_path, source_version=None):

        src_iter = self._get_src_objects(self.path, source_version)

        src_N = isinstance(src_iter, tuple)

        add_prefix = self['add_prefix'].strip('/')

        if dst_path and dst_path.endswith('/'):

            dst_path = dst_path[:-1]

        try:

            dstobj = self.dst_client.get_object_info(dst_path)

        except ClientError as trgerr:

            if trgerr.status in (404,):

                if src_N:

                    raiseCLIError(

                        'Cannot merge multiple paths to path %s' % dst_path,

                        details=[

                            'Try to use / or a directory as destination',

                            'or create the destination dir (/file mkdir)',

                            'or use a single object as source'])

            elif trgerr.status not in (204,):

                raise

        else:

            if self._is_dir(dstobj):

                add_prefix = '%s/%s' % (dst_path.strip('/'), add_prefix)

            elif src_N:

                raiseCLIError(

                    'Cannot merge multiple paths to path' % dst_path,

                    details=[

                        'Try to use / or a directory as destination',

                        'or create the destination dir (/file mkdir)',

                        'or use a single object as source'])

        if src_N:

            (method, kwargs) = src_iter

            for obj in method(**kwargs):

                name = obj['name']

                if name.endswith(self['suffix']):

                    yield (name, self._get_new_object(name, add_prefix))

        elif src_iter['name'].endswith(self['suffix']):

            name = src_iter['name']

            yield (name, self._get_new_object(dst_path or name, add_prefix))

        else:

            raiseCLIError('Source path %s conflicts with suffix %s' % (

                src_iter['name'], self['suffix']))

    def _get_new_object(self, obj, add_prefix):

        if self['prefix_replace'] and obj.startswith(self['prefix_replace']):

            obj = obj[len(self['prefix_replace']):]