Synnefo » ./kamaki

# Copyright 2011-2012 GRNET S.A. All rights reserved.

#

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

#

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

#

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

#

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.command

from sys import stdout

from time import localtime, strftime

from os import path, makedirs, walk

from kamaki.cli import command

from kamaki.cli.command_tree import CommandTree

from kamaki.cli.errors import raiseCLIError, CLISyntaxError, CLIBaseUrlError

from kamaki.cli.utils import (

    format_size, to_bytes, print_dict, print_items, pretty_keys, pretty_dict,

    page_hold, bold, ask_user, get_path_size, print_json)

from kamaki.cli.argument import FlagArgument, ValueArgument, IntArgument

from kamaki.cli.argument import KeyValueArgument, DateArgument

from kamaki.cli.argument import ProgressBarArgument

from kamaki.cli.commands import _command_init, errors

from kamaki.cli.commands import _optional_output_cmd, _optional_json

from kamaki.clients.pithos import PithosClient, ClientError

from kamaki.clients.astakos import AstakosClient

pithos_cmds = CommandTree('file', 'Pithos+/Storage API commands')

_commands = [pithos_cmds]

# Argument functionality

class DelimiterArgument(ValueArgument):

    """

    :value type: string

    :value returns: given string or /

    """

    def __init__(self, caller_obj, help='', parsed_name=None, default=None):

        super(DelimiterArgument, self).__init__(help, parsed_name, default)

        self.caller_obj = caller_obj

    @property

    def value(self):

        if self.caller_obj['recursive']:

            return '/'

        return getattr(self, '_value', self.default)

    @value.setter

    def value(self, newvalue):

        self._value = newvalue

class SharingArgument(ValueArgument):

    """Set sharing (read and/or write) groups

    .

    :value type: "read=term1,term2,... write=term1,term2,..."

    .

    :value returns: {'read':['term1', 'term2', ...],

    .   'write':['term1', 'term2', ...]}

    """

    @property

    def value(self):

        return getattr(self, '_value', self.default)

    @value.setter

    def value(self, newvalue):

        perms = {}

        try:

            permlist = newvalue.split(' ')

        except AttributeError:

            return

        for p in permlist:

            try:

                (key, val) = p.split('=')

            except ValueError as err:

                raiseCLIError(

                    err,

                    'Error in --sharing',

                    details='Incorrect format',

                    importance=1)

            if key.lower() not in ('read', 'write'):

                msg = 'Error in --sharing'

                raiseCLIError(err, msg, importance=1, details=[

                    'Invalid permission key %s' % key])

            val_list = val.split(',')

            if not key in perms:

                perms[key] = []

            for item in val_list:

                if item not in perms[key]:

                    perms[key].append(item)

        self._value = perms

class RangeArgument(ValueArgument):

    """

    :value type: string of the form <start>-<end> where <start> and <end> are

        integers

    :value returns: the input string, after type checking <start> and <end>

    """

    @property

    def value(self):

        return getattr(self, '_value', self.default)

    @value.setter

    def value(self, newvalue):

        if newvalue is None:

            self._value = self.default

            return

        (start, end) = newvalue.split('-')

        (start, end) = (int(start), int(end))

        self._value = '%s-%s' % (start, end)

# Command specs

class _pithos_init(_command_init):

    """Initialize a pithos+ kamaki client"""

    @staticmethod

    def _is_dir(remote_dict):

        return 'application/directory' == remote_dict.get(

            'content_type', remote_dict.get('content-type', ''))

    @errors.generic.all

    def _run(self):

        self.token = self.config.get('file', 'token')\

            or self.config.get('global', 'token')

        if getattr(self, 'auth_base', False):

            pithos_endpoints = self.auth_base.get_service_endpoints(

                self.config.get('pithos', 'type'),

                self.config.get('pithos', 'version'))

            self.base_url = pithos_endpoints['publicURL']

        else:

            self.base_url = self.config.get('file', 'url')\

                or self.config.get('store', 'url')\

                or self.config.get('pithos', 'url')

        if not self.base_url:

            raise CLIBaseUrlError(service='pithos')

        self._set_account()

        self.container = self.config.get('file', 'container')\

            or self.config.get('store', 'container')\

            or self.config.get('pithos', 'container')\

            or self.config.get('global', 'container')

        self.client = PithosClient(

            base_url=self.base_url,

            token=self.token,

            account=self.account,

            container=self.container)

        self._set_log_params()

        self._update_max_threads()

    def main(self):

        self._run()

    def _set_account(self):

        if getattr(self, 'auth_base', False):

            self.account = self.auth_base.user_term('id', self.token)

        else:

            astakos_url = self.config.get('user', 'url')\

                or self.config.get('astakos', 'url')

            if not astakos_url:

                raise CLIBaseUrlError(service='astakos')

            astakos = AstakosClient(astakos_url, self.token)

            self.account = astakos.user_term('id')

class _file_account_command(_pithos_init):

    """Base class for account level storage commands"""

    def __init__(self, arguments={}, auth_base=None):

        super(_file_account_command, self).__init__(arguments, auth_base)

        self['account'] = ValueArgument(

            'Set user account (not permanent)', ('-A', '--account'))

    def _run(self, custom_account=None):

        super(_file_account_command, self)._run()

        if custom_account:

            self.client.account = custom_account

        elif self['account']:

            self.client.account = self['account']

    @errors.generic.all

    def main(self):

        self._run()

class _file_container_command(_file_account_command):

    """Base class for container level storage commands"""

    container = None

    path = None

    def __init__(self, arguments={}, auth_base=None):

        super(_file_container_command, self).__init__(arguments, auth_base)

        self['container'] = ValueArgument(

            'Set container to work with (temporary)', ('-C', '--container'))

    def extract_container_and_path(

            self,

            container_with_path,

            path_is_optional=True):

        """Contains all heuristics for deciding what should be used as

        container or path. Options are:

        * user string of the form container:path

        * self.container, self.path variables set by super constructor, or

        explicitly by the caller application

        Error handling is explicit as these error cases happen only here

        """

        try:

            assert isinstance(container_with_path, str)

        except AssertionError as err:

            if self['container'] and path_is_optional:

                self.container = self['container']

                self.client.container = self['container']

                return

            raiseCLIError(err)

        user_cont, sep, userpath = container_with_path.partition(':')

        if sep:

            if not user_cont:

                raiseCLIError(CLISyntaxError(

                    'Container is missing\n',

                    details=errors.pithos.container_howto))

            alt_cont = self['container']

            if alt_cont and user_cont != alt_cont:

                raiseCLIError(CLISyntaxError(

                    'Conflict: 2 containers (%s, %s)' % (user_cont, alt_cont),

                    details=errors.pithos.container_howto)

                )

            self.container = user_cont

            if not userpath:

                raiseCLIError(CLISyntaxError(

                    'Path is missing for object in container %s' % user_cont,

                    details=errors.pithos.container_howto)

                )

            self.path = userpath

        else:

            alt_cont = self['container'] or self.client.container

            if alt_cont:

                self.container = alt_cont

                self.path = user_cont

            elif path_is_optional:

                self.container = user_cont

                self.path = None

            else:

                self.container = user_cont

                raiseCLIError(CLISyntaxError(

                    'Both container and path are required',

                    details=errors.pithos.container_howto)

                )

    @errors.generic.all

    def _run(self, container_with_path=None, path_is_optional=True):

        super(_file_container_command, self)._run()

        if self['container']:

            self.client.container = self['container']

            if container_with_path:

                self.path = container_with_path

            elif not path_is_optional:

                raise CLISyntaxError(

                    'Both container and path are required',

                    details=errors.pithos.container_howto)

        elif container_with_path:

            self.extract_container_and_path(

                container_with_path,

                path_is_optional)

            self.client.container = self.container

        self.container = self.client.container

    def main(self, container_with_path=None, path_is_optional=True):

        self._run(container_with_path, path_is_optional)

@command(pithos_cmds)

class file_list(_file_container_command, _optional_json):

    """List containers, object trees or objects in a directory

    Use with:

    1 no parameters : containers in current account

    2. one parameter (container) or --container : contents of container

    3. <container>:<prefix> or --container=<container> <prefix>: objects in

    .   container starting with prefix

    """

    arguments = dict(

        detail=FlagArgument('detailed output', ('-l', '--list')),

        limit=IntArgument('limit number of listed items', ('-n', '--number')),

        marker=ValueArgument('output greater that marker', '--marker'),

        prefix=ValueArgument('output starting with prefix', '--prefix'),

        delimiter=ValueArgument('show output up to delimiter', '--delimiter'),

        path=ValueArgument(

            'show output starting with prefix up to /', '--path'),

        meta=ValueArgument(

            'show output with specified meta keys', '--meta',

            default=[]),

        if_modified_since=ValueArgument(

            'show output modified since then', '--if-modified-since'),

        if_unmodified_since=ValueArgument(

            'show output not modified since then', '--if-unmodified-since'),

        until=DateArgument('show metadata until then', '--until'),

        format=ValueArgument(

            'format to parse until data (default: d/m/Y H:M:S )', '--format'),

        shared=FlagArgument('show only shared', '--shared'),

        more=FlagArgument(

            'output results in pages (-n to set items per page, default 10)',

            '--more'),

        exact_match=FlagArgument(

            'Show only objects that match exactly with path',

            '--exact-match'),

        enum=FlagArgument('Enumerate results', '--enumerate')

    )

    def print_objects(self, object_list):

        if self['json_output']:

            print_json(object_list)

            return

        limit = int(self['limit']) if self['limit'] > 0 else len(object_list)

        for index, obj in enumerate(object_list):

            if self['exact_match'] and self.path and not (

                    obj['name'] == self.path or 'content_type' in obj):

                continue

            pretty_obj = obj.copy()

            index += 1

            empty_space = ' ' * (len(str(len(object_list))) - len(str(index)))

            if obj['content_type'] == 'application/directory':

                isDir = True

                size = 'D'

            else:

                isDir = False

                size = format_size(obj['bytes'])

                pretty_obj['bytes'] = '%s (%s)' % (obj['bytes'], size)

            oname = bold(obj['name'])

            prfx = ('%s%s. ' % (empty_space, index)) if self['enum'] else ''

            if self['detail']:

                print('%s%s' % (prfx, oname))

                print_dict(pretty_keys(pretty_obj), exclude=('name'))

                print

            else:

                oname = '%s%9s %s' % (prfx, size, oname)

                oname += '/' if isDir else ''

                print(oname)

            if self['more']:

                page_hold(index, limit, len(object_list))

    def print_containers(self, container_list):

        if self['json_output']:

            print_json(container_list)

            return

        limit = int(self['limit']) if self['limit'] > 0\

            else len(container_list)

        for index, container in enumerate(container_list):

            if 'bytes' in container:

                size = format_size(container['bytes'])

            prfx = ('%s. ' % (index + 1)) if self['enum'] else ''

            cname = '%s%s' % (prfx, bold(container['name']))

            if self['detail']:

                print(cname)

                pretty_c = container.copy()

                if 'bytes' in container:

                    pretty_c['bytes'] = '%s (%s)' % (container['bytes'], size)

                print_dict(pretty_keys(pretty_c), exclude=('name'))

                print

            else:

                if 'count' in container and 'bytes' in container:

                    print('%s (%s, %s objects)' % (

                        cname,

                        size,

                        container['count']))

                else:

                    print(cname)

            if self['more']:

                page_hold(index + 1, limit, len(container_list))

    @errors.generic.all

    @errors.pithos.connection

    @errors.pithos.object_path

    @errors.pithos.container

    def _run(self):

        if self.container is None:

            r = self.client.account_get(

                limit=False if self['more'] else self['limit'],

                marker=self['marker'],

                if_modified_since=self['if_modified_since'],

                if_unmodified_since=self['if_unmodified_since'],

                until=self['until'],

                show_only_shared=self['shared'])

            self._print(r.json, self.print_containers)

        else:

            prefix = self.path or self['prefix']

            r = self.client.container_get(

                limit=False if self['more'] else self['limit'],

                marker=self['marker'],

                prefix=prefix,

                delimiter=self['delimiter'],

                path=self['path'],

                if_modified_since=self['if_modified_since'],

                if_unmodified_since=self['if_unmodified_since'],

                until=self['until'],

                meta=self['meta'],

                show_only_shared=self['shared'])

            self._print(r.json, self.print_objects)

    def main(self, container____path__=None):

        super(self.__class__, self)._run(container____path__)

        self._run()

@command(pithos_cmds)

class file_mkdir(_file_container_command, _optional_output_cmd):

    """Create a directory"""

    __doc__ += '\n. '.join([

        'Kamaki hanldes directories the same way as OOS Storage and Pithos+:',

        'A   directory  is   an  object  with  type  "application/directory"',

        'An object with path  dir/name can exist even if  dir does not exist',

        'or even if dir  is  a non  directory  object.  Users can modify dir',

        'without affecting the dir/name object in any way.'])

    @errors.generic.all

    @errors.pithos.connection

    @errors.pithos.container

    def _run(self):

        self._optional_output(self.client.create_directory(self.path))

    def main(self, container___directory):

        super(self.__class__, self)._run(

            container___directory,

            path_is_optional=False)

        self._run()

@command(pithos_cmds)

class file_touch(_file_container_command, _optional_output_cmd):

    """Create an empty object (file)

    If object exists, this command will reset it to 0 length

    """

    arguments = dict(

        content_type=ValueArgument(

            'Set content type (default: application/octet-stream)',

            '--content-type',

            default='application/octet-stream')

    )

    @errors.generic.all

    @errors.pithos.connection

    @errors.pithos.container

    def _run(self):

        self._optional_output(

            self.client.create_object(self.path, self['content_type']))

    def main(self, container___path):

        super(file_touch, self)._run(

            container___path,

            path_is_optional=False)

        self._run()

@command(pithos_cmds)

class file_create(_file_container_command, _optional_output_cmd):

    """Create a container"""

    arguments = dict(

        versioning=ValueArgument(

            'set container versioning (auto/none)', '--versioning'),

        limit=IntArgument('set default container limit', '--limit'),

        meta=KeyValueArgument(

            'set container metadata (can be repeated)', '--meta')

    )

    @errors.generic.all

    @errors.pithos.connection

    @errors.pithos.container

    def _run(self, container):

        self._optional_output(self.client.create_container(

            container=container,

            sizelimit=self['limit'],

            versioning=self['versioning'],

            metadata=self['meta']))

    def main(self, container=None):

        super(self.__class__, self)._run(container)

        if container and self.container != container:

            raiseCLIError('Invalid container name %s' % container, details=[

                'Did you mean "%s" ?' % self.container,

                'Use --container for names containing :'])

        self._run(container)

class _source_destination_command(_file_container_command):

    arguments = dict(

        destination_account=ValueArgument('', ('a', '--dst-account')),

        recursive=FlagArgument('', ('-R', '--recursive')),

        prefix=FlagArgument('', '--with-prefix', default=''),

        suffix=ValueArgument('', '--with-suffix', default=''),

        add_prefix=ValueArgument('', '--add-prefix', default=''),

        add_suffix=ValueArgument('', '--add-suffix', default=''),

        prefix_replace=ValueArgument('', '--prefix-to-replace', default=''),

        suffix_replace=ValueArgument('', '--suffix-to-replace', default=''),

    )

    def __init__(self, arguments={}, auth_base=None):

        self.arguments.update(arguments)

        super(_source_destination_command, self).__init__(

            self.arguments, auth_base)

    def _run(self, source_container___path, path_is_optional=False):

        super(_source_destination_command, self)._run(

            source_container___path,

            path_is_optional)

        self.dst_client = PithosClient(

            base_url=self.client.base_url,

            token=self.client.token,

            account=self['destination_account'] or self.client.account)

    @errors.generic.all

    @errors.pithos.account

    def _dest_container_path(self, dest_container_path):

        if self['destination_container']:

            self.dst_client.container = self['destination_container']

            return (self['destination_container'], dest_container_path)

        if dest_container_path:

            dst = dest_container_path.split(':')

            if len(dst) > 1:

                try:

                    self.dst_client.container = dst[0]

                    self.dst_client.get_container_info(dst[0])

                except ClientError as err:

                    if err.status in (404, 204):

                        raiseCLIError(

                            'Destination container %s not found' % dst[0])

                    raise

                else:

                    self.dst_client.container = dst[0]

                return (dst[0], dst[1])

            return(None, dst[0])

        raiseCLIError('No destination container:path provided')

    def _get_all(self, prefix):

        return self.client.container_get(prefix=prefix).json

    def _get_src_objects(self, src_path, source_version=None):

        """Get a list of the source objects to be called

        :param src_path: (str) source path

        :returns: (method, params) a method that returns a list when called

        or (object) if it is a single object

        """

        if src_path and src_path[-1] == '/':

            src_path = src_path[:-1]

        if self['prefix']:

            return (self._get_all, dict(prefix=src_path))

        try:

            srcobj = self.client.get_object_info(

                src_path, version=source_version)

        except ClientError as srcerr:

            if srcerr.status == 404:

                raiseCLIError(

                    'Source object %s not in source container %s' % (

                        src_path, self.client.container),

                    details=['Hint: --with-prefix to match multiple objects'])

            elif srcerr.status not in (204,):

                raise

            return (self.client.list_objects, {})

        if self._is_dir(srcobj):

            if not self['recursive']:

                raiseCLIError(

                    'Object %s of cont. %s is a dir' % (

                        src_path, self.client.container),

                    details=['Use --recursive to access directories'])

            return (self._get_all, dict(prefix=src_path))

        srcobj['name'] = src_path

        return srcobj

    def src_dst_pairs(self, dst_path, source_version=None):

        src_iter = self._get_src_objects(self.path, source_version)

        src_N = isinstance(src_iter, tuple)

        add_prefix = self['add_prefix'].strip('/')

        if dst_path and dst_path.endswith('/'):

            dst_path = dst_path[:-1]

        try:

            dstobj = self.dst_client.get_object_info(dst_path)

        except ClientError as trgerr:

            if trgerr.status in (404,):

                if src_N:

                    raiseCLIError(

                        'Cannot merge multiple paths to path %s' % dst_path,

                        details=[

                            'Try to use / or a directory as destination',

                            'or create the destination dir (/file mkdir)',

                            'or use a single object as source'])

            elif trgerr.status not in (204,):

                raise

        else:

            if self._is_dir(dstobj):

                add_prefix = '%s/%s' % (dst_path.strip('/'), add_prefix)

            elif src_N:

                raiseCLIError(

                    'Cannot merge multiple paths to path' % dst_path,

                    details=[

                        'Try to use / or a directory as destination',

                        'or create the destination dir (/file mkdir)',

                        'or use a single object as source'])

        if src_N:

            (method, kwargs) = src_iter

            for obj in method(**kwargs):

                name = obj['name']

                if name.endswith(self['suffix']):

                    yield (name, self._get_new_object(name, add_prefix))

        elif src_iter['name'].endswith(self['suffix']):

            name = src_iter['name']

            yield (name, self._get_new_object(dst_path or name, add_prefix))

        else:

            raiseCLIError('Source path %s conflicts with suffix %s' % (

                src_iter['name'], self['suffix']))

    def _get_new_object(self, obj, add_prefix):

        if self['prefix_replace'] and obj.startswith(self['prefix_replace']):

            obj = obj[len(self['prefix_replace']):]

        if self['suffix_replace'] and obj.endswith(self['suffix_replace']):

            obj = obj[:-len(self['suffix_replace'])]

        return add_prefix + obj + self['add_suffix']

@command(pithos_cmds)

class file_copy(_source_destination_command, _optional_output_cmd):

    """Copy objects from container to (another) container

    Semantics:

    copy cont:path dir

    .   transfer path as dir/path

    copy cont:path cont2:

    .   trasnfer all <obj> prefixed with path to container cont2

    copy cont:path [cont2:]path2

    .   transfer path to path2

    Use options:

    1. <container1>:<path1> [container2:]<path2> : if container2 is not given,

    destination is container1:path2

    2. <container>:<path1> <path2> : make a copy in the same container

    3. Can use --container= instead of <container1>

    """

    arguments = dict(

        destination_account=ValueArgument(

            'Account to copy to', ('-a', '--dst-account')),

        destination_container=ValueArgument(

            'use it if destination container name contains a : character',

            ('-D', '--dst-container')),

        public=ValueArgument('make object publicly accessible', '--public'),

        content_type=ValueArgument(

            'change object\'s content type', '--content-type'),

        recursive=FlagArgument(

            'copy directory and contents', ('-R', '--recursive')),

        prefix=FlagArgument(

            'Match objects prefixed with src path (feels like src_path*)',

            '--with-prefix',

            default=''),

        suffix=ValueArgument(

            'Suffix of source objects (feels like *suffix)', '--with-suffix',

            default=''),

        add_prefix=ValueArgument('Prefix targets', '--add-prefix', default=''),

        add_suffix=ValueArgument('Suffix targets', '--add-suffix', default=''),

        prefix_replace=ValueArgument(

            'Prefix of src to replace with dst path + add_prefix, if matched',

            '--prefix-to-replace',

            default=''),

        suffix_replace=ValueArgument(

            'Suffix of src to replace with add_suffix, if matched',

            '--suffix-to-replace',