Synnefo » ./kamaki

# Copyright 2011-2012 GRNET S.A. All rights reserved.

#

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

#

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

#

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

#

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.command

from sys import stdout

from time import localtime, strftime

from logging import getLogger

from os import path, makedirs

from kamaki.cli import command

from kamaki.cli.command_tree import CommandTree

from kamaki.cli.errors import raiseCLIError, CLISyntaxError

from kamaki.cli.utils import (

    format_size,

    to_bytes,

    print_dict,

    print_items,

    pretty_keys,

    page_hold,

    bold,

    ask_user)

from kamaki.cli.argument import FlagArgument, ValueArgument, IntArgument

from kamaki.cli.argument import KeyValueArgument, DateArgument

from kamaki.cli.argument import ProgressBarArgument

from kamaki.cli.commands import _command_init, errors

from kamaki.clients.pithos import PithosClient, ClientError

from kamaki.clients.astakos import AstakosClient

kloger = getLogger('kamaki')

pithos_cmds = CommandTree('store', 'Pithos+ storage commands')

_commands = [pithos_cmds]

about_directories = [

    'Kamaki hanldes directories the same way as OOS Storage and Pithos+:',

    'A directory is an object with type "application/directory"',

    'An object with path dir/name can exist even if dir does not exist or',

    'even if dir is a non directory object. Users can modify dir without',

    'affecting the dir/name object in any way.']

# Argument functionality

def raise_connection_errors(e):

    if e.status in range(200) + [403, 401]:

        raiseCLIError(e, details=[

            'Please check the service url and the authentication information',

            ' ',

            '  to get the service url: /config get store.url',

            '  to set the service url: /config set store.url <url>',

            ' ',

            '  to get authentication token: /config get token',

            '  to set authentication token: /config set token <token>'])

    elif e.status == 413:

        raiseCLIError(e, details=[

            'Get quotas:',

            '- total quota:      /store quota',

            '- container quota:  /store quota <container>',

            'Users shall set a higher container quota, if available:',

            '-                  /store setquota <quota>[unit] <container>'])

class DelimiterArgument(ValueArgument):

    """

    :value type: string

    :value returns: given string or /

    """

    def __init__(self, caller_obj, help='', parsed_name=None, default=None):

        super(DelimiterArgument, self).__init__(help, parsed_name, default)

        self.caller_obj = caller_obj

    @property

    def value(self):

        if self.caller_obj['recursive']:

            return '/'

        return getattr(self, '_value', self.default)

    @value.setter

    def value(self, newvalue):

        self._value = newvalue

class SharingArgument(ValueArgument):

    """Set sharing (read and/or write) groups

    .

    :value type: "read=term1,term2,... write=term1,term2,..."

    .

    :value returns: {'read':['term1', 'term2', ...],

    .   'write':['term1', 'term2', ...]}

    """

    @property

    def value(self):

        return getattr(self, '_value', self.default)

    @value.setter

    def value(self, newvalue):

        perms = {}

        try:

            permlist = newvalue.split(' ')

        except AttributeError:

            return

        for p in permlist:

            try:

                (key, val) = p.split('=')

            except ValueError as err:

                raiseCLIError(

                    err,

                    'Error in --sharing',

                    details='Incorrect format',

                    importance=1)

            if key.lower() not in ('read', 'write'):

                msg = 'Error in --sharing'

                raiseCLIError(err, msg, importance=1, details=[

                    'Invalid permission key %s' % key])

            val_list = val.split(',')

            if not key in perms:

                perms[key] = []

            for item in val_list:

                if item not in perms[key]:

                    perms[key].append(item)

        self._value = perms

class RangeArgument(ValueArgument):

    """

    :value type: string of the form <start>-<end> where <start> and <end> are

        integers

    :value returns: the input string, after type checking <start> and <end>

    """

    @property

    def value(self):

        return getattr(self, '_value', self.default)

    @value.setter

    def value(self, newvalue):

        if newvalue is None:

            self._value = self.default

            return

        (start, end) = newvalue.split('-')

        (start, end) = (int(start), int(end))

        self._value = '%s-%s' % (start, end)

# Command specs

class _pithos_init(_command_init):

    """Initialize a pithos+ kamaki client"""

    @errors.generic.all

    def _run(self):

        self.token = self.config.get('store', 'token')\

            or self.config.get('global', 'token')

        self.base_url = self.config.get('store', 'url')\

            or self.config.get('global', 'url')

        self._set_account()

        self.container = self.config.get('store', 'container')\

            or self.config.get('global', 'container')

        self.client = PithosClient(

            base_url=self.base_url,

            token=self.token,

            account=self.account,

            container=self.container)

    def main(self):

        self._run()

    def _set_account(self):

        astakos = AstakosClient(self.config.get('astakos', 'url'), self.token)

        self.account = self['account'] or astakos.term('uuid')

        """Backwards compatibility"""

        self.account = self.account\

            or self.config.get('store', 'account')\

            or self.config.get('global', 'account')

class _store_account_command(_pithos_init):

    """Base class for account level storage commands"""

    def __init__(self, arguments={}):

        super(_store_account_command, self).__init__(arguments)

        self['account'] = ValueArgument(

            'Set user account (not permanent)',

            '--account')

    def _run(self):

        super(_store_account_command, self)._run()

        if self['account']:

            self.client.account = self['account']

    @errors.generic.all

    def main(self):

        self._run()

class _store_container_command(_store_account_command):

    """Base class for container level storage commands"""

    container = None

    path = None

    def __init__(self, arguments={}):

        super(_store_container_command, self).__init__(arguments)

        self['container'] = ValueArgument(

            'Set container to work with (temporary)',

            '--container')

    @errors.generic.all

    def _dest_container_path(self, dest_container_path):

        if self['destination_container']:

            return (self['destination_container'], dest_container_path)

        dst = dest_container_path.split(':')

        return (dst[0], dst[1]) if len(dst) > 1 else (None, dst[0])

    def extract_container_and_path(

            self,

            container_with_path,

            path_is_optional=True):

        """Contains all heuristics for deciding what should be used as

        container or path. Options are:

        * user string of the form container:path

        * self.container, self.path variables set by super constructor, or

        explicitly by the caller application

        Error handling is explicit as these error cases happen only here

        """

        try:

            assert isinstance(container_with_path, str)

        except AssertionError as err:

            if self['container'] and path_is_optional:

                self.container = self['container']

                self.client.container = self['container']

                return

            raiseCLIError(err)

        user_cont, sep, userpath = container_with_path.partition(':')

        if sep:

            if not user_cont:

                raiseCLIError(CLISyntaxError(

                    'Container is missing\n',

                    details=errors.pithos.container_howto))

            alt_cont = self['container']

            if alt_cont and user_cont != alt_cont:

                raiseCLIError(CLISyntaxError(

                    'Conflict: 2 containers (%s, %s)' % (user_cont, alt_cont),

                    details=errors.pithos.container_howto)

                )

            self.container = user_cont

            if not userpath:

                raiseCLIError(CLISyntaxError(

                    'Path is missing for object in container %s' % user_cont,

                    details=errors.pithos.container_howto)

                )

            self.path = userpath

        else:

            alt_cont = self['container'] or self.client.container

            if alt_cont:

                self.container = alt_cont

                self.path = user_cont

            elif path_is_optional:

                self.container = user_cont

                self.path = None

            else:

                self.container = user_cont

                raiseCLIError(CLISyntaxError(

                    'Both container and path are required',

                    details=errors.pithos.container_howto)

                )

    @errors.generic.all

    def _run(self, container_with_path=None, path_is_optional=True):

        super(_store_container_command, self)._run()

        if self['container']:

            self.client.container = self['container']

            if container_with_path:

                self.path = container_with_path

            elif not path_is_optional:

                raise CLISyntaxError(

                    'Both container and path are required',

                    details=errors.pithos.container_howto)

        elif container_with_path:

            self.extract_container_and_path(

                container_with_path,

                path_is_optional)

            self.client.container = self.container

        self.container = self.client.container

    def main(self, container_with_path=None, path_is_optional=True):

        self._run(container_with_path, path_is_optional)

@command(pithos_cmds)

class store_list(_store_container_command):

    """List containers, object trees or objects in a directory

    Use with:

    1 no parameters : containers in current account

    2. one parameter (container) or --container : contents of container

    3. <container>:<prefix> or --container=<container> <prefix>: objects in

    .   container starting with prefix

    """

    arguments = dict(

        detail=FlagArgument('show detailed output', '-l'),

        limit=IntArgument('limit the number of listed items', '-n'),

        marker=ValueArgument('show output greater that marker', '--marker'),

        prefix=ValueArgument('show output starting with prefix', '--prefix'),

        delimiter=ValueArgument('show output up to delimiter', '--delimiter'),

        path=ValueArgument(

            'show output starting with prefix up to /',

            '--path'),

        meta=ValueArgument(

            'show output with specified meta keys',

            '--meta',

            default=[]),

        if_modified_since=ValueArgument(

            'show output modified since then',

            '--if-modified-since'),

        if_unmodified_since=ValueArgument(

            'show output not modified since then',

            '--if-unmodified-since'),

        until=DateArgument('show metadata until then', '--until'),

        format=ValueArgument(

            'format to parse until data (default: d/m/Y H:M:S )',

            '--format'),

        shared=FlagArgument('show only shared', '--shared'),

        public=FlagArgument('show only public', '--public'),

        more=FlagArgument(

            'output results in pages (-n to set items per page, default 10)',

            '--more'),

        exact_match=FlagArgument(

            'Show only objects that match exactly with path',

            '--exact-match')

    )

    def print_objects(self, object_list):

        limit = int(self['limit']) if self['limit'] > 0 else len(object_list)

        for index, obj in enumerate(object_list):

            if self['exact_match'] and self.path and not (

                    obj['name'] == self.path or 'content_type' in obj):

                continue

            pretty_obj = obj.copy()

            index += 1

            empty_space = ' ' * (len(str(len(object_list))) - len(str(index)))

            if obj['content_type'] == 'application/directory':

                isDir = True

                size = 'D'

            else:

                isDir = False

                size = format_size(obj['bytes'])

                pretty_obj['bytes'] = '%s (%s)' % (obj['bytes'], size)

            oname = bold(obj['name'])

            if self['detail']:

                print('%s%s. %s' % (empty_space, index, oname))

                print_dict(pretty_keys(pretty_obj), exclude=('name'))

                print

            else:

                oname = '%s%s. %6s %s' % (empty_space, index, size, oname)

                oname += '/' if isDir else ''

                print(oname)

            if self['more']:

                page_hold(index, limit, len(object_list))

    def print_containers(self, container_list):

        limit = int(self['limit']) if self['limit'] > 0\

            else len(container_list)

        for index, container in enumerate(container_list):

            if 'bytes' in container:

                size = format_size(container['bytes'])

            cname = '%s. %s' % (index + 1, bold(container['name']))

            if self['detail']:

                print(cname)

                pretty_c = container.copy()

                if 'bytes' in container:

                    pretty_c['bytes'] = '%s (%s)' % (container['bytes'], size)

                print_dict(pretty_keys(pretty_c), exclude=('name'))

                print

            else:

                if 'count' in container and 'bytes' in container:

                    print('%s (%s, %s objects)' % (

                        cname,

                        size,

                        container['count']))

                else:

                    print(cname)

            if self['more']:

                page_hold(index + 1, limit, len(container_list))

    @errors.generic.all

    @errors.pithos.connection

    @errors.pithos.object_path

    @errors.pithos.container

    def _run(self):

        if self.container is None:

            r = self.client.account_get(

                limit=False if self['more'] else self['limit'],

                marker=self['marker'],

                if_modified_since=self['if_modified_since'],

                if_unmodified_since=self['if_unmodified_since'],

                until=self['until'],

                show_only_shared=self['shared'])

            self.print_containers(r.json)

        else:

            prefix = self.path or self['prefix']

            r = self.client.container_get(

                limit=False if self['more'] else self['limit'],

                marker=self['marker'],

                prefix=prefix,

                delimiter=self['delimiter'],

                path=self['path'],

                if_modified_since=self['if_modified_since'],

                if_unmodified_since=self['if_unmodified_since'],

                until=self['until'],

                meta=self['meta'],

                show_only_shared=self['shared'])

            self.print_objects(r.json)

    def main(self, container____path__=None):

        super(self.__class__, self)._run(container____path__)

        self._run()

@command(pithos_cmds)

class store_mkdir(_store_container_command):

    """Create a directory"""

    __doc__ += '\n. '.join(about_directories)

    @errors.generic.all

    @errors.pithos.connection

    @errors.pithos.container

    def _run(self):

        self.client.create_directory(self.path)

    def main(self, container___directory):

        super(self.__class__, self)._run(

            container___directory,

            path_is_optional=False)

        self._run()

@command(pithos_cmds)

class store_touch(_store_container_command):

    """Create an empty object (file)

    If object exists, this command will reset it to 0 length

    """

    arguments = dict(

        content_type=ValueArgument(

            'Set content type (default: application/octet-stream)',

            '--content-type',

            default='application/octet-stream')

    )

    @errors.generic.all

    @errors.pithos.connection

    @errors.pithos.container

    def _run(self):

        self.client.create_object(self.path, self['content_type'])

    def main(self, container___path):

        super(store_touch, self)._run(

            container___path,

            path_is_optional=False)

        self._run()

@command(pithos_cmds)

class store_create(_store_container_command):

    """Create a container"""

    arguments = dict(

        versioning=ValueArgument(

            'set container versioning (auto/none)',

            '--versioning'),

        quota=IntArgument('set default container quota', '--quota'),

        meta=KeyValueArgument(

            'set container metadata (can be repeated)',

            '--meta')

    )

    @errors.generic.all

    @errors.pithos.connection

    @errors.pithos.container

    def _run(self):

        self.client.container_put(

            quota=self['quota'],

            versioning=self['versioning'],

            metadata=self['meta'])

    def main(self, container=None):

        super(self.__class__, self)._run(container)

        if container and self.container != container:

            raiseCLIError('Invalid container name %s' % container, details=[

                'Did you mean "%s" ?' % self.container,

                'Use --container for names containing :'])

        self._run()

@command(pithos_cmds)

class store_copy(_store_container_command):

    """Copy objects from container to (another) container

    Semantics:

    copy cont:path path2

    .   will copy all <obj> prefixed with path, as path2<obj>

    .   or as path2 if path corresponds to just one whole object

    copy cont:path cont2:

    .   will copy all <obj> prefixed with path to container cont2

    copy cont:path [cont2:]path2 --exact-match

    .   will copy at most one <obj> as a new object named path2,

    .   provided path corresponds to a whole object path

    copy cont:path [cont2:]path2 --replace

    .   will copy all <obj> prefixed with path, replacing path with path2

    where <obj> is a full file or directory object path.

    Use options:

    1. <container1>:<path1> [container2:]<path2> : if container2 is not given,

    destination is container1:path2

    2. <container>:<path1> <path2> : make a copy in the same container

    3. Can use --container= instead of <container1>

    """

    arguments = dict(

        destination_container=ValueArgument(

            'use it if destination container name contains a : character',

            '--dst-container'),

        source_version=ValueArgument(

            'copy specific version',

            '--source-version'),

        public=ValueArgument('make object publicly accessible', '--public'),

        content_type=ValueArgument(

            'change object\'s content type',

            '--content-type'),

        recursive=FlagArgument(

            'mass copy with delimiter /',

            ('-r', '--recursive')),

        exact_match=FlagArgument(

            'Copy only the object that fully matches path',

            '--exact-match'),

        replace=FlagArgument('Replace src. path with dst. path', '--replace')

    )

    def _objlist(self, dst_path):

        if self['exact_match']:

            return [(dst_path or self.path, self.path)]

        r = self.client.container_get(prefix=self.path)

        if len(r.json) == 1:

            obj = r.json[0]

            return [(obj['name'], dst_path or obj['name'])]

        start = len(self.path) if self['replace'] else 0

        return [(obj['name'], '%s%s' % (

            dst_path,

            obj['name'][start:])) for obj in r.json]

    @errors.generic.all

    @errors.pithos.connection

    @errors.pithos.container

    def _run(self, dst_cont, dst_path):

        no_source_object = True

        for src_object, dst_object in self._objlist(dst_path):

            no_source_object = False

            self.client.copy_object(

                src_container=self.container,

                src_object=src_object,

                dst_container=dst_cont or self.container,

                dst_object=dst_object,

                source_version=self['source_version'],

                public=self['public'],

                content_type=self['content_type'])

        if no_source_object:

            raiseCLIError('No object %s in container %s' % (

                self.path,

                self.container))

    def main(

            self,

            source_container___path,

            destination_container___path=None):

        super(self.__class__, self)._run(

            source_container___path,

            path_is_optional=False)

        (dst_cont, dst_path) = self._dest_container_path(

            destination_container___path)

        self._run(dst_cont=dst_cont, dst_path=dst_path or '')

@command(pithos_cmds)

class store_move(_store_container_command):

    """Move/rename objects

    Semantics:

    move cont:path path2

    .   will move all <obj> prefixed with path, as path2<obj>

    .   or as path2 if path corresponds to just one whole object

    move cont:path cont2:

    .   will move all <obj> prefixed with path to container cont2

    move cont:path [cont2:]path2 --exact-match

    .   will move at most one <obj> as a new object named path2,

    .   provided path corresponds to a whole object path

    move cont:path [cont2:]path2 --replace

    .   will move all <obj> prefixed with path, replacing path with path2

    where <obj> is a full file or directory object path.

    Use options:

    1. <container1>:<path1> [container2:]<path2> : if container2 not given,

    destination is container1:path2

    2. <container>:<path1> path2 : rename

    3. Can use --container= instead of <container1>

    """

    arguments = dict(

        destination_container=ValueArgument(

            'use it if destination container name contains a : character',

            '--dst-container'),

        source_version=ValueArgument('specify version', '--source-version'),

        public=FlagArgument('make object publicly accessible', '--public'),

        content_type=ValueArgument('modify content type', '--content-type'),

        recursive=FlagArgument('up to delimiter /', ('-r', '--recursive')),

        exact_match=FlagArgument(

            'Copy only the object that fully matches path',

            '--exact-match'),

        replace=FlagArgument('Replace src. path with dst. path', '--replace')

    )

    def _objlist(self, dst_path):

        if self['exact_match']:

            return [(dst_path or self.path, self.path)]

        r = self.client.container_get(prefix=self.path)

        if len(r.json) == 1:

            obj = r.json[0]

            return [(obj['name'], dst_path or obj['name'])]

        return [(

            obj['name'],

            '%s%s' % (

                dst_path,

                obj['name'][len(self.path) if self['replace'] else 0:]

            )) for obj in r.json]

    @errors.generic.all

    @errors.pithos.connection

    @errors.pithos.container

    def _run(self, dst_cont, dst_path):

        no_source_object = True

        for src_object, dst_object in self._objlist(dst_path):

            no_source_object = False

            self.client.move_object(

                src_container=self.container,

                src_object=src_object,

                dst_container=dst_cont or self.container,

                dst_object=dst_object,

                source_version=self['source_version'],

                public=self['public'],

                content_type=self['content_type'])

        if no_source_object:

            raiseCLIError('No object %s in container %s' % (

                self.path,

                self.container))

    def main(

            self,

            source_container___path,

            destination_container___path=None):

        super(self.__class__, self)._run(

            source_container___path,

            path_is_optional=False)

        (dst_cont, dst_path) = self._dest_container_path(

            destination_container___path)

        self._run(dst_cont=dst_cont, dst_path=dst_path or '')

@command(pithos_cmds)

class store_append(_store_container_command):

    """Append local file to (existing) remote object

    The remote object should exist.

    If the remote object is a directory, it is transformed into a file.

    In the later case, objects under the directory remain intact.

    """

    arguments = dict(

        progress_bar=ProgressBarArgument(

            'do not show progress bar',

            '--no-progress-bar',

            default=False)

    )

    @errors.generic.all

    @errors.pithos.connection

    @errors.pithos.container

    @errors.pithos.object_path

    def _run(self, local_path):

        (progress_bar, upload_cb) = self._safe_progress_bar('Appending')

        try:

            f = open(local_path, 'rb')

            self.client.append_object(self.path, f, upload_cb)