Synnefo » ./kamaki

# Copyright 2011-2012 GRNET S.A. All rights reserved.

#

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

#

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

#

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

#

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.command

from kamaki.cli import command

from kamaki.cli.command_tree import CommandTree

from kamaki.cli.errors import raiseCLIError, CLISyntaxError

from kamaki.cli.utils import (

    format_size,

    print_dict,

    pretty_keys,

    page_hold,

    ask_user)

from kamaki.cli.argument import FlagArgument, ValueArgument, IntArgument

from kamaki.cli.argument import KeyValueArgument, DateArgument

from kamaki.cli.argument import ProgressBarArgument

from kamaki.cli.commands import _command_init

from kamaki.clients.pithos import PithosClient, ClientError

from kamaki.cli.utils import bold

from sys import stdout

from time import localtime, strftime

from logging import getLogger

from os import path

kloger = getLogger('kamaki')

pithos_cmds = CommandTree('store', 'Pithos+ storage commands')

_commands = [pithos_cmds]

about_directories = [

    'Kamaki hanldes directories the same way as OOS Storage and Pithos+:',

    'A directory is an object with type "application/directory"',

    'An object with path dir/name can exist even if dir does not exist or',

    'even if dir is a non directory object. Users can modify dir without',

    'affecting the dir/name object in any way.']

# Argument functionality

def raise_connection_errors(e):

    if e.status in range(200) + [403, 401]:

        raiseCLIError(e, details=[

            'Please check the service url and the authentication information',

            ' ',

            '  to get the service url: /config get store.url',

            '  to set the service url: /config set store.url <url>',

            ' ',

            '  to get user the account: /config get store.account',

            '           or              /config get account',

            '  to set the user account: /config set store.account <account>',

            ' ',

            '  to get authentication token: /config get token',

            '  to set authentication token: /config set token <token>'

            ])

    elif e.status == 413:

        raiseCLIError(e, details=[

            'Get quotas:',

            '- total quota:      /store quota',

            '- container quota:  /store quota <container>',

            'Users shall set a higher container quota, if available:',

            '-                  /store setquota <limit in KB> <container>'

            ])

def check_range(start, end):

    """

    :param start: (int)

    :param end: (int)

    :returns: (int(start), int(end))

    :raises CLIError - Invalid start/end value in range

    :raises CLIError - Invalid range

    """

    try:

        start = int(start)

    except ValueError as e:

        raiseCLIError(e, 'Invalid start value %s in range' % start)

    try:

        end = int(end)

    except ValueError as e:

        raiseCLIError(e, 'Invalid end value %s in range' % end)

    if start > end:

        raiseCLIError('Invalid range %s-%s' % (start, end))

    return (start, end)

class DelimiterArgument(ValueArgument):

    """

    :value type: string

    :value returns: given string or /

    """

    def __init__(self, caller_obj, help='', parsed_name=None, default=None):

        super(DelimiterArgument, self).__init__(help, parsed_name, default)

        self.caller_obj = caller_obj

    @property

    def value(self):

        if self.caller_obj['recursive']:

            return '/'

        return getattr(self, '_value', self.default)

    @value.setter

    def value(self, newvalue):

        self._value = newvalue

class SharingArgument(ValueArgument):

    """Set sharing (read and/or write) groups

    .

    :value type: "read=term1,term2,... write=term1,term2,..."

    .

    :value returns: {'read':['term1', 'term2', ...],

    .   'write':['term1', 'term2', ...]}

    """

    @property

    def value(self):

        return getattr(self, '_value', self.default)

    @value.setter

    def value(self, newvalue):

        perms = {}

        try:

            permlist = newvalue.split(' ')

        except AttributeError:

            return

        for p in permlist:

            try:

                (key, val) = p.split('=')

            except ValueError as err:

                raiseCLIError(err, 'Error in --sharing',

                    details='Incorrect format',

                    importance=1)

            if key.lower() not in ('read', 'write'):

                raiseCLIError(err, 'Error in --sharing',

                    details='Invalid permission key %s' % key,

                    importance=1)

            val_list = val.split(',')

            if not key in perms:

                perms[key] = []

            for item in val_list:

                if item not in perms[key]:

                    perms[key].append(item)

        self._value = perms

class RangeArgument(ValueArgument):

    """

    :value type: string of the form <start>-<end> where <start> and <end> are

        integers

    :value returns: the input string, after type checking <start> and <end>

    """

    @property

    def value(self):

        return getattr(self, '_value', self.default)

    @value.setter

    def value(self, newvalue):

        if newvalue is None:

            self._value = self.default

            return

        (start, end) = newvalue.split('-')

        (start, end) = (int(start), int(end))

        self._value = '%s-%s' % (start, end)

# Command specs

class _pithos_init(_command_init):

    """Initialize a pithos+ kamaki client"""

    def main(self):

        self.token = self.config.get('store', 'token')\

            or self.config.get('global', 'token')

        self.base_url = self.config.get('store', 'url')\

            or self.config.get('global', 'url')

        self.account = self.config.get('store', 'account')\

            or self.config.get('global', 'account')

        self.container = self.config.get('store', 'container')\

            or self.config.get('global', 'container')

        self.client = PithosClient(base_url=self.base_url,

            token=self.token,

            account=self.account,

            container=self.container)

class _store_account_command(_pithos_init):

    """Base class for account level storage commands"""

    def __init__(self, arguments={}):

        super(_store_account_command, self).__init__(arguments)

        self['account'] = ValueArgument(

            'Set user account (not permanent)',

            '--account')

    def main(self):

        super(_store_account_command, self).main()

        if self['account']:

            self.client.account = self['account']

class _store_container_command(_store_account_command):

    """Base class for container level storage commands"""

    generic_err_details = ['To specify a container:',

    '  1. Set store.container variable (permanent)',

    '     /config set store.container <container>',

    '  2. --container=<container> (temporary, overrides 1)',

    '  3. Use the container:path format (temporary, overrides all)']

    container = None

    path = None

    def __init__(self, arguments={}):

        super(_store_container_command, self).__init__(arguments)

        self['container'] = ValueArgument(

            'Set container to work with (temporary)',

            '--container')

    def extract_container_and_path(self,

        container_with_path,

        path_is_optional=True):

        try:

            assert isinstance(container_with_path, str)

        except AssertionError as err:

            raiseCLIError(err)

        cont, sep, path = container_with_path.partition(':')

        if sep:

            if not cont:

                raiseCLIError(CLISyntaxError('Container is missing\n',

                    details=self.generic_err_details))

            alt_cont = self['container']

            if alt_cont and cont != alt_cont:

                raiseCLIError(CLISyntaxError(

                    'Conflict: 2 containers (%s, %s)' % (cont, alt_cont),

                    details=self.generic_err_details)

                )

            self.container = cont

            if not path:

                raiseCLIError(CLISyntaxError(

                    'Path is missing for object in container %s' % cont,

                    details=self.generic_err_details)

                )

            self.path = path

        else:

            alt_cont = self['container'] or self.client.container

            if alt_cont:

                self.container = alt_cont

                self.path = cont

            elif path_is_optional:

                self.container = cont

                self.path = None

            else:

                self.container = cont

                raiseCLIError(CLISyntaxError(

                    'Both container and path are required',

                    details=self.generic_err_details)

                )

    def main(self, container_with_path=None, path_is_optional=True):

        super(_store_container_command, self).main()

        if container_with_path is not None:

            self.extract_container_and_path(

                container_with_path,

                path_is_optional)

            self.client.container = self.container

        elif self['container']:

            self.client.container = self['container']

        self.container = self.client.container

@command(pithos_cmds)

class store_list(_store_container_command):

    """List containers, object trees or objects in a directory

    Use with:

    1 no parameters : containers in set account

    2. one parameter (container) or --container : contents of container

    3. <container>:<prefix> or --container=<container> <prefix>: objects in

    .   container starting with prefix

    """

    arguments = dict(

        detail=FlagArgument('show detailed output', '-l'),

        limit=IntArgument('limit the number of listed items', '-n'),

        marker=ValueArgument('show output greater that marker', '--marker'),

        prefix=ValueArgument('show output starting with prefix', '--prefix'),

        delimiter=ValueArgument('show output up to delimiter', '--delimiter'),

        path=ValueArgument(

            'show output starting with prefix up to /',

            '--path'),

        meta=ValueArgument(

            'show output with specified meta keys',

            '--meta',

            default=[]),

        if_modified_since=ValueArgument(

            'show output modified since then',

            '--if-modified-since'),

        if_unmodified_since=ValueArgument(

            'show output not modified since then',

            '--if-unmodified-since'),

        until=DateArgument('show metadata until then', '--until'),

        format=ValueArgument(

            'format to parse until data (default: d/m/Y H:M:S )',

            '--format'),

        shared=FlagArgument('show only shared', '--shared'),

        public=FlagArgument('show only public', '--public'),

        more=FlagArgument(

            'output results in pages (-n to set items per page, default 10)',

            '--more'),

        exact_match=FlagArgument(

            'Show only objects that match exactly with path',

            '--exact-match')

    )

    def print_objects(self, object_list):

        limit = int(self['limit']) if self['limit'] > 0 else len(object_list)

        for index, obj in enumerate(object_list):

            if (self['exact_match'] and self.path and\

                obj['name'] != self.path) or 'content_type' not in obj:

                continue

            pretty_obj = obj.copy()

            index += 1

            empty_space = ' ' * (len(str(len(object_list))) - len(str(index)))

            if obj['content_type'] == 'application/directory':

                isDir = True

                size = 'D'

            else:

                isDir = False

                size = format_size(obj['bytes'])

                pretty_obj['bytes'] = '%s (%s)' % (obj['bytes'], size)

            oname = bold(obj['name'])

            if self['detail']:

                print('%s%s. %s' % (empty_space, index, oname))

                print_dict(pretty_keys(pretty_obj), exclude=('name'))

                print

            else:

                oname = '%s%s. %6s %s' % (empty_space, index, size, oname)

                oname += '/' if isDir else ''

                print(oname)

            if self['more']:

                page_hold(index, limit, len(object_list))

    def print_containers(self, container_list):

        limit = int(self['limit']) if self['limit'] > 0\

            else len(container_list)

        for index, container in enumerate(container_list):

            if 'bytes' in container:

                size = format_size(container['bytes'])

            cname = '%s. %s' % (index + 1, bold(container['name']))

            if self['detail']:

                print(cname)

                pretty_c = container.copy()

                if 'bytes' in container:

                    pretty_c['bytes'] = '%s (%s)' % (container['bytes'], size)

                print_dict(pretty_keys(pretty_c), exclude=('name'))

                print

            else:

                if 'count' in container and 'bytes' in container:

                    print('%s (%s, %s objects)'\

                    % (cname, size, container['count']))

                else:

                    print(cname)

            if self['more']:

                page_hold(index + 1, limit, len(container_list))

    def main(self, container____path__=None):

        super(self.__class__, self).main(container____path__)

        try:

            if self.container is None:

                r = self.client.account_get(

                    limit=False if self['more'] else self['limit'],

                    marker=self['marker'],

                    if_modified_since=self['if_modified_since'],

                    if_unmodified_since=self['if_unmodified_since'],

                    until=self['until'],

                    show_only_shared=self['shared'])

                self.print_containers(r.json)

            else:

                prefix = self.path if self.path else self['prefix']

                r = self.client.container_get(

                    limit=False if self['more'] else self['limit'],

                    marker=self['marker'],

                    prefix=prefix,

                    delimiter=self['delimiter'],

                    path=self['path'],

                    if_modified_since=self['if_modified_since'],

                    if_unmodified_since=self['if_unmodified_since'],

                    until=self['until'],

                    meta=self['meta'],

                    show_only_shared=self['shared'])

                self.print_objects(r.json)

        except ClientError as err:

            if err.status == 404:

                if 'container' in ('%s' % err).lower():

                    raiseCLIError(

                        err,

                        'No container %s in account %s'\

                        % (self.container, self.account),

                        details=self.generic_err_details)

                elif 'object' in ('%s' % err).lower():

                    raiseCLIError(

                        err,

                        'No object %s in %s\'s container %s'\

                        % (self.path, self.account, self.container),

                        details=self.generic_err_details)

            raise_connection_errors(err)

            raiseCLIError(err)

        except Exception as e:

            raiseCLIError(e)

@command(pithos_cmds)

class store_mkdir(_store_container_command):

    """Create a directory"""

    __doc__ += '\n. '.join(about_directories)

    def main(self, container___directory):

        super(self.__class__,

            self).main(container___directory, path_is_optional=False)

        try:

            self.client.create_directory(self.path)

        except ClientError as err:

            if err.status == 404:

                if 'container' in ('%s' % err).lower():

                    raiseCLIError(

                        err,

                        'No container %s in account %s'\

                        % (self.container, self.account),

                        details=self.generic_err_details)

                elif 'object' in ('%s' % err).lower():

                    raiseCLIError(

                        err,

                        'No object %s in container %s'\

                        % (self.path, self.container),

                        details=self.generic_err_details)

            raise_connection_errors(err)

            raiseCLIError(err)

        except Exception as err:

            raiseCLIError(err)

@command(pithos_cmds)

class store_touch(_store_container_command):

    """Create an empty object (file)

    If object exists, this command will reset it to 0 length

    """

    arguments = dict(

        content_type=ValueArgument(

            'Set content type (default: application/octet-stream)',

            '--content-type',

            default='application/octet-stream')

    )

    def main(self, container___path):

        super(store_touch, self).main(container___path)

        try:

            self.client.create_object(self.path, self['content_type'])

        except ClientError as err:

            if err.status == 404:

                if 'container' in ('%s' % err).lower():

                    raiseCLIError(

                        err,

                        'No container %s in account %s'\

                        % (self.container, self.account),

                        details=self.generic_err_details)

                elif 'object' in ('%s' % err).lower():

                    raiseCLIError(

                        err,

                        'No object %s in container %s'\

                        % (self.path, self.container),

                        details=self.generic_err_details)

            raise_connection_errors(err)

            raiseCLIError(err)

        except Exception as err:

            raiseCLIError(err)

@command(pithos_cmds)

class store_create(_store_container_command):

    """Create a container"""

    arguments = dict(

        versioning=ValueArgument(

            'set container versioning (auto/none)',

            '--versioning'),

        quota=IntArgument('set default container quota', '--quota'),

        meta=KeyValueArgument(

            'set container metadata (can be repeated)',

            '--meta')

    )

    def main(self, container):

        super(self.__class__, self).main(container)

        try:

            self.client.container_put(quota=self['quota'],

                versioning=self['versioning'],

                metadata=self['meta'])

        except ClientError as err:

            if err.status == 404:

                if 'container' in ('%s' % err).lower():

                    raiseCLIError(

                        err,

                        'No container %s in account %s'\

                        % (self.container, self.account),

                        details=self.generic_err_details)

                elif 'object' in ('%s' % err).lower():

                    raiseCLIError(

                        err,

                        'No object %s in container %s'\

                        % (self.path, self.container),

                        details=self.generic_err_details)

            raise_connection_errors(err)

            raiseCLIError(err)

        except Exception as e:

            raiseCLIError(e)

@command(pithos_cmds)

class store_copy(_store_container_command):

    """Copy objects from container to (another) container

    Semantics:

    copy cont:path path2

    .   will copy all <obj> prefixed with path, as path2<obj>

    .   or as path2 if path corresponds to just one whole object

    copy cont:path cont2:

    .   will copy all <obj> prefixed with path to container cont2

    copy cont:path [cont2:]path2 --exact-match

    .   will copy at most one <obj> as a new object named path2,

    .   provided path corresponds to a whole object path

    copy cont:path [cont2:]path2 --replace

    .   will copy all <obj> prefixed with path, replacing path with path2

    where <obj> is a full file or directory object path.

    Use options:

    1. <container1>:<path1> [container2:]<path2> : if container2 is not given,

    destination is container1:path2

    2. <container>:<path1> <path2> : make a copy in the same container

    3. Can use --container= instead of <container1>

    """

    arguments = dict(

        source_version=ValueArgument(

            'copy specific version',

            '--source-version'),

        public=ValueArgument('make object publicly accessible', '--public'),

        content_type=ValueArgument(

            'change object\'s content type',

            '--content-type'),

        recursive=FlagArgument(

            'mass copy with delimiter /',

            ('-r', '--recursive')),

        exact_match=FlagArgument(

            'Copy only the object that fully matches path',

            '--exact-match'),

        replace=FlagArgument('Replace src. path with dst. path', '--replace')

    )

    def _objlist(self, dst_path):

        if self['exact_match']:

            return [(dst_path if dst_path else self.path, self.path)]

        r = self.client.container_get(prefix=self.path)

        if len(r.json) == 1:

            obj = r.json[0]

            return [(obj['name'], dst_path if dst_path else obj['name'])]

        return [(obj['name'], '%s%s' % (

                    dst_path,

                    obj['name'][len(self.path) if self['replace'] else 0:])

                ) for obj in r.json]

    def main(self, source_container___path, destination_container___path):

        super(self.__class__,

            self).main(source_container___path, path_is_optional=False)

        try:

            dst = destination_container___path.split(':')

            (dst_cont, dst_path) = (dst[0], dst[1])\

            if len(dst) > 1 else (None, dst[0])

            for src_object, dst_object in self._objlist(dst_path):

                self.client.copy_object(src_container=self.container,

                    src_object=src_object,

                    dst_container=dst_cont if dst_cont else self.container,

                    dst_object=dst_object,

                    source_version=self['source_version'],

                    public=self['public'],

                    content_type=self['content_type'])

        except ClientError as err:

            if err.status == 404:

                if 'object' in ('%s' % err).lower():

                    raiseCLIError(

                        err,

                        'No object %s in container %s'\

                        % (self.path, self.container),

                        details=self.generic_err_details)

                elif 'container' in ('%s' % err).lower():

                    cont_msg = '(either %s or %s)' % (

                        self.container,

                        dst_cont

                    ) if dst_cont else self.container

                    raiseCLIError(

                        err,

                        'No container %s in account %s'\

                        % (cont_msg, self.account),

                        details=self.generic_err_details)

            raise_connection_errors(err)

            raiseCLIError(err)

        except Exception as e:

            raiseCLIError(e)

@command(pithos_cmds)

class store_move(_store_container_command):

    """Move/rename objects

    Semantics:

    move cont:path path2

    .   will move all <obj> prefixed with path, as path2<obj>

    .   or as path2 if path corresponds to just one whole object

    move cont:path cont2:

    .   will move all <obj> prefixed with path to container cont2

    move cont:path [cont2:]path2 --exact-match

    .   will move at most one <obj> as a new object named path2,

    .   provided path corresponds to a whole object path

    move cont:path [cont2:]path2 --replace

    .   will move all <obj> prefixed with path, replacing path with path2

    where <obj> is a full file or directory object path.

    Use options:

    1. <container1>:<path1> [container2:]<path2> : if container2 not given,

    destination is container1:path2

    2. <container>:<path1> path2 : rename

    3. Can use --container= instead of <container1>

    """

    arguments = dict(

        source_version=ValueArgument('specify version', '--source-version'),

        public=FlagArgument('make object publicly accessible', '--public'),

        content_type=ValueArgument('modify content type', '--content-type'),

        recursive=FlagArgument('up to delimiter /', ('-r', '--recursive')),

        exact_match=FlagArgument(

            'Copy only the object that fully matches path',

            '--exact-match'),

        replace=FlagArgument('Replace src. path with dst. path', '--replace')

    )

    def _objlist(self, dst_path):

        if self['exact_match']:

            return [(dst_path if dst_path else self.path, self.path)]

        r = self.client.container_get(prefix=self.path)

        if len(r.json) == 1:

            obj = r.json[0]

            return [(obj['name'], dst_path if dst_path else obj['name'])]

        return [(obj['name'], '%s%s' % (

                    dst_path,

                    obj['name'][len(self.path) if self['replace'] else 0:])

                ) for obj in r.json]

    def main(self, source_container___path, destination_container____path__):

        super(self.__class__,

            self).main(source_container___path, path_is_optional=False)

        try:

            dst = destination_container____path__.split(':')

            (dst_cont, dst_path) = (dst[0], dst[1])\

            if len(dst) > 1 else (None, dst[0])

            for src_object, dst_object in self._objlist(dst_path):

                self.client.move_object(

                    src_container=self.container,

                    src_object=src_object,

                    dst_container=dst_cont if dst_cont else self.container,

                    dst_object=dst_object,

                    source_version=self['source_version'],

                    public=self['public'],

                    content_type=self['content_type'])

        except ClientError as err:

            if err.status == 404:

                if 'object' in ('%s' % err).lower():

                    raiseCLIError(

                        err,

                        'No object %s in container %s'\

                        % (self.path, self.container),