Synnefo » Pithos

#!/usr/bin/env python

from getpass import getuser

from optparse import OptionParser

from os.path import basename

from sys import argv, exit, stdin, stdout

from pithos.lib.client import Client, Fault

from datetime import datetime

import json

import logging

import types

import re

import time as _time

DEFAULT_HOST = 'pithos.dev.grnet.gr'

DEFAULT_API = 'v1'

_cli_commands = {}

def cli_command(*args):

    def decorator(cls):

        cls.commands = args

        for name in args:

            _cli_commands[name] = cls

        return cls

    return decorator

def class_for_cli_command(name):

    return _cli_commands[name]

class Command(object):

    def __init__(self, argv):

        parser = OptionParser()

        parser.add_option('--host', dest='host', metavar='HOST',

                          default=DEFAULT_HOST, help='use server HOST')

        parser.add_option('--user', dest='user', metavar='USERNAME',

                          default=getuser(), help='use account USERNAME')

        parser.add_option('--api', dest='api', metavar='API',

                          default=DEFAULT_API, help='use api API')

        parser.add_option('-v', action='store_true', dest='verbose',

                          default=False, help='use verbose output')

        parser.add_option('-d', action='store_true', dest='debug',

                          default=False, help='use debug output')

        self.add_options(parser)

        options, args = parser.parse_args(argv)

        # Add options to self

        for opt in parser.option_list:

            key = opt.dest

            if key:

                val = getattr(options, key)

                setattr(self, key, val)

        self.client = Client(self.host, self.user, self.api, self.verbose,

                             self.debug)

        self.parser = parser

        self.args = args

    def add_options(self, parser):

        pass

    def execute(self, *args):

        pass

@cli_command('list', 'ls')

class List(Command):

    syntax = '[<container>[/<object>]]'

    description = 'list containers or objects'

    def add_options(self, parser):

        parser.add_option('-l', action='store_true', dest='detail',

                          default=False, help='show detailed output')

        parser.add_option('-n', action='store', type='int', dest='limit',

                          default=1000, help='show limited output')

        parser.add_option('--marker', action='store', type='str',

                          dest='marker', default=None,

                          help='show output greater then marker')

        parser.add_option('--prefix', action='store', type='str',

                          dest='prefix', default=None,

                          help='show output starting with prefix')

        parser.add_option('--delimiter', action='store', type='str',

                          dest='delimiter', default=None,

                          help='show output up to the delimiter')

        parser.add_option('--path', action='store', type='str',

                          dest='path', default=None,

                          help='show output starting with prefix up to /')

        parser.add_option('--meta', action='store', type='str',

                          dest='meta', default=None,

                          help='show output having the specified meta keys')

        parser.add_option('--if-modified-since', action='store', type='str',

                          dest='if_modified_since', default=None,

                          help='show output if modified since then')

        parser.add_option('--if-unmodified-since', action='store', type='str',

                          dest='if_unmodified_since', default=None,

                          help='show output if not modified since then')

        parser.add_option('--until', action='store', dest='until',

                          default=False, help='show metadata until that date')

        parser.add_option('--format', action='store', dest='format',

                          default='%d/%m/%Y', help='format to parse until date')

    def execute(self, container=None):

        if container:

            self.list_objects(container)

        else:

            self.list_containers()

    def list_containers(self):

        params = {'limit':self.limit, 'marker':self.marker}

        headers = {'IF_MODIFIED_SINCE':self.if_modified_since,

                   'IF_UNMODIFIED_SINCE':self.if_unmodified_since}

        if self.until:

            t = _time.strptime(self.until, self.format)

            params['until'] = int(_time.mktime(t))

        l = self.client.list_containers(self.detail, params, headers)

        print_list(l)

    def list_objects(self, container):

        params = {'limit':self.limit, 'marker':self.marker,

                  'prefix':self.prefix, 'delimiter':self.delimiter,

                  'path':self.path, 'meta':self.meta}

        headers = {'IF_MODIFIED_SINCE':self.if_modified_since,

                   'IF_UNMODIFIED_SINCE':self.if_unmodified_since}

        container, sep, object = container.partition('/')

        if object:

            return

        if self.until:

            t = _time.strptime(self.until, self.format)

            params['until'] = int(_time.mktime(t))

        l = self.client.list_objects(container, self.detail, params, headers)

        print_list(l)

@cli_command('meta')

class Meta(Command):

    syntax = '[<container>[/<object>]]'

    description = 'get the metadata of an account, a container or an object'

    def add_options(self, parser):

        parser.add_option('-r', action='store_true', dest='restricted',

                          default=False, help='show only user defined metadata')

        parser.add_option('--until', action='store', dest='until',

                          default=False, help='show metadata until that date')

        parser.add_option('--format', action='store', dest='format',

                          default='%d/%m/%Y', help='format to parse until date')

        parser.add_option('--version', action='store', dest='version',

                          default=None, help='show specific version \

                                  (applies only for objects)')

    def execute(self, path=''):

        container, sep, object = path.partition('/')

        if self.until:

            t = _time.strptime(self.until, self.format)

            self.until = int(_time.mktime(t))

        if object:

            meta = self.client.retrieve_object_metadata(container, object,

                                                        self.restricted,

                                                        self.version)

        elif container:

            meta = self.client.retrieve_container_metadata(container,

                                                           self.restricted,

                                                           self.until)

        else:

            meta = self.client.account_metadata(self.restricted, self.until)

        if meta == None:

            print 'Entity does not exist'

        else:

            print_dict(meta, header=None)

@cli_command('create')

class CreateContainer(Command):

    syntax = '<container> [key=val] [...]'

    description = 'create a container'

    def execute(self, container, *args):

        headers = {}

        for arg in args:

            key, sep, val = arg.partition('=')

            headers['X_CONTAINER_META_%s' %key.strip().upper()] = val.strip()

        ret = self.client.create_container(container, headers)

        if not ret:

            print 'Container already exists'

@cli_command('delete', 'rm')

class Delete(Command):

    syntax = '<container>[/<object>]'

    description = 'delete a container or an object'

    def execute(self, path):

        container, sep, object = path.partition('/')

        if object:

            self.client.delete_object(container, object)

        else:

            self.client.delete_container(container)

@cli_command('get')

class GetObject(Command):

    syntax = '<container>/<object>'

    description = 'get the data of an object'

    def add_options(self, parser):

        parser.add_option('-l', action='store_true', dest='detail',

                          default=False, help='show detailed output')

        parser.add_option('--range', action='store', dest='range',

                          default=None, help='show range of data')

        parser.add_option('--if-match', action='store', dest='if-match',

                          default=None, help='show output if ETags match')

        parser.add_option('--if-none-match', action='store',

                          dest='if-none-match', default=None,

                          help='show output if ETags don\'t match')

        parser.add_option('--if-modified-since', action='store', type='str',

                          dest='if-modified-since', default=None,

                          help='show output if modified since then')

        parser.add_option('--if-unmodified-since', action='store', type='str',

                          dest='if-unmodified-since', default=None,

                          help='show output if not modified since then')

        parser.add_option('-f', action='store', type='str',

                          dest='file', default=None,

                          help='save output in file')

        parser.add_option('--version', action='store', type='str',

                          dest='version', default=None,

                          help='get the specific \

                               version')

        parser.add_option('--versionlist', action='store_true',

                          dest='versionlist', default=False,

                          help='get the full object version list')

    def execute(self, path):

        headers = {}

        if self.range:

            headers['RANGE'] = 'bytes=%s' %self.range

        attrs = ['if-match', 'if-none-match', 'if-modified-since',

                 'if-unmodified-since']

        attrs = [a for a in attrs if getattr(self, a)]

        for a in attrs:

            headers[a.replace('-', '_').upper()] = getattr(self, a)

        container, sep, object = path.partition('/')

        if self.versionlist:

            self.version = 'list'

            self.detail = True

        data = self.client.retrieve_object(container, object, self.detail,

                                          headers, self.version)

        f = self.file and open(self.file, 'w') or stdout

        if self.detail:

            data = json.loads(data)

            if self.versionlist:

                print_versions(data, f=f)

            else:

                print_dict(data, f=f)

        else:

            f.write(data)

        f.close()

@cli_command('mkdir')

class PutMarker(Command):

    syntax = '<container>/<directory marker>'

    description = 'create a directory marker'

    def execute(self, path):

        container, sep, object = path.partition('/')

        self.client.create_directory_marker(container, object)

@cli_command('put')

class PutObject(Command):

    syntax = '<container>/<object> <path> [key=val] [...]'

    description = 'create/override object with path contents or standard input'

    def add_options(self, parser):

        parser.add_option('--chunked', action='store_true', dest='chunked',

                          default=False, help='set chunked transfer mode')

        parser.add_option('--etag', action='store', dest='etag',

                          default=None, help='check written data')

        parser.add_option('--content-encoding', action='store',

                          dest='content-encoding', default=None,

                          help='provide the object MIME content type')

        parser.add_option('--content-disposition', action='store', type='str',

                          dest='content-disposition', default=None,

                          help='provide the presentation style of the object')

        parser.add_option('--manifest', action='store', type='str',

                          dest='manifest', default=None,

                          help='use for large file support')

        parser.add_option('--type', action='store',

                          dest='content-type', default=False,

                          help='create object with specific content type')

        parser.add_option('--touch', action='store_true',

                          dest='touch', default=False,

                          help='create object with zero data')

    def execute(self, path, srcpath='-', *args):

        headers = {}

        if self.manifest:

            headers['X_OBJECT_MANIFEST'] = self.manifest

        attrs = ['etag', 'content-encoding', 'content-disposition']

        attrs = ['etag', 'content-encoding', 'content-disposition',

                 'content-type']

        attrs = [a for a in attrs if getattr(self, a)]

        for a in attrs:

            headers[a.replace('-', '_').upper()] = getattr(self, a)

        #prepare user defined meta

        for arg in args:

            key, sep, val = arg.partition('=')

            headers['X_OBJECT_META_%s' %key.strip().upper()] = val.strip()

        container, sep, object = path.partition('/')

        f = None

        chunked = False

        if not self.touch:

            f = srcpath != '-' and open(srcpath) or stdin

            chunked = (self.chunked or f == stdin) and True or False

        self.client.create_object(container, object, f, chunked=chunked,

                                  headers=headers)

        if f:

            f.close()

@cli_command('copy', 'cp')

class CopyObject(Command):

    syntax = '<src container>/<src object> [<dst container>/]<dst object>'

    description = 'copies an object to a different location'

    def add_options(self, parser):

        parser.add_option('--version', action='store',

                          dest='version', default=False,

                          help='copy specific version')

    def execute(self, src, dst):

        src_container, sep, src_object = src.partition('/')

        dst_container, sep, dst_object = dst.partition('/')

        if not sep:

            dst_container = src_container

            dst_object = dst

        version = getattr(self, 'version')

        if version:

            headers = {}

            headers['X_SOURCE_VERSION'] = version

        self.client.copy_object(src_container, src_object, dst_container,

                                dst_object, headers)

@cli_command('set')

class SetMeta(Command):

    syntax = '[<container>[/<object>]] key=val [key=val] [...]'

    description = 'set metadata'

    def execute(self, path, *args):

        #in case of account fix the args

        if path.find('=') != -1:

            args = list(args)

            args.append(path)

            args = tuple(args)

            path = ''

        meta = {}

        for arg in args:

            key, sep, val = arg.partition('=')

            meta[key.strip()] = val.strip()

        container, sep, object = path.partition('/')

        if object:

            self.client.update_object_metadata(container, object, **meta)

        elif container:

            self.client.update_container_metadata(container, **meta)

        else:

            self.client.update_account_metadata(**meta)

@cli_command('update')

class UpdateObject(Command):

    syntax = '<container>/<object> path [key=val] [...]'

    description = 'update object metadata/data (default mode: append)'

    def add_options(self, parser):

        parser.add_option('-a', action='store_true', dest='append',

                          default=True, help='append data')

        parser.add_option('--start', action='store',

                          dest='start',

                          default=None, help='range of data to be updated')

        parser.add_option('--range', action='store', dest='content-range',

                          default=None, help='range of data to be updated')

        parser.add_option('--chunked', action='store_true', dest='chunked',

                          default=False, help='set chunked transfer mode')

        parser.add_option('--content-encoding', action='store',

                          dest='content-encoding', default=None,

                          help='provide the object MIME content type')

        parser.add_option('--content-disposition', action='store', type='str',

                          dest='content-disposition', default=None,

                          help='provide the presentation style of the object')

        parser.add_option('--manifest', action='store', type='str',

                          dest='manifest', default=None,

                          help='use for large file support')

    def execute(self, path, srcpath='-', *args):

        headers = {}

        if self.manifest:

            headers['X_OBJECT_MANIFEST'] = self.manifest

        if getattr(self, 'start'):

            headers['CONTENT_RANGE'] = 'bytes %s-/*' % getattr(self, 'start')

        elif self.append:

            headers['CONTENT_RANGE'] = 'bytes */*'

        attrs = ['content-encoding', 'content-disposition']

        attrs = [a for a in attrs if getattr(self, a)]

        for a in attrs:

            headers[a.replace('-', '_').upper()] = getattr(self, a)

        #prepare user defined meta

        for arg in args:

            key, sep, val = arg.partition('=')

            headers['X_OBJECT_META_%s' %key.strip().upper()] = val.strip()

        container, sep, object = path.partition('/')

        f = srcpath != '-' and open(srcpath) or stdin

        chunked = (self.chunked or f == stdin) and True or False

        self.client.update_object(container, object, f, chunked=chunked,

                                  headers=headers)

        f.close()

@cli_command('move', 'mv')

class MoveObject(Command):

    syntax = '<src container>/<src object> [<dst container>/]<dst object>'

    description = 'moves an object to a different location'

    def add_options(self, parser):

        parser.add_option('--version', action='store',

                          dest='version', default=False,

                          help='move specific version')

    def execute(self, src, dst):

        src_container, sep, src_object = src.partition('/')

        dst_container, sep, dst_object = dst.partition('/')

        if not sep:

            dst_container = src_container

            dst_object = dst

        version = getattr(self, 'version')

        if version:

            headers = {}

            headers['X_SOURCE_VERSION'] = version 

        self.client.move_object(src_container, src_object, dst_container,

                                dst_object, headers)

@cli_command('remove', 'rm')

class TrashObject(Command):

    syntax = '<container>/<object>'

    description = 'trashes an object'

    def execute(self, src):

        src_container, sep, src_object = src.partition('/')

        self.client.trash_object(src_container, src_object)

@cli_command('restore')

class TrashObject(Command):

    syntax = '<container>/<object>'

    description = 'trashes an object'

    def execute(self, src):

        src_container, sep, src_object = src.partition('/')

        self.client.restore_object(src_container, src_object)

@cli_command('unset')

class TrashObject(Command):

    syntax = '<container>/[<object>] key [key] [...]'

    description = 'deletes metadata info'

    def execute(self, path, *args):

        #in case of account fix the args

        if path.find('=') != -1:

            args = list(args)

            args.append(path)

            args = tuple(args)

            path = ''

        meta = []

        for key in args:

            meta.append(key)

        container, sep, object = path.partition('/')

        if object:

            self.client.delete_object_metadata(container, object, meta)

        elif container:

            self.client.delete_container_metadata(container, meta)

        else:

            self.client.delete_account_metadata(meta)

def print_usage():

    cmd = Command([])

    parser = cmd.parser

    parser.usage = '%prog <command> [options]'

    parser.print_help()

    commands = []

    for cls in set(_cli_commands.values()):

        name = ', '.join(cls.commands)

        description = getattr(cls, 'description', '')

        commands.append('  %s %s' % (name.ljust(12), description))

    print '\nCommands:\n' + '\n'.join(sorted(commands))

def print_dict(d, header='name', f=stdout):

    header = header in d and header or 'subdir'

    if header and header in d:

        f.write('%s\n' %d.pop(header))

    patterns = ['^x_(account|container|object)_meta_(\w+)$']

    patterns.append(patterns[0].replace('_', '-'))

    for key, val in sorted(d.items()):

        for p in patterns:

            p = re.compile(p)

            m = p.match(key)

            if m:

                key = m.group(2)

        f.write('%s: %s\n' % (key.rjust(30), val))

def print_list(l, verbose=False, f=stdout):

    for elem in l:

        #if it's empty string continue

        if not elem:

            continue

        if type(elem) == types.DictionaryType:

            print_dict(elem, f=f)

        elif type(elem) == types.StringType:

            if not verbose:

                elem = elem.split('Traceback')[0]

            f.write('%s\n' % elem)

        else:

            f.write('%s\n' % elem)

def print_versions(data, f=stdout):

    if 'versions' not in data:

        f.write('%s\n' %data)

        return

    f.write('versions:\n')

    for id, t in data['versions']:

        f.write('%s @ %s\n' % (str(id).rjust(30), datetime.fromtimestamp(t)))

def main():

    try:

        name = argv[1]

        cls = class_for_cli_command(name)

    except (IndexError, KeyError):

        print_usage()

        exit(1)

    cmd = cls(argv[2:])

    try:

        cmd.execute(*cmd.args)

    except TypeError, e:

        print e

        cmd.parser.usage = '%%prog %s [options] %s' % (name, cmd.syntax)

        cmd.parser.print_help()

        exit(1)

    except Fault, f:

        print f.status, f.data

        status = f.status and '%s ' % f.status or ''

        print '%s%s' % (status, f.data)

if __name__ == '__main__':

    main()