X-Git-Url: https://code.grnet.gr/git/pithos/blobdiff_plain/7e999dec36ebd192d401ba24718fa4cc406a98c7..b980169d8828d5f42afaea95488ca2ba7436b21e:/tools/store diff --git a/tools/store b/tools/store index 993b1b3..316ae52 100755 --- a/tools/store +++ b/tools/store @@ -1,8 +1,41 @@ #!/usr/bin/env python +# Copyright 2011 GRNET S.A. All rights reserved. +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# 1. Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# 2. Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials +# provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS +# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# The views and conclusions contained in the software and +# documentation are those of the authors and should not be +# interpreted as representing official policies, either expressed +# or implied, of GRNET S.A. + from getpass import getuser from optparse import OptionParser -from os.path import basename +from os import environ from sys import argv, exit, stdin, stdout from pithos.lib.client import Client, Fault from datetime import datetime @@ -12,6 +45,7 @@ import logging import types import re import time as _time +import os DEFAULT_HOST = 'pithos.dev.grnet.gr' DEFAULT_API = 'v1' @@ -30,12 +64,18 @@ def class_for_cli_command(name): return _cli_commands[name] class Command(object): - def __init__(self, argv): - parser = OptionParser() + syntax = '' + + def __init__(self, name, argv): + parser = OptionParser('%%prog %s [options] %s' % (name, self.syntax)) parser.add_option('--host', dest='host', metavar='HOST', default=DEFAULT_HOST, help='use server HOST') parser.add_option('--user', dest='user', metavar='USERNAME', - default=getuser(), help='use account USERNAME') + default=_get_user(), + help='use account USERNAME') + parser.add_option('--token', dest='token', metavar='AUTH', + default=_get_auth(), + help='use account AUTH') parser.add_option('--api', dest='api', metavar='API', default=DEFAULT_API, help='use api API') parser.add_option('-v', action='store_true', dest='verbose', @@ -52,15 +92,15 @@ class Command(object): val = getattr(options, key) setattr(self, key, val) - self.client = Client(self.host, self.user, self.api, self.verbose, + self.client = Client(self.host, self.token, self.user, self.api, self.verbose, self.debug) self.parser = parser self.args = args - + def add_options(self, parser): pass - + def execute(self, *args): pass @@ -99,13 +139,13 @@ class List(Command): default=False, help='show metadata until that date') parser.add_option('--format', action='store', dest='format', default='%d/%m/%Y', help='format to parse until date') - + def execute(self, container=None): if container: self.list_objects(container) else: self.list_containers() - + def list_containers(self): params = {'limit':self.limit, 'marker':self.marker} headers = {'IF_MODIFIED_SINCE':self.if_modified_since, @@ -117,23 +157,30 @@ class List(Command): l = self.client.list_containers(self.detail, params, headers) print_list(l) - + def list_objects(self, container): - params = {'limit':self.limit, 'marker':self.marker, - 'prefix':self.prefix, 'delimiter':self.delimiter, - 'path':self.path, 'meta':self.meta} + #prepate params + params = {} + attrs = ['limit', 'marker', 'prefix', 'delimiter', 'path', 'meta'] + for a in [a for a in attrs if getattr(self, a)]: + params[a] = getattr(self, a) + + if self.until: + t = _time.strptime(self.until, self.format) + params['until'] = int(_time.mktime(t)) + headers = {'IF_MODIFIED_SINCE':self.if_modified_since, 'IF_UNMODIFIED_SINCE':self.if_unmodified_since} container, sep, object = container.partition('/') if object: return - if self.until: - t = _time.strptime(self.until, self.format) - params['until'] = int(_time.mktime(t)) - - l = self.client.list_objects(container, self.detail, params, headers) - print_list(l) + detail = 'json' + #if request with meta quering disable trash filtering + show_trashed = True if self.meta else False + l = self.client.list_objects(container, detail, headers, + include_trashed = show_trashed, **params) + print_list(l, detail=self.detail) @cli_command('meta') class Meta(Command): @@ -150,7 +197,7 @@ class Meta(Command): parser.add_option('--version', action='store', dest='version', default=None, help='show specific version \ (applies only for objects)') - + def execute(self, path=''): container, sep, object = path.partition('/') if self.until: @@ -178,10 +225,11 @@ class CreateContainer(Command): def execute(self, container, *args): headers = {} + meta = {} for arg in args: key, sep, val = arg.partition('=') - headers['X_CONTAINER_META_%s' %key.strip().upper()] = val.strip() - ret = self.client.create_container(container, headers) + meta[key] = val + ret = self.client.create_container(container, headers, **meta) if not ret: print 'Container already exists' @@ -207,6 +255,8 @@ class GetObject(Command): default=False, help='show detailed output') parser.add_option('--range', action='store', dest='range', default=None, help='show range of data') + parser.add_option('--if-range', action='store', dest='if-range', + default=None, help='show range of data') parser.add_option('--if-match', action='store', dest='if-match', default=None, help='show output if ETags match') parser.add_option('--if-none-match', action='store', @@ -218,7 +268,7 @@ class GetObject(Command): parser.add_option('--if-unmodified-since', action='store', type='str', dest='if-unmodified-since', default=None, help='show output if not modified since then') - parser.add_option('-f', action='store', type='str', + parser.add_option('-o', action='store', type='str', dest='file', default=None, help='save output in file') parser.add_option('--version', action='store', type='str', @@ -228,11 +278,13 @@ class GetObject(Command): parser.add_option('--versionlist', action='store_true', dest='versionlist', default=False, help='get the full object version list') - + def execute(self, path): headers = {} if self.range: headers['RANGE'] = 'bytes=%s' %self.range + if getattr(self, 'if-range'): + headers['IF_RANGE'] = 'If-Range:%s' % getattr(self, 'if-range') attrs = ['if-match', 'if-none-match', 'if-modified-since', 'if-unmodified-since'] attrs = [a for a in attrs if getattr(self, a)] @@ -255,12 +307,23 @@ class GetObject(Command): f.write(data) f.close() +@cli_command('mkdir') +class PutMarker(Command): + syntax = '/' + description = 'create a directory marker' + + def execute(self, path): + container, sep, object = path.partition('/') + self.client.create_directory_marker(container, object) + @cli_command('put') class PutObject(Command): syntax = '/ [key=val] [...]' - description = 'create/override object with path contents or standard input' - + description = 'create/override object' + def add_options(self, parser): + parser.add_option('--use_hashes', action='store_true', dest='use_hashes', + default=False, help='provide hashmap instead of data') parser.add_option('--chunked', action='store_true', dest='chunked', default=False, help='set chunked transfer mode') parser.add_option('--etag', action='store', dest='etag', @@ -271,50 +334,81 @@ class PutObject(Command): parser.add_option('--content-disposition', action='store', type='str', dest='content-disposition', default=None, help='provide the presentation style of the object') - parser.add_option('--manifest', action='store', type='str', - dest='manifest', default=None, + parser.add_option('-S', action='store', + dest='segment-size', default=False, help='use for large file support') - parser.add_option('--touch', action='store_true', - dest='touch', default=False, - help='create object with zero data') - - def execute(self, path, srcpath='-', *args): - headers = {} - if self.manifest: - headers['X_OBJECT_MANIFEST'] = self.manifest - - attrs = ['etag', 'content-encoding', 'content-disposition'] - attrs = [a for a in attrs if getattr(self, a)] - for a in attrs: - headers[a.replace('-', '_').upper()] = getattr(self, a) + parser.add_option('--manifest', action='store_true', + dest='manifest', default=None, + help='upload a manifestation file') + parser.add_option('--type', action='store', + dest='content-type', default=False, + help='create object with specific content type') + parser.add_option('--sharing', action='store', + dest='sharing', default=None, + help='define sharing object policy') + parser.add_option('-f', action='store', + dest='srcpath', default=None, + help='file descriptor to read from (pass - for standard input)') + parser.add_option('--public', action='store', + dest='public', default=None, + help='make object publicly accessible (\'True\'/\'False\')') + + def execute(self, path, *args): + if path.find('=') != -1: + raise Fault('Missing path argument') #prepare user defined meta + meta = {} for arg in args: key, sep, val = arg.partition('=') - headers['X_OBJECT_META_%s' %key.strip().upper()] = val.strip() + meta[key] = val + + headers = {} + manifest = getattr(self, 'manifest') + if manifest: + # if it's manifestation file + # send zero-byte data with X-Object-Manifest header + self.touch = True + headers['X_OBJECT_MANIFEST'] = manifest + if self.sharing: + headers['X_OBJECT_SHARING'] = self.sharing + + attrs = ['etag', 'content-encoding', 'content-disposition', + 'content-type'] + attrs = [a for a in attrs if getattr(self, a)] + for a in attrs: + headers[a.replace('-', '_').upper()] = getattr(self, a) container, sep, object = path.partition('/') f = None - chunked = False - if not self.touch: - f = srcpath != '-' and open(srcpath) or stdin - chunked = (self.chunked or f == stdin) and True or False - self.client.create_object(container, object, f, chunked=chunked, - headers=headers) + if self.srcpath: + f = open(self.srcpath) if self.srcpath != '-' else stdin + + if self.use_hashes and not f: + raise Fault('Illegal option combination') + if self.public not in ['True', 'False', None]: + raise Fault('Not acceptable value for public') + public = eval(self.public) if self.public else None + self.client.create_object(container, object, f, chunked=self.chunked, + headers=headers, use_hashes=self.use_hashes, + public=public, **meta) if f: f.close() @cli_command('copy', 'cp') class CopyObject(Command): syntax = '/ [/]' - description = 'copies an object to a different location' + description = 'copy an object to a different location' def add_options(self, parser): parser.add_option('--version', action='store', dest='version', default=False, help='copy specific version') - + parser.add_option('--public', action='store', + dest='public', default=None, + help='publish/unpublish object (\'True\'/\'False\')') + def execute(self, src, dst): src_container, sep, src_object = src.partition('/') dst_container, sep, dst_object = dst.partition('/') @@ -325,8 +419,13 @@ class CopyObject(Command): if version: headers = {} headers['X_SOURCE_VERSION'] = version + if self.public and self.nopublic: + raise Fault('Conflicting options') + if self.public not in ['True', 'False', None]: + raise Fault('Not acceptable value for public') + public = eval(self.public) if self.public else None self.client.copy_object(src_container, src_object, dst_container, - dst_object, headers) + dst_object, public, headers) @cli_command('set') class SetMeta(Command): @@ -360,9 +459,9 @@ class UpdateObject(Command): def add_options(self, parser): parser.add_option('-a', action='store_true', dest='append', default=True, help='append data') - parser.add_option('--start', action='store', - dest='start', - default=None, help='range of data to be updated') + parser.add_option('--offset', action='store', + dest='offset', + default=None, help='starting offest to be updated') parser.add_option('--range', action='store', dest='content-range', default=None, help='range of data to be updated') parser.add_option('--chunked', action='store_true', dest='chunked', @@ -375,17 +474,32 @@ class UpdateObject(Command): help='provide the presentation style of the object') parser.add_option('--manifest', action='store', type='str', dest='manifest', default=None, - help='use for large file support') - - def execute(self, path, srcpath='-', *args): + help='use for large file support') + parser.add_option('--sharing', action='store', + dest='sharing', default=None, + help='define sharing object policy') + parser.add_option('--nosharing', action='store_true', + dest='no_sharing', default=None, + help='clear object sharing policy') + parser.add_option('-f', action='store', + dest='srcpath', default=None, + help='file descriptor to read from: pass - for standard input') + parser.add_option('--public', action='store', + dest='public', default=None, + help='publish/unpublish object (\'True\'/\'False\')') + + def execute(self, path, *args): + if path.find('=') != -1: + raise Fault('Missing path argument') + headers = {} if self.manifest: headers['X_OBJECT_MANIFEST'] = self.manifest + if self.sharing: + headers['X_OBJECT_SHARING'] = self.sharing - if getattr(self, 'start'): - headers['CONTENT_RANGE'] = 'bytes %s-/*' % getattr(self, 'start') - elif self.append: - headers['CONTENT_RANGE'] = 'bytes */*' + if self.no_sharing: + headers['X_OBJECT_SHARING'] = '' attrs = ['content-encoding', 'content-disposition'] attrs = [a for a in attrs if getattr(self, a)] @@ -393,44 +507,148 @@ class UpdateObject(Command): headers[a.replace('-', '_').upper()] = getattr(self, a) #prepare user defined meta + meta = {} for arg in args: key, sep, val = arg.partition('=') - headers['X_OBJECT_META_%s' %key.strip().upper()] = val.strip() + meta[key] = val container, sep, object = path.partition('/') - f = srcpath != '-' and open(srcpath) or stdin - chunked = (self.chunked or f == stdin) and True or False + f = None + chunked = False + if self.srcpath: + f = self.srcpath != '-' and open(self.srcpath) or stdin + if f: + chunked = True if (self.chunked or f == stdin) else False + if self.public not in ['True', 'False', None]: + raise Fault('Not acceptable value for public') + public = eval(self.public) if self.public else None self.client.update_object(container, object, f, chunked=chunked, - headers=headers) - f.close() + headers=headers, offset=self.offset, + public=public, **meta) + if f: + f.close() @cli_command('move', 'mv') class MoveObject(Command): syntax = '/ [/]' - description = 'moves an object to a different location' + description = 'move an object to a different location' def add_options(self, parser): - parser.add_option('--version', action='store', - dest='version', default=False, - help='move specific version') - + parser.add_option('--public', action='store', + dest='public', default=None, + help='publish/unpublish object (\'True\'/\'False\')') + def execute(self, src, dst): src_container, sep, src_object = src.partition('/') dst_container, sep, dst_object = dst.partition('/') if not sep: dst_container = src_container dst_object = dst - - version = getattr(self, 'version') - if version: - headers = {} - headers['X_SOURCE_VERSION'] = version + if self.public not in ['True', 'False', None]: + raise Fault('Not acceptable value for public') + public = eval(self.public) if self.public else None self.client.move_object(src_container, src_object, dst_container, - dst_object, headers) + dst_object, public, headers) + +@cli_command('remove') +class TrashObject(Command): + syntax = '/' + description = 'trash an object' + + def execute(self, src): + src_container, sep, src_object = src.partition('/') + + self.client.trash_object(src_container, src_object) + +@cli_command('restore') +class RestoreObject(Command): + syntax = '/' + description = 'restore a trashed object' + + def execute(self, src): + src_container, sep, src_object = src.partition('/') + + self.client.restore_object(src_container, src_object) + +@cli_command('unset') +class UnsetObject(Command): + syntax = '/[] key [key] [...]' + description = 'delete metadata info' + + def execute(self, path, *args): + #in case of account fix the args + if len(args) == 0: + args = list(args) + args.append(path) + args = tuple(args) + path = '' + meta = [] + for key in args: + meta.append(key) + container, sep, object = path.partition('/') + if object: + self.client.delete_object_metadata(container, object, meta) + elif container: + self.client.delete_container_metadata(container, meta) + else: + self.client.delete_account_metadata(meta) + +@cli_command('group') +class SetGroup(Command): + syntax = 'key=val [key=val] [...]' + description = 'set group account info' + + def execute(self, *args): + groups = {} + for arg in args: + key, sep, val = arg.partition('=') + groups[key] = val + self.client.set_account_groups(**groups) + +@cli_command('policy') +class SetPolicy(Command): + syntax = 'container key=val [key=val] [...]' + description = 'set container policies' + + def execute(self, path, *args): + if path.find('=') != -1: + raise Fault('Missing container argument') + + container, sep, object = path.partition('/') + + if object: + raise Fault('Only containers have policies') + + policies = {} + for arg in args: + key, sep, val = arg.partition('=') + policies[key] = val + + self.client.set_container_policies(container, **policies) + +@cli_command('publish') +class PublishObject(Command): + syntax = '/' + description = 'publish an object' + + def execute(self, src): + src_container, sep, src_object = src.partition('/') + + self.client.publish_object(src_container, src_object) + +@cli_command('unpublish') +class UnpublishObject(Command): + syntax = '/' + description = 'unpublish an object' + + def execute(self, src): + src_container, sep, src_object = src.partition('/') + + self.client.unpublish_object(src_container, src_object) def print_usage(): - cmd = Command([]) + cmd = Command('', []) parser = cmd.parser parser.usage = '%prog [options]' parser.print_help() @@ -442,27 +660,28 @@ def print_usage(): commands.append(' %s %s' % (name.ljust(12), description)) print '\nCommands:\n' + '\n'.join(sorted(commands)) -def print_dict(d, header='name', f=stdout): +def print_dict(d, header='name', f=stdout, detail=True): header = header in d and header or 'subdir' if header and header in d: f.write('%s\n' %d.pop(header)) - patterns = ['^x_(account|container|object)_meta_(\w+)$'] - patterns.append(patterns[0].replace('_', '-')) - for key, val in sorted(d.items()): - for p in patterns: - p = re.compile(p) - m = p.match(key) - if m: - key = m.group(2) - f.write('%s: %s\n' % (key.rjust(30), val)) - -def print_list(l, verbose=False, f=stdout): + if detail: + patterns = ['^x_(account|container|object)_meta_(\w+)$'] + patterns.append(patterns[0].replace('_', '-')) + for key, val in sorted(d.items()): + for p in patterns: + p = re.compile(p) + m = p.match(key) + if m: + key = m.group(2) + f.write('%s: %s\n' % (key.rjust(30), val)) + +def print_list(l, verbose=False, f=stdout, detail=True): for elem in l: #if it's empty string continue if not elem: continue if type(elem) == types.DictionaryType: - print_dict(elem, f=f) + print_dict(elem, f=f, detail=detail) elif type(elem) == types.StringType: if not verbose: elem = elem.split('Traceback')[0] @@ -478,6 +697,19 @@ def print_versions(data, f=stdout): for id, t in data['versions']: f.write('%s @ %s\n' % (str(id).rjust(30), datetime.fromtimestamp(t))) +def _get_user(): + try: + return os.environ['PITHOS_USER'] + except KeyError: + return getuser() + +def _get_auth(): + try: + return os.environ['PITHOS_AUTH'] + except KeyError: + return '0000' + + def main(): try: name = argv[1] @@ -486,17 +718,16 @@ def main(): print_usage() exit(1) - cmd = cls(argv[2:]) + cmd = cls(name, argv[2:]) try: cmd.execute(*cmd.args) except TypeError, e: - print e - cmd.parser.usage = '%%prog %s [options] %s' % (name, cmd.syntax) cmd.parser.print_help() exit(1) except Fault, f: - print f.status, f.data + status = f.status and '%s ' % f.status or '' + print '%s%s' % (status, f.data) if __name__ == '__main__': main()