X-Git-Url: https://code.grnet.gr/git/pithos/blobdiff_plain/e5cd3e33e8600824426f763cde43627d1d505cb0..b980169d8828d5f42afaea95488ca2ba7436b21e:/tools/store diff --git a/tools/store b/tools/store index 3ff526e..316ae52 100755 --- a/tools/store +++ b/tools/store @@ -1,14 +1,51 @@ #!/usr/bin/env python +# Copyright 2011 GRNET S.A. All rights reserved. +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# 1. Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# 2. Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials +# provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS +# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# The views and conclusions contained in the software and +# documentation are those of the authors and should not be +# interpreted as representing official policies, either expressed +# or implied, of GRNET S.A. + from getpass import getuser from optparse import OptionParser -from os.path import basename +from os import environ from sys import argv, exit, stdin, stdout from pithos.lib.client import Client, Fault +from datetime import datetime import json import logging import types +import re +import time as _time +import os DEFAULT_HOST = 'pithos.dev.grnet.gr' DEFAULT_API = 'v1' @@ -27,12 +64,18 @@ def class_for_cli_command(name): return _cli_commands[name] class Command(object): - def __init__(self, argv): - parser = OptionParser() + syntax = '' + + def __init__(self, name, argv): + parser = OptionParser('%%prog %s [options] %s' % (name, self.syntax)) parser.add_option('--host', dest='host', metavar='HOST', default=DEFAULT_HOST, help='use server HOST') parser.add_option('--user', dest='user', metavar='USERNAME', - default=getuser(), help='use account USERNAME') + default=_get_user(), + help='use account USERNAME') + parser.add_option('--token', dest='token', metavar='AUTH', + default=_get_auth(), + help='use account AUTH') parser.add_option('--api', dest='api', metavar='API', default=DEFAULT_API, help='use api API') parser.add_option('-v', action='store_true', dest='verbose', @@ -49,15 +92,15 @@ class Command(object): val = getattr(options, key) setattr(self, key, val) - self.client = Client(self.host, self.user, self.api, self.verbose, + self.client = Client(self.host, self.token, self.user, self.api, self.verbose, self.debug) self.parser = parser self.args = args - + def add_options(self, parser): pass - + def execute(self, *args): pass @@ -92,7 +135,11 @@ class List(Command): parser.add_option('--if-unmodified-since', action='store', type='str', dest='if_unmodified_since', default=None, help='show output if not modified since then') - + parser.add_option('--until', action='store', dest='until', + default=False, help='show metadata until that date') + parser.add_option('--format', action='store', dest='format', + default='%d/%m/%Y', help='format to parse until date') + def execute(self, container=None): if container: self.list_objects(container) @@ -103,31 +150,69 @@ class List(Command): params = {'limit':self.limit, 'marker':self.marker} headers = {'IF_MODIFIED_SINCE':self.if_modified_since, 'IF_UNMODIFIED_SINCE':self.if_unmodified_since} + + if self.until: + t = _time.strptime(self.until, self.format) + params['until'] = int(_time.mktime(t)) + l = self.client.list_containers(self.detail, params, headers) print_list(l) - + def list_objects(self, container): - params = {'limit':self.limit, 'marker':self.marker, - 'prefix':self.prefix, 'delimiter':self.delimiter, - 'path':self.path, 'meta':self.meta} + #prepate params + params = {} + attrs = ['limit', 'marker', 'prefix', 'delimiter', 'path', 'meta'] + for a in [a for a in attrs if getattr(self, a)]: + params[a] = getattr(self, a) + + if self.until: + t = _time.strptime(self.until, self.format) + params['until'] = int(_time.mktime(t)) + headers = {'IF_MODIFIED_SINCE':self.if_modified_since, 'IF_UNMODIFIED_SINCE':self.if_unmodified_since} - l = self.client.list_objects(container, self.detail, params, headers) - print_list(l) + container, sep, object = container.partition('/') + if object: + return + + detail = 'json' + #if request with meta quering disable trash filtering + show_trashed = True if self.meta else False + l = self.client.list_objects(container, detail, headers, + include_trashed = show_trashed, **params) + print_list(l, detail=self.detail) @cli_command('meta') class Meta(Command): syntax = '[[/]]' description = 'get the metadata of an account, a container or an object' - + + def add_options(self, parser): + parser.add_option('-r', action='store_true', dest='restricted', + default=False, help='show only user defined metadata') + parser.add_option('--until', action='store', dest='until', + default=False, help='show metadata until that date') + parser.add_option('--format', action='store', dest='format', + default='%d/%m/%Y', help='format to parse until date') + parser.add_option('--version', action='store', dest='version', + default=None, help='show specific version \ + (applies only for objects)') + def execute(self, path=''): container, sep, object = path.partition('/') + if self.until: + t = _time.strptime(self.until, self.format) + self.until = int(_time.mktime(t)) if object: - meta = self.client.retrieve_object_metadata(container, object) + meta = self.client.retrieve_object_metadata(container, object, + self.restricted, + self.version) elif container: - meta = self.client.retrieve_container_metadata(container) + meta = self.client.retrieve_container_metadata(container, + self.restricted, + self.until) else: - meta = self.client.account_metadata() + meta = self.client.account_metadata(self.restricted, self.until) if meta == None: print 'Entity does not exist' else: @@ -140,10 +225,11 @@ class CreateContainer(Command): def execute(self, container, *args): headers = {} + meta = {} for arg in args: key, sep, val = arg.partition('=') - headers['X_CONTAINER_META_%s' %key.strip().upper()] = val.strip() - ret = self.client.create_container(container, headers) + meta[key] = val + ret = self.client.create_container(container, headers, **meta) if not ret: print 'Container already exists' @@ -169,6 +255,8 @@ class GetObject(Command): default=False, help='show detailed output') parser.add_option('--range', action='store', dest='range', default=None, help='show range of data') + parser.add_option('--if-range', action='store', dest='if-range', + default=None, help='show range of data') parser.add_option('--if-match', action='store', dest='if-match', default=None, help='show output if ETags match') parser.add_option('--if-none-match', action='store', @@ -180,40 +268,62 @@ class GetObject(Command): parser.add_option('--if-unmodified-since', action='store', type='str', dest='if-unmodified-since', default=None, help='show output if not modified since then') - parser.add_option('-f', action='store', type='str', + parser.add_option('-o', action='store', type='str', dest='file', default=None, help='save output in file') - + parser.add_option('--version', action='store', type='str', + dest='version', default=None, + help='get the specific \ + version') + parser.add_option('--versionlist', action='store_true', + dest='versionlist', default=False, + help='get the full object version list') + def execute(self, path): headers = {} if self.range: headers['RANGE'] = 'bytes=%s' %self.range + if getattr(self, 'if-range'): + headers['IF_RANGE'] = 'If-Range:%s' % getattr(self, 'if-range') attrs = ['if-match', 'if-none-match', 'if-modified-since', 'if-unmodified-since'] attrs = [a for a in attrs if getattr(self, a)] for a in attrs: headers[a.replace('-', '_').upper()] = getattr(self, a) container, sep, object = path.partition('/') + if self.versionlist: + self.version = 'list' + self.detail = True data = self.client.retrieve_object(container, object, self.detail, - headers) - if self.file: - if self.detail: - f = self.file and open(self.file, 'w') or stdout - data = json.loads(data) - print_dict(data, f=f) + headers, self.version) + f = self.file and open(self.file, 'w') or stdout + if self.detail: + data = json.loads(data) + if self.versionlist: + print_versions(data, f=f) else: - fw = open(self.file, 'w') - fw.write(data) - fw.close() + print_dict(data, f=f) else: - print data + f.write(data) + f.close() + +@cli_command('mkdir') +class PutMarker(Command): + syntax = '/' + description = 'create a directory marker' + + def execute(self, path): + container, sep, object = path.partition('/') + self.client.create_directory_marker(container, object) @cli_command('put') class PutObject(Command): syntax = '/ [key=val] [...]' - description = 'create/override object with path contents or standard input' - + description = 'create/override object' + def add_options(self, parser): + parser.add_option('--use_hashes', action='store_true', dest='use_hashes', + default=False, help='provide hashmap instead of data') parser.add_option('--chunked', action='store_true', dest='chunked', default=False, help='set chunked transfer mode') parser.add_option('--etag', action='store', dest='etag', @@ -224,37 +334,80 @@ class PutObject(Command): parser.add_option('--content-disposition', action='store', type='str', dest='content-disposition', default=None, help='provide the presentation style of the object') - parser.add_option('--manifest', action='store', type='str', - dest='manifest', default=None, + parser.add_option('-S', action='store', + dest='segment-size', default=False, help='use for large file support') - - def execute(self, path, srcpath, *args): - headers = {} - if self.manifest: - headers['X_OBJECT_MANIFEST'] = self.manifest - - attrs = ['etag', 'content-encoding', 'content-disposition'] - attrs = [a for a in attrs if getattr(self, a)] - for a in attrs: - headers[a.replace('-', '_').upper()] = getattr(self, a) + parser.add_option('--manifest', action='store_true', + dest='manifest', default=None, + help='upload a manifestation file') + parser.add_option('--type', action='store', + dest='content-type', default=False, + help='create object with specific content type') + parser.add_option('--sharing', action='store', + dest='sharing', default=None, + help='define sharing object policy') + parser.add_option('-f', action='store', + dest='srcpath', default=None, + help='file descriptor to read from (pass - for standard input)') + parser.add_option('--public', action='store', + dest='public', default=None, + help='make object publicly accessible (\'True\'/\'False\')') + + def execute(self, path, *args): + if path.find('=') != -1: + raise Fault('Missing path argument') #prepare user defined meta + meta = {} for arg in args: key, sep, val = arg.partition('=') - headers['X_OBJECT_META_%s' %key.strip().upper()] = val.strip() + meta[key] = val + + headers = {} + manifest = getattr(self, 'manifest') + if manifest: + # if it's manifestation file + # send zero-byte data with X-Object-Manifest header + self.touch = True + headers['X_OBJECT_MANIFEST'] = manifest + if self.sharing: + headers['X_OBJECT_SHARING'] = self.sharing + + attrs = ['etag', 'content-encoding', 'content-disposition', + 'content-type'] + attrs = [a for a in attrs if getattr(self, a)] + for a in attrs: + headers[a.replace('-', '_').upper()] = getattr(self, a) container, sep, object = path.partition('/') - f = srcpath != '-' and open(srcpath) or stdin - chunked = (self.chunked or f == stdin) and True or False - self.client.create_object(container, object, f, chunked=chunked, - headers=headers) - f.close() + f = None + if self.srcpath: + f = open(self.srcpath) if self.srcpath != '-' else stdin + + if self.use_hashes and not f: + raise Fault('Illegal option combination') + if self.public not in ['True', 'False', None]: + raise Fault('Not acceptable value for public') + public = eval(self.public) if self.public else None + self.client.create_object(container, object, f, chunked=self.chunked, + headers=headers, use_hashes=self.use_hashes, + public=public, **meta) + if f: + f.close() @cli_command('copy', 'cp') class CopyObject(Command): syntax = '/ [/]' - description = 'copies an object to a different location' + description = 'copy an object to a different location' + + def add_options(self, parser): + parser.add_option('--version', action='store', + dest='version', default=False, + help='copy specific version') + parser.add_option('--public', action='store', + dest='public', default=None, + help='publish/unpublish object (\'True\'/\'False\')') def execute(self, src, dst): src_container, sep, src_object = src.partition('/') @@ -262,8 +415,17 @@ class CopyObject(Command): if not sep: dst_container = src_container dst_object = dst + version = getattr(self, 'version') + if version: + headers = {} + headers['X_SOURCE_VERSION'] = version + if self.public and self.nopublic: + raise Fault('Conflicting options') + if self.public not in ['True', 'False', None]: + raise Fault('Not acceptable value for public') + public = eval(self.public) if self.public else None self.client.copy_object(src_container, src_object, dst_container, - dst_object) + dst_object, public, headers) @cli_command('set') class SetMeta(Command): @@ -292,14 +454,14 @@ class SetMeta(Command): @cli_command('update') class UpdateObject(Command): syntax = '/ path [key=val] [...]' - description = 'update object metadata/data' + description = 'update object metadata/data (default mode: append)' def add_options(self, parser): parser.add_option('-a', action='store_true', dest='append', - default=None, help='append data') - parser.add_option('--start', action='store', - dest='start', - default=None, help='range of data to be updated') + default=True, help='append data') + parser.add_option('--offset', action='store', + dest='offset', + default=None, help='starting offest to be updated') parser.add_option('--range', action='store', dest='content-range', default=None, help='range of data to be updated') parser.add_option('--chunked', action='store_true', dest='chunked', @@ -312,17 +474,32 @@ class UpdateObject(Command): help='provide the presentation style of the object') parser.add_option('--manifest', action='store', type='str', dest='manifest', default=None, - help='use for large file support') - - def execute(self, path, srcpath, *args): + help='use for large file support') + parser.add_option('--sharing', action='store', + dest='sharing', default=None, + help='define sharing object policy') + parser.add_option('--nosharing', action='store_true', + dest='no_sharing', default=None, + help='clear object sharing policy') + parser.add_option('-f', action='store', + dest='srcpath', default=None, + help='file descriptor to read from: pass - for standard input') + parser.add_option('--public', action='store', + dest='public', default=None, + help='publish/unpublish object (\'True\'/\'False\')') + + def execute(self, path, *args): + if path.find('=') != -1: + raise Fault('Missing path argument') + headers = {} if self.manifest: headers['X_OBJECT_MANIFEST'] = self.manifest + if self.sharing: + headers['X_OBJECT_SHARING'] = self.sharing - if self.append: - headers['CONTENT_RANGE'] = 'bytes */*' - elif self.start: - headers['CONTENT_RANGE'] = 'bytes %s-/*' % self.first-byte-pos + if self.no_sharing: + headers['X_OBJECT_SHARING'] = '' attrs = ['content-encoding', 'content-disposition'] attrs = [a for a in attrs if getattr(self, a)] @@ -330,22 +507,37 @@ class UpdateObject(Command): headers[a.replace('-', '_').upper()] = getattr(self, a) #prepare user defined meta + meta = {} for arg in args: key, sep, val = arg.partition('=') - headers['X_OBJECT_META_%s' %key.strip().upper()] = val.strip() + meta[key] = val container, sep, object = path.partition('/') - f = srcpath != '-' and open(srcpath) or stdin - chunked = (self.chunked or f == stdin) and True or False + f = None + chunked = False + if self.srcpath: + f = self.srcpath != '-' and open(self.srcpath) or stdin + if f: + chunked = True if (self.chunked or f == stdin) else False + if self.public not in ['True', 'False', None]: + raise Fault('Not acceptable value for public') + public = eval(self.public) if self.public else None self.client.update_object(container, object, f, chunked=chunked, - headers=headers) - f.close() + headers=headers, offset=self.offset, + public=public, **meta) + if f: + f.close() @cli_command('move', 'mv') class MoveObject(Command): syntax = '/ [/]' - description = 'moves an object to a different location' + description = 'move an object to a different location' + + def add_options(self, parser): + parser.add_option('--public', action='store', + dest='public', default=None, + help='publish/unpublish object (\'True\'/\'False\')') def execute(self, src, dst): src_container, sep, src_object = src.partition('/') @@ -353,11 +545,110 @@ class MoveObject(Command): if not sep: dst_container = src_container dst_object = dst + if self.public not in ['True', 'False', None]: + raise Fault('Not acceptable value for public') + public = eval(self.public) if self.public else None self.client.move_object(src_container, src_object, dst_container, - dst_object) + dst_object, public, headers) + +@cli_command('remove') +class TrashObject(Command): + syntax = '/' + description = 'trash an object' + + def execute(self, src): + src_container, sep, src_object = src.partition('/') + + self.client.trash_object(src_container, src_object) + +@cli_command('restore') +class RestoreObject(Command): + syntax = '/' + description = 'restore a trashed object' + + def execute(self, src): + src_container, sep, src_object = src.partition('/') + + self.client.restore_object(src_container, src_object) + +@cli_command('unset') +class UnsetObject(Command): + syntax = '/[] key [key] [...]' + description = 'delete metadata info' + + def execute(self, path, *args): + #in case of account fix the args + if len(args) == 0: + args = list(args) + args.append(path) + args = tuple(args) + path = '' + meta = [] + for key in args: + meta.append(key) + container, sep, object = path.partition('/') + if object: + self.client.delete_object_metadata(container, object, meta) + elif container: + self.client.delete_container_metadata(container, meta) + else: + self.client.delete_account_metadata(meta) + +@cli_command('group') +class SetGroup(Command): + syntax = 'key=val [key=val] [...]' + description = 'set group account info' + + def execute(self, *args): + groups = {} + for arg in args: + key, sep, val = arg.partition('=') + groups[key] = val + self.client.set_account_groups(**groups) + +@cli_command('policy') +class SetPolicy(Command): + syntax = 'container key=val [key=val] [...]' + description = 'set container policies' + + def execute(self, path, *args): + if path.find('=') != -1: + raise Fault('Missing container argument') + + container, sep, object = path.partition('/') + + if object: + raise Fault('Only containers have policies') + + policies = {} + for arg in args: + key, sep, val = arg.partition('=') + policies[key] = val + + self.client.set_container_policies(container, **policies) + +@cli_command('publish') +class PublishObject(Command): + syntax = '/' + description = 'publish an object' + + def execute(self, src): + src_container, sep, src_object = src.partition('/') + + self.client.publish_object(src_container, src_object) + +@cli_command('unpublish') +class UnpublishObject(Command): + syntax = '/' + description = 'unpublish an object' + + def execute(self, src): + src_container, sep, src_object = src.partition('/') + + self.client.unpublish_object(src_container, src_object) def print_usage(): - cmd = Command([]) + cmd = Command('', []) parser = cmd.parser parser.usage = '%prog [options]' parser.print_help() @@ -369,20 +660,28 @@ def print_usage(): commands.append(' %s %s' % (name.ljust(12), description)) print '\nCommands:\n' + '\n'.join(sorted(commands)) -def print_dict(d, header='name', f=stdout): +def print_dict(d, header='name', f=stdout, detail=True): header = header in d and header or 'subdir' if header and header in d: f.write('%s\n' %d.pop(header)) - for key, val in sorted(d.items()): - f.write('%s: %s\n' % (key.rjust(15), val)) + if detail: + patterns = ['^x_(account|container|object)_meta_(\w+)$'] + patterns.append(patterns[0].replace('_', '-')) + for key, val in sorted(d.items()): + for p in patterns: + p = re.compile(p) + m = p.match(key) + if m: + key = m.group(2) + f.write('%s: %s\n' % (key.rjust(30), val)) -def print_list(l, verbose=False, f=stdout): +def print_list(l, verbose=False, f=stdout, detail=True): for elem in l: #if it's empty string continue if not elem: continue if type(elem) == types.DictionaryType: - print_dict(elem, f=f) + print_dict(elem, f=f, detail=detail) elif type(elem) == types.StringType: if not verbose: elem = elem.split('Traceback')[0] @@ -390,6 +689,27 @@ def print_list(l, verbose=False, f=stdout): else: f.write('%s\n' % elem) +def print_versions(data, f=stdout): + if 'versions' not in data: + f.write('%s\n' %data) + return + f.write('versions:\n') + for id, t in data['versions']: + f.write('%s @ %s\n' % (str(id).rjust(30), datetime.fromtimestamp(t))) + +def _get_user(): + try: + return os.environ['PITHOS_USER'] + except KeyError: + return getuser() + +def _get_auth(): + try: + return os.environ['PITHOS_AUTH'] + except KeyError: + return '0000' + + def main(): try: name = argv[1] @@ -398,16 +718,16 @@ def main(): print_usage() exit(1) - cmd = cls(argv[2:]) + cmd = cls(name, argv[2:]) try: cmd.execute(*cmd.args) - except TypeError: - cmd.parser.usage = '%%prog %s [options] %s' % (name, cmd.syntax) + except TypeError, e: cmd.parser.print_help() exit(1) except Fault, f: - print f.data + status = f.status and '%s ' % f.status or '' + print '%s%s' % (status, f.data) if __name__ == '__main__': main()