X-Git-Url: https://code.grnet.gr/git/pithos/blobdiff_plain/da62aafb3bc6f83c50a0164f97bb1b2c6681261a..611001c27fe1edb42b1cf3a7ddee4928df1e97a2:/tools/store diff --git a/tools/store b/tools/store index 61ad273..0cd55df 100755 --- a/tools/store +++ b/tools/store @@ -1,160 +1,54 @@ #!/usr/bin/env python +# Copyright 2011 GRNET S.A. All rights reserved. +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# 1. Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# 2. Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials +# provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS +# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# The views and conclusions contained in the software and +# documentation are those of the authors and should not be +# interpreted as representing official policies, either expressed +# or implied, of GRNET S.A. + from getpass import getuser -from httplib import HTTPConnection from optparse import OptionParser from os.path import basename -from sys import argv, exit, stdin +from sys import argv, exit, stdin, stdout +from pithos.lib.client import Client, Fault +from datetime import datetime import json import logging - +import types +import re +import time as _time DEFAULT_HOST = 'pithos.dev.grnet.gr' DEFAULT_API = 'v1' - -class Fault(Exception): - def __init__(self, data=''): - Exception.__init__(self, data) - self.data = data - - -class Client(object): - def __init__(self, host, account, api='v1', verbose=False, debug=False): - """`host` can also include a port, e.g '127.0.0.1:8000'.""" - - self.host = host - self.account = account - self.api = api - self.verbose = verbose or debug - self.debug = debug - - def req(self, method, path, body=None, headers=None, format='text'): - full_path = '/%s/%s%s?format=%s' % (self.api, self.account, path, format) - conn = HTTPConnection(self.host) - - kwargs = {} - kwargs['headers'] = headers or {} - kwargs['headers']['Content-Length'] = len(body) if body else 0 - if body: - kwargs['body'] = body - - conn.request(method, full_path, **kwargs) - resp = conn.getresponse() - headers = dict(resp.getheaders()) - - if self.verbose: - print '%d %s' % (resp.status, resp.reason) - for key, val in headers.items(): - print '%s: %s' % (key.capitalize(), val) - print - - data = resp.read() - - if self.debug: - print data - print - - if format == 'json': - data = json.loads(data) - - return resp.status, headers, data - - def delete(self, path, format='text'): - return self.req('DELETE', path, format=format) - - def get(self, path, format='text'): - return self.req('GET', path, format=format) - - def head(self, path, format='text'): - return self.req('HEAD', path, format=format) - - def post(self, path, body=None, format='text', headers=None): - return self.req('POST', path, body, headers=headers, format=format) - - def put(self, path, body=None, format='text', headers=None): - return self.req('PUT', path, body, headers=headers, format=format) - - def _list(self, path, detail=False): - format = 'json' if detail else 'text' - status, headers, data = self.get(path, format=format) - return data - - def _get_metadata(self, path, prefix): - status, headers, data = self.head(path) - prefixlen = len(prefix) - meta = {} - for key, val in headers.items(): - if key.startswith(prefix): - key = key[prefixlen:] - meta[key] = val - return meta - - - # Storage Account Services - - def list_containers(self, detail=False): - return self._list('', detail) - - def account_metadata(self): - return self._get_metadata('', 'x-account-') - - - # Storage Container Services - - def list_objects(self, container, detail=False): - return self._list('/' + container, detail) - - def create_container(self, container): - status, header, data = self.put('/' + container) - if status == 202: - return False - elif status != 201: - raise Fault(data) - return True - - def delete_container(self, container): - self.delete('/' + container) - - def retrieve_container_metadata(self, container): - return self._get_metadata('/%s' % container, 'x-container-') - - - # Storage Object Services - - def retrieve_object(self, container, object): - path = '/%s/%s' % (container, object) - status, headers, data = self.get(path) - return data - - def create_object(self, container, object, data): - path = '/%s/%s' % (container, object) - self.put(path, data) - - def copy_object(self, src_container, src_object, dst_container, dst_object): - path = '/%s/%s' % (dst_container, dst_object) - headers = {} - headers['X-Copy-From'] = '/%s/%s' % (src_container, src_object) - headers['Content-Length'] = 0 - self.put(path, headers=headers) - - def delete_object(self, container, object): - self.delete('/%s/%s' % (container, object)) - - def retrieve_object_metadata(self, container, object): - path = '/%s/%s' % (container, object) - return self._get_metadata(path, 'x-object-meta-') - - def update_object_metadata(self, container, object, **meta): - path = '/%s/%s' % (container, object) - headers = {} - for key, val in meta.items(): - http_key = 'X-Object-Meta-' + key - headers[http_key] = val - self.post(path, headers=headers) - - _cli_commands = {} def cli_command(*args): @@ -168,26 +62,19 @@ def cli_command(*args): def class_for_cli_command(name): return _cli_commands[name] -def print_dict(d, header='name'): - if header: - print d.pop(header) - for key, val in sorted(d.items()): - print '%s: %s' % (key.rjust(15), val) - - class Command(object): - def __init__(self, argv): - parser = OptionParser() - parser.add_option('--host', dest='host', metavar='HOST', default=DEFAULT_HOST, - help='use server HOST') - parser.add_option('--user', dest='user', metavar='USERNAME', default=getuser(), - help='use account USERNAME') - parser.add_option('--api', dest='api', metavar='API', default=DEFAULT_API, - help='use api API') - parser.add_option('-v', action='store_true', dest='verbose', default=False, - help='use verbose output') - parser.add_option('-d', action='store_true', dest='debug', default=False, - help='use debug output') + def __init__(self, name, argv): + parser = OptionParser('%%prog %s [options] %s' % (name, self.syntax)) + parser.add_option('--host', dest='host', metavar='HOST', + default=DEFAULT_HOST, help='use server HOST') + parser.add_option('--user', dest='user', metavar='USERNAME', + default=getuser(), help='use account USERNAME') + parser.add_option('--api', dest='api', metavar='API', + default=DEFAULT_API, help='use api API') + parser.add_option('-v', action='store_true', dest='verbose', + default=False, help='use verbose output') + parser.add_option('-d', action='store_true', dest='debug', + default=False, help='use debug output') self.add_options(parser) options, args = parser.parse_args(argv) @@ -198,26 +85,53 @@ class Command(object): val = getattr(options, key) setattr(self, key, val) - self.client = Client(self.host, self.user, self.api, self.verbose, self.debug) + self.client = Client(self.host, self.user, self.api, self.verbose, + self.debug) self.parser = parser self.args = args - + def add_options(self, parser): pass - + def execute(self, *args): pass - @cli_command('list', 'ls') class List(Command): - syntax = '[container]' + syntax = '[[/]]' description = 'list containers or objects' def add_options(self, parser): - parser.add_option('-l', action='store_true', dest='detail', default=False, - help='show detailed output') + parser.add_option('-l', action='store_true', dest='detail', + default=False, help='show detailed output') + parser.add_option('-n', action='store', type='int', dest='limit', + default=1000, help='show limited output') + parser.add_option('--marker', action='store', type='str', + dest='marker', default=None, + help='show output greater then marker') + parser.add_option('--prefix', action='store', type='str', + dest='prefix', default=None, + help='show output starting with prefix') + parser.add_option('--delimiter', action='store', type='str', + dest='delimiter', default=None, + help='show output up to the delimiter') + parser.add_option('--path', action='store', type='str', + dest='path', default=None, + help='show output starting with prefix up to /') + parser.add_option('--meta', action='store', type='str', + dest='meta', default=None, + help='show output having the specified meta keys') + parser.add_option('--if-modified-since', action='store', type='str', + dest='if_modified_since', default=None, + help='show output if modified since then') + parser.add_option('--if-unmodified-since', action='store', type='str', + dest='if_unmodified_since', default=None, + help='show output if not modified since then') + parser.add_option('--until', action='store', dest='until', + default=False, help='show metadata until that date') + parser.add_option('--format', action='store', dest='format', + default='%d/%m/%Y', help='format to parse until date') def execute(self, container=None): if container: @@ -226,49 +140,88 @@ class List(Command): self.list_containers() def list_containers(self): - if self.detail: - for container in self.client.list_containers(detail=True): - print_dict(container) - print - else: - print self.client.list_containers().strip() + params = {'limit':self.limit, 'marker':self.marker} + headers = {'IF_MODIFIED_SINCE':self.if_modified_since, + 'IF_UNMODIFIED_SINCE':self.if_unmodified_since} + + if self.until: + t = _time.strptime(self.until, self.format) + params['until'] = int(_time.mktime(t)) + + l = self.client.list_containers(self.detail, params, headers) + print_list(l) def list_objects(self, container): - if self.detail: - for obj in self.client.list_objects(container, detail=True): - print_dict(obj) - print - else: - print self.client.list_objects(container).strip() - + params = {'limit':self.limit, 'marker':self.marker, + 'prefix':self.prefix, 'delimiter':self.delimiter, + 'path':self.path, 'meta':self.meta} + headers = {'IF_MODIFIED_SINCE':self.if_modified_since, + 'IF_UNMODIFIED_SINCE':self.if_unmodified_since} + container, sep, object = container.partition('/') + if object: + return + + if self.until: + t = _time.strptime(self.until, self.format) + params['until'] = int(_time.mktime(t)) + + detail = 'json' + #if request with meta quering disable trash filtering + show_trashed = True if self.meta else False + l = self.client.list_objects(container, detail, params, headers, + include_trashed = show_trashed) + print_list(l, detail=self.detail) @cli_command('meta') class Meta(Command): syntax = '[[/]]' description = 'get the metadata of an account, a container or an object' - + + def add_options(self, parser): + parser.add_option('-r', action='store_true', dest='restricted', + default=False, help='show only user defined metadata') + parser.add_option('--until', action='store', dest='until', + default=False, help='show metadata until that date') + parser.add_option('--format', action='store', dest='format', + default='%d/%m/%Y', help='format to parse until date') + parser.add_option('--version', action='store', dest='version', + default=None, help='show specific version \ + (applies only for objects)') + def execute(self, path=''): container, sep, object = path.partition('/') + if self.until: + t = _time.strptime(self.until, self.format) + self.until = int(_time.mktime(t)) if object: - meta = self.client.retrieve_object_metadata(container, object) + meta = self.client.retrieve_object_metadata(container, object, + self.restricted, + self.version) elif container: - meta = self.client.retrieve_container_metadata(container) + meta = self.client.retrieve_container_metadata(container, + self.restricted, + self.until) else: - meta = self.client.account_metadata() - print_dict(meta, header=None) - + meta = self.client.account_metadata(self.restricted, self.until) + if meta == None: + print 'Entity does not exist' + else: + print_dict(meta, header=None) @cli_command('create') class CreateContainer(Command): - syntax = '' + syntax = ' [key=val] [...]' description = 'create a container' - def execute(self, container): - ret = self.client.create_container(container) + def execute(self, container, *args): + headers = {} + for arg in args: + key, sep, val = arg.partition('=') + headers['X_CONTAINER_META_%s' %key.strip().upper()] = val.strip() + ret = self.client.create_container(container, headers) if not ret: print 'Container already exists' - @cli_command('delete', 'rm') class Delete(Command): syntax = '[/]' @@ -281,57 +234,323 @@ class Delete(Command): else: self.client.delete_container(container) - @cli_command('get') class GetObject(Command): syntax = '/' description = 'get the data of an object' + def add_options(self, parser): + parser.add_option('-l', action='store_true', dest='detail', + default=False, help='show detailed output') + parser.add_option('--range', action='store', dest='range', + default=None, help='show range of data') + parser.add_option('--if-match', action='store', dest='if-match', + default=None, help='show output if ETags match') + parser.add_option('--if-none-match', action='store', + dest='if-none-match', default=None, + help='show output if ETags don\'t match') + parser.add_option('--if-modified-since', action='store', type='str', + dest='if-modified-since', default=None, + help='show output if modified since then') + parser.add_option('--if-unmodified-since', action='store', type='str', + dest='if-unmodified-since', default=None, + help='show output if not modified since then') + parser.add_option('-f', action='store', type='str', + dest='file', default=None, + help='save output in file') + parser.add_option('--version', action='store', type='str', + dest='version', default=None, + help='get the specific \ + version') + parser.add_option('--versionlist', action='store_true', + dest='versionlist', default=False, + help='get the full object version list') + def execute(self, path): + headers = {} + if self.range: + headers['RANGE'] = 'bytes=%s' %self.range + attrs = ['if-match', 'if-none-match', 'if-modified-since', + 'if-unmodified-since'] + attrs = [a for a in attrs if getattr(self, a)] + for a in attrs: + headers[a.replace('-', '_').upper()] = getattr(self, a) container, sep, object = path.partition('/') - print self.client.retrieve_object(container, object) + if self.versionlist: + self.version = 'list' + self.detail = True + data = self.client.retrieve_object(container, object, self.detail, + headers, self.version) + f = self.file and open(self.file, 'w') or stdout + if self.detail: + data = json.loads(data) + if self.versionlist: + print_versions(data, f=f) + else: + print_dict(data, f=f) + else: + f.write(data) + f.close() +@cli_command('mkdir') +class PutMarker(Command): + syntax = '/' + description = 'create a directory marker' + + def execute(self, path): + container, sep, object = path.partition('/') + self.client.create_directory_marker(container, object) @cli_command('put') class PutObject(Command): - syntax = '/ ' - description = 'create or update an object with contents of path' - - def execute(self, path, srcpath): + syntax = '/ [key=val] [...]' + description = 'create/override object with path contents or standard input' + + def add_options(self, parser): + parser.add_option('--chunked', action='store_true', dest='chunked', + default=False, help='set chunked transfer mode') + parser.add_option('--etag', action='store', dest='etag', + default=None, help='check written data') + parser.add_option('--content-encoding', action='store', + dest='content-encoding', default=None, + help='provide the object MIME content type') + parser.add_option('--content-disposition', action='store', type='str', + dest='content-disposition', default=None, + help='provide the presentation style of the object') + parser.add_option('-S', action='store', + dest='segment-size', default=False, + help='use for large file support') + parser.add_option('--manifest', action='store_true', + dest='manifest', default=None, + help='upload a manifestation file') + parser.add_option('--type', action='store', + dest='content-type', default=False, + help='create object with specific content type') + parser.add_option('--touch', action='store_true', + dest='touch', default=False, + help='create object with zero data') + parser.add_option('--sharing', action='store', + dest='sharing', default=None, + help='define sharing object policy') + + def execute(self, path, srcpath='-', *args): + headers = {} + if self.manifest: + headers['X_OBJECT_MANIFEST'] = self.manifest + if self.sharing: + headers['X_OBJECT_SHARING'] = self.sharing + + attrs = ['etag', 'content-encoding', 'content-disposition', + 'content-type'] + attrs = [a for a in attrs if getattr(self, a)] + for a in attrs: + headers[a.replace('-', '_').upper()] = getattr(self, a) + + #prepare user defined meta + for arg in args: + key, sep, val = arg.partition('=') + headers['X_OBJECT_META_%s' %key.strip().upper()] = val.strip() + container, sep, object = path.partition('/') - f = open(srcpath) if srcpath != '-' else stdin - data = f.read() - self.client.create_object(container, object, data) - f.close() - + + f = None + chunked = False + if not self.touch: + f = srcpath != '-' and open(srcpath) or stdin + chunked = (self.chunked or f == stdin) and True or False + self.client.create_object(container, object, f, chunked=chunked, + headers=headers) + if f: + f.close() @cli_command('copy', 'cp') class CopyObject(Command): syntax = '/ [/]' description = 'copies an object to a different location' + def add_options(self, parser): + parser.add_option('--version', action='store', + dest='version', default=False, + help='copy specific version') + def execute(self, src, dst): src_container, sep, src_object = src.partition('/') dst_container, sep, dst_object = dst.partition('/') if not sep: dst_container = src_container dst_object = dst - self.client.copy_object(src_container, src_object, dst_container, dst_object) - + version = getattr(self, 'version') + if version: + headers = {} + headers['X_SOURCE_VERSION'] = version + self.client.copy_object(src_container, src_object, dst_container, + dst_object, headers) @cli_command('set') -class SetOjectMeta(Command): - syntax = '/ key=val [key=val] [...]' - description = 'set object metadata' +class SetMeta(Command): + syntax = '[[/]] key=val [key=val] [...]' + description = 'set metadata' def execute(self, path, *args): - container, sep, object = path.partition('/') + #in case of account fix the args + if path.find('=') != -1: + args = list(args) + args.append(path) + args = tuple(args) + path = '' meta = {} for arg in args: key, sep, val = arg.partition('=') meta[key.strip()] = val.strip() - self.client.update_object_metadata(container, object, **meta) + container, sep, object = path.partition('/') + if object: + self.client.update_object_metadata(container, object, **meta) + elif container: + self.client.update_container_metadata(container, **meta) + else: + self.client.update_account_metadata(**meta) +@cli_command('update') +class UpdateObject(Command): + syntax = '/ path [key=val] [...]' + description = 'update object metadata/data (default mode: append)' + + def add_options(self, parser): + parser.add_option('-a', action='store_true', dest='append', + default=True, help='append data') + parser.add_option('--start', action='store', + dest='start', + default=None, help='starting offest to be updated') + parser.add_option('--range', action='store', dest='content-range', + default=None, help='range of data to be updated') + parser.add_option('--chunked', action='store_true', dest='chunked', + default=False, help='set chunked transfer mode') + parser.add_option('--content-encoding', action='store', + dest='content-encoding', default=None, + help='provide the object MIME content type') + parser.add_option('--content-disposition', action='store', type='str', + dest='content-disposition', default=None, + help='provide the presentation style of the object') + parser.add_option('--manifest', action='store', type='str', + dest='manifest', default=None, + help='use for large file support') + parser.add_option('--sharing', action='store', + dest='sharing', default=None, + help='define sharing object policy') + parser.add_option('--touch', action='store_true', + dest='touch', default=False, + help='change file properties') + + + + def execute(self, path, srcpath='-', *args): + headers = {} + if self.manifest: + headers['X_OBJECT_MANIFEST'] = self.manifest + if self.sharing: + headers['X_OBJECT_SHARING'] = self.sharing + + if getattr(self, 'start'): + headers['CONTENT_RANGE'] = 'bytes %s-/*' % getattr(self, 'start') + elif self.append: + headers['CONTENT_RANGE'] = 'bytes */*' + + attrs = ['content-encoding', 'content-disposition'] + attrs = [a for a in attrs if getattr(self, a)] + for a in attrs: + headers[a.replace('-', '_').upper()] = getattr(self, a) + + #prepare user defined meta + for arg in args: + key, sep, val = arg.partition('=') + headers['X_OBJECT_META_%s' %key.strip().upper()] = val.strip() + + container, sep, object = path.partition('/') + + f = None + chunked = False + if not self.touch: + f = srcpath != '-' and open(srcpath) or stdin + chunked = (self.chunked or f == stdin) and True or False + self.client.update_object(container, object, f, chunked=chunked, + headers=headers) + if f: + f.close() + +@cli_command('move', 'mv') +class MoveObject(Command): + syntax = '/ [/]' + description = 'moves an object to a different location' + + def execute(self, src, dst): + src_container, sep, src_object = src.partition('/') + dst_container, sep, dst_object = dst.partition('/') + if not sep: + dst_container = src_container + dst_object = dst + + self.client.move_object(src_container, src_object, dst_container, + dst_object, headers) + +@cli_command('remove') +class TrashObject(Command): + syntax = '/' + description = 'trashes an object' + + def execute(self, src): + src_container, sep, src_object = src.partition('/') + + self.client.trash_object(src_container, src_object) + +@cli_command('restore') +class TrashObject(Command): + syntax = '/' + description = 'trashes an object' + + def execute(self, src): + src_container, sep, src_object = src.partition('/') + + self.client.restore_object(src_container, src_object) + +@cli_command('unset') +class TrashObject(Command): + syntax = '/[] key [key] [...]' + description = 'deletes metadata info' + + def execute(self, path, *args): + #in case of account fix the args + if len(args) == 0: + args = list(args) + args.append(path) + args = tuple(args) + path = '' + meta = [] + for key in args: + meta.append(key) + container, sep, object = path.partition('/') + if object: + self.client.delete_object_metadata(container, object, meta) + elif container: + self.client.delete_container_metadata(container, meta) + else: + self.client.delete_account_metadata(meta) + +@cli_command('group') +class SetGroup(Command): + syntax = 'key=val [key=val] [...]' + description = 'sets group account info' + + def execute(self, path='', **args): + if len(args) == 0: + args = list(args) + args.append(path) + args = tuple(args) + path = '' + groups = {} + for arg in args: + key, sep, val = arg.partition('=') + groups[key] = val + self.client.set_account_groups(groups) def print_usage(): cmd = Command([]) @@ -346,7 +565,42 @@ def print_usage(): commands.append(' %s %s' % (name.ljust(12), description)) print '\nCommands:\n' + '\n'.join(sorted(commands)) +def print_dict(d, header='name', f=stdout, detail=True): + header = header in d and header or 'subdir' + if header and header in d: + f.write('%s\n' %d.pop(header)) + if detail: + patterns = ['^x_(account|container|object)_meta_(\w+)$'] + patterns.append(patterns[0].replace('_', '-')) + for key, val in sorted(d.items()): + for p in patterns: + p = re.compile(p) + m = p.match(key) + if m: + key = m.group(2) + f.write('%s: %s\n' % (key.rjust(30), val)) + +def print_list(l, verbose=False, f=stdout, detail=True): + for elem in l: + #if it's empty string continue + if not elem: + continue + if type(elem) == types.DictionaryType: + print_dict(elem, f=f, detail=detail) + elif type(elem) == types.StringType: + if not verbose: + elem = elem.split('Traceback')[0] + f.write('%s\n' % elem) + else: + f.write('%s\n' % elem) +def print_versions(data, f=stdout): + if 'versions' not in data: + f.write('%s\n' %data) + return + f.write('versions:\n') + for id, t in data['versions']: + f.write('%s @ %s\n' % (str(id).rjust(30), datetime.fromtimestamp(t))) def main(): try: @@ -355,18 +609,17 @@ def main(): except (IndexError, KeyError): print_usage() exit(1) - - cmd = cls(argv[2:]) + + cmd = cls(name, argv[2:]) try: cmd.execute(*cmd.args) - except TypeError: - cmd.parser.usage = '%%prog %s [options] %s' % (name, cmd.syntax) + except TypeError, e: cmd.parser.print_help() exit(1) except Fault, f: - print f.data - + status = f.status and '%s ' % f.status or '' + print '%s%s' % (status, f.data) if __name__ == '__main__': main()