#!/usr/bin/env python
+# Copyright 2011 GRNET S.A. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+# 1. Redistributions of source code must retain the above
+# copyright notice, this list of conditions and the following
+# disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials
+# provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and
+# documentation are those of the authors and should not be
+# interpreted as representing official policies, either expressed
+# or implied, of GRNET S.A.
+
from getpass import getuser
from optparse import OptionParser
-from os.path import basename
+from os import environ
from sys import argv, exit, stdin, stdout
from pithos.lib.client import Client, Fault
from datetime import datetime
import types
import re
import time as _time
+import os
DEFAULT_HOST = 'pithos.dev.grnet.gr'
DEFAULT_API = 'v1'
return _cli_commands[name]
class Command(object):
- def __init__(self, argv):
- parser = OptionParser()
+ syntax = ''
+
+ def __init__(self, name, argv):
+ parser = OptionParser('%%prog %s [options] %s' % (name, self.syntax))
parser.add_option('--host', dest='host', metavar='HOST',
default=DEFAULT_HOST, help='use server HOST')
parser.add_option('--user', dest='user', metavar='USERNAME',
- default=getuser(), help='use account USERNAME')
+ default=_get_user(),
+ help='use account USERNAME')
+ parser.add_option('--token', dest='token', metavar='AUTH',
+ default=_get_auth(),
+ help='use account AUTH')
parser.add_option('--api', dest='api', metavar='API',
default=DEFAULT_API, help='use api API')
parser.add_option('-v', action='store_true', dest='verbose',
val = getattr(options, key)
setattr(self, key, val)
- self.client = Client(self.host, self.user, self.api, self.verbose,
+ self.client = Client(self.host, self.token, self.user, self.api, self.verbose,
self.debug)
self.parser = parser
self.args = args
-
+
def add_options(self, parser):
pass
-
+
def execute(self, *args):
pass
default=False, help='show metadata until that date')
parser.add_option('--format', action='store', dest='format',
default='%d/%m/%Y', help='format to parse until date')
-
+
def execute(self, container=None):
if container:
self.list_objects(container)
l = self.client.list_containers(self.detail, params, headers)
print_list(l)
-
+
def list_objects(self, container):
- params = {'limit':self.limit, 'marker':self.marker,
- 'prefix':self.prefix, 'delimiter':self.delimiter,
- 'path':self.path, 'meta':self.meta}
+ #prepate params
+ params = {}
+ attrs = ['limit', 'marker', 'prefix', 'delimiter', 'path', 'meta']
+ for a in [a for a in attrs if getattr(self, a)]:
+ params[a] = getattr(self, a)
+
+ if self.until:
+ t = _time.strptime(self.until, self.format)
+ params['until'] = int(_time.mktime(t))
+
headers = {'IF_MODIFIED_SINCE':self.if_modified_since,
'IF_UNMODIFIED_SINCE':self.if_unmodified_since}
container, sep, object = container.partition('/')
if object:
- print '%s/%s is an object' %(container, object)
return
- if self.until:
- t = _time.strptime(self.until, self.format)
- params['until'] = int(_time.mktime(t))
-
- l = self.client.list_objects(container, self.detail, params, headers)
- print_list(l)
+ detail = 'json'
+ #if request with meta quering disable trash filtering
+ show_trashed = True if self.meta else False
+ l = self.client.list_objects(container, detail, headers,
+ include_trashed = show_trashed, **params)
+ print_list(l, detail=self.detail)
@cli_command('meta')
class Meta(Command):
default=False, help='show metadata until that date')
parser.add_option('--format', action='store', dest='format',
default='%d/%m/%Y', help='format to parse until date')
-
+ parser.add_option('--version', action='store', dest='version',
+ default=None, help='show specific version \
+ (applies only for objects)')
+
def execute(self, path=''):
container, sep, object = path.partition('/')
if self.until:
self.until = int(_time.mktime(t))
if object:
meta = self.client.retrieve_object_metadata(container, object,
- self.restricted)
+ self.restricted,
+ self.version)
elif container:
meta = self.client.retrieve_container_metadata(container,
self.restricted,
def execute(self, container, *args):
headers = {}
+ meta = {}
for arg in args:
key, sep, val = arg.partition('=')
- headers['X_CONTAINER_META_%s' %key.strip().upper()] = val.strip()
- ret = self.client.create_container(container, headers)
+ meta[key] = val
+ ret = self.client.create_container(container, headers, **meta)
if not ret:
print 'Container already exists'
parser.add_option('--if-unmodified-since', action='store', type='str',
dest='if-unmodified-since', default=None,
help='show output if not modified since then')
- parser.add_option('-f', action='store', type='str',
+ parser.add_option('-o', action='store', type='str',
dest='file', default=None,
help='save output in file')
-
+ parser.add_option('--version', action='store', type='str',
+ dest='version', default=None,
+ help='get the specific \
+ version')
+ parser.add_option('--versionlist', action='store_true',
+ dest='versionlist', default=False,
+ help='get the full object version list')
+
def execute(self, path):
headers = {}
if self.range:
for a in attrs:
headers[a.replace('-', '_').upper()] = getattr(self, a)
container, sep, object = path.partition('/')
+ if self.versionlist:
+ self.version = 'list'
+ self.detail = True
data = self.client.retrieve_object(container, object, self.detail,
- headers)
- if self.file:
- if self.detail:
- f = self.file and open(self.file, 'w') or stdout
- data = json.loads(data)
- print_dict(data, f=f)
+ headers, self.version)
+ f = self.file and open(self.file, 'w') or stdout
+ if self.detail:
+ data = json.loads(data)
+ if self.versionlist:
+ print_versions(data, f=f)
else:
- fw = open(self.file, 'w')
- fw.write(data)
- fw.close()
+ print_dict(data, f=f)
else:
- print data
+ f.write(data)
+ f.close()
+
+@cli_command('mkdir')
+class PutMarker(Command):
+ syntax = '<container>/<directory marker>'
+ description = 'create a directory marker'
+
+ def execute(self, path):
+ container, sep, object = path.partition('/')
+ self.client.create_directory_marker(container, object)
@cli_command('put')
class PutObject(Command):
syntax = '<container>/<object> <path> [key=val] [...]'
- description = 'create/override object with path contents or standard input'
-
+ description = 'create/override object'
+
def add_options(self, parser):
+ parser.add_option('--use_hashes', action='store_true', dest='use_hashes',
+ default=False, help='provide hashmap instead of data')
parser.add_option('--chunked', action='store_true', dest='chunked',
default=False, help='set chunked transfer mode')
parser.add_option('--etag', action='store', dest='etag',
parser.add_option('--content-disposition', action='store', type='str',
dest='content-disposition', default=None,
help='provide the presentation style of the object')
- parser.add_option('--manifest', action='store', type='str',
- dest='manifest', default=None,
+ parser.add_option('-S', action='store',
+ dest='segment-size', default=False,
help='use for large file support')
- parser.add_option('--touch', action='store_true',
- dest='touch', default=True,
- help='create object with zero data')
-
- def execute(self, path, srcpath='-', *args):
- headers = {}
- if self.manifest:
- headers['X_OBJECT_MANIFEST'] = self.manifest
-
- attrs = ['etag', 'content-encoding', 'content-disposition']
- attrs = [a for a in attrs if getattr(self, a)]
- for a in attrs:
- headers[a.replace('-', '_').upper()] = getattr(self, a)
+ parser.add_option('--manifest', action='store_true',
+ dest='manifest', default=None,
+ help='upload a manifestation file')
+ parser.add_option('--type', action='store',
+ dest='content-type', default=False,
+ help='create object with specific content type')
+ parser.add_option('--sharing', action='store',
+ dest='sharing', default=None,
+ help='define sharing object policy')
+ parser.add_option('-f', action='store',
+ dest='srcpath', default=None,
+ help='file descriptor to read from (pass - for standard input)')
+ parser.add_option('--public', action='store',
+ dest='public', default=None,
+ help='make object publicly accessible (\'True\'/\'False\')')
+
+ def execute(self, path, *args):
+ if path.find('=') != -1:
+ raise Fault('Missing path argument')
#prepare user defined meta
+ meta = {}
for arg in args:
key, sep, val = arg.partition('=')
- headers['X_OBJECT_META_%s' %key.strip().upper()] = val.strip()
+ meta[key] = val
+
+ headers = {}
+ manifest = getattr(self, 'manifest')
+ if manifest:
+ # if it's manifestation file
+ # send zero-byte data with X-Object-Manifest header
+ self.touch = True
+ headers['X_OBJECT_MANIFEST'] = manifest
+ if self.sharing:
+ headers['X_OBJECT_SHARING'] = self.sharing
+
+ attrs = ['etag', 'content-encoding', 'content-disposition',
+ 'content-type']
+ attrs = [a for a in attrs if getattr(self, a)]
+ for a in attrs:
+ headers[a.replace('-', '_').upper()] = getattr(self, a)
container, sep, object = path.partition('/')
f = None
- chunked = False
- if not self.touch:
- f = srcpath != '-' and open(srcpath) or stdin
- chunked = (self.chunked or f == stdin) and True or False
- self.client.create_object(container, object, f, chunked=chunked,
- headers=headers)
+ if self.srcpath:
+ f = open(self.srcpath) if self.srcpath != '-' else stdin
+
+ if self.use_hashes and not f:
+ raise Fault('Illegal option combination')
+ if self.public not in ['True', 'False', None]:
+ raise Fault('Not acceptable value for public')
+ public = eval(self.public) if self.public else None
+ self.client.create_object(container, object, f, chunked=self.chunked,
+ headers=headers, use_hashes=self.use_hashes,
+ public=public, **meta)
if f:
f.close()
@cli_command('copy', 'cp')
class CopyObject(Command):
syntax = '<src container>/<src object> [<dst container>/]<dst object>'
- description = 'copies an object to a different location'
+ description = 'copy an object to a different location'
+
+ def add_options(self, parser):
+ parser.add_option('--version', action='store',
+ dest='version', default=False,
+ help='copy specific version')
+ parser.add_option('--public', action='store',
+ dest='public', default=None,
+ help='publish/unpublish object (\'True\'/\'False\')')
def execute(self, src, dst):
src_container, sep, src_object = src.partition('/')
if not sep:
dst_container = src_container
dst_object = dst
+ version = getattr(self, 'version')
+ if version:
+ headers = {}
+ headers['X_SOURCE_VERSION'] = version
+ if self.public and self.nopublic:
+ raise Fault('Conflicting options')
+ if self.public not in ['True', 'False', None]:
+ raise Fault('Not acceptable value for public')
+ public = eval(self.public) if self.public else None
self.client.copy_object(src_container, src_object, dst_container,
- dst_object)
+ dst_object, public, headers)
@cli_command('set')
class SetMeta(Command):
@cli_command('update')
class UpdateObject(Command):
syntax = '<container>/<object> path [key=val] [...]'
- description = 'update object metadata/data'
+ description = 'update object metadata/data (default mode: append)'
def add_options(self, parser):
parser.add_option('-a', action='store_true', dest='append',
- default=None, help='append data')
- parser.add_option('--start', action='store',
- dest='start',
- default=None, help='range of data to be updated')
+ default=True, help='append data')
+ parser.add_option('--offset', action='store',
+ dest='offset',
+ default=None, help='starting offest to be updated')
parser.add_option('--range', action='store', dest='content-range',
default=None, help='range of data to be updated')
parser.add_option('--chunked', action='store_true', dest='chunked',
help='provide the presentation style of the object')
parser.add_option('--manifest', action='store', type='str',
dest='manifest', default=None,
- help='use for large file support')
-
- def execute(self, path, srcpath, *args):
+ help='use for large file support')
+ parser.add_option('--sharing', action='store',
+ dest='sharing', default=None,
+ help='define sharing object policy')
+ parser.add_option('--nosharing', action='store_true',
+ dest='no_sharing', default=None,
+ help='clear object sharing policy')
+ parser.add_option('-f', action='store',
+ dest='srcpath', default=None,
+ help='file descriptor to read from: pass - for standard input')
+ parser.add_option('--public', action='store',
+ dest='public', default=None,
+ help='publish/unpublish object (\'True\'/\'False\')')
+
+ def execute(self, path, *args):
+ if path.find('=') != -1:
+ raise Fault('Missing path argument')
+
headers = {}
if self.manifest:
headers['X_OBJECT_MANIFEST'] = self.manifest
+ if self.sharing:
+ headers['X_OBJECT_SHARING'] = self.sharing
- if self.append:
- headers['CONTENT_RANGE'] = 'bytes */*'
- elif self.start:
- headers['CONTENT_RANGE'] = 'bytes %s-/*' % self.first-byte-pos
+ if self.no_sharing:
+ headers['X_OBJECT_SHARING'] = ''
attrs = ['content-encoding', 'content-disposition']
attrs = [a for a in attrs if getattr(self, a)]
headers[a.replace('-', '_').upper()] = getattr(self, a)
#prepare user defined meta
+ meta = {}
for arg in args:
key, sep, val = arg.partition('=')
- headers['X_OBJECT_META_%s' %key.strip().upper()] = val.strip()
+ meta[key] = val
container, sep, object = path.partition('/')
- f = srcpath != '-' and open(srcpath) or stdin
- chunked = (self.chunked or f == stdin) and True or False
+ f = None
+ chunked = False
+ if self.srcpath:
+ f = self.srcpath != '-' and open(self.srcpath) or stdin
+ if f:
+ chunked = True if (self.chunked or f == stdin) else False
+ if self.public not in ['True', 'False', None]:
+ raise Fault('Not acceptable value for public')
+ public = eval(self.public) if self.public else None
self.client.update_object(container, object, f, chunked=chunked,
- headers=headers)
- f.close()
+ headers=headers, offset=self.offset,
+ public=public, **meta)
+ if f:
+ f.close()
@cli_command('move', 'mv')
class MoveObject(Command):
syntax = '<src container>/<src object> [<dst container>/]<dst object>'
- description = 'moves an object to a different location'
+ description = 'move an object to a different location'
+
+ def add_options(self, parser):
+ parser.add_option('--public', action='store',
+ dest='public', default=None,
+ help='publish/unpublish object (\'True\'/\'False\')')
def execute(self, src, dst):
src_container, sep, src_object = src.partition('/')
if not sep:
dst_container = src_container
dst_object = dst
+ if self.public not in ['True', 'False', None]:
+ raise Fault('Not acceptable value for public')
+ public = eval(self.public) if self.public else None
self.client.move_object(src_container, src_object, dst_container,
- dst_object)
+ dst_object, public, headers)
+
+@cli_command('remove')
+class TrashObject(Command):
+ syntax = '<container>/<object>'
+ description = 'trash an object'
+
+ def execute(self, src):
+ src_container, sep, src_object = src.partition('/')
+
+ self.client.trash_object(src_container, src_object)
+
+@cli_command('restore')
+class RestoreObject(Command):
+ syntax = '<container>/<object>'
+ description = 'restore a trashed object'
+
+ def execute(self, src):
+ src_container, sep, src_object = src.partition('/')
+
+ self.client.restore_object(src_container, src_object)
+
+@cli_command('unset')
+class UnsetObject(Command):
+ syntax = '<container>/[<object>] key [key] [...]'
+ description = 'delete metadata info'
+
+ def execute(self, path, *args):
+ #in case of account fix the args
+ if len(args) == 0:
+ args = list(args)
+ args.append(path)
+ args = tuple(args)
+ path = ''
+ meta = []
+ for key in args:
+ meta.append(key)
+ container, sep, object = path.partition('/')
+ if object:
+ self.client.delete_object_metadata(container, object, meta)
+ elif container:
+ self.client.delete_container_metadata(container, meta)
+ else:
+ self.client.delete_account_metadata(meta)
+
+@cli_command('group')
+class SetGroup(Command):
+ syntax = 'key=val [key=val] [...]'
+ description = 'set group account info'
+
+ def execute(self, *args):
+ groups = {}
+ for arg in args:
+ key, sep, val = arg.partition('=')
+ groups[key] = val
+ self.client.set_account_groups(**groups)
+
+@cli_command('policy')
+class SetPolicy(Command):
+ syntax = 'container key=val [key=val] [...]'
+ description = 'set container policies'
+
+ def execute(self, path, *args):
+ if path.find('=') != -1:
+ raise Fault('Missing container argument')
+
+ container, sep, object = path.partition('/')
+
+ if object:
+ raise Fault('Only containers have policies')
+
+ policies = {}
+ for arg in args:
+ key, sep, val = arg.partition('=')
+ policies[key] = val
+
+ self.client.set_container_policies(container, **policies)
+
+@cli_command('publish')
+class PublishObject(Command):
+ syntax = '<container>/<object>'
+ description = 'publish an object'
+
+ def execute(self, src):
+ src_container, sep, src_object = src.partition('/')
+
+ self.client.publish_object(src_container, src_object)
+
+@cli_command('unpublish')
+class UnpublishObject(Command):
+ syntax = '<container>/<object>'
+ description = 'unpublish an object'
+
+ def execute(self, src):
+ src_container, sep, src_object = src.partition('/')
+
+ self.client.unpublish_object(src_container, src_object)
def print_usage():
- cmd = Command([])
+ cmd = Command('', [])
parser = cmd.parser
parser.usage = '%prog <command> [options]'
parser.print_help()
commands.append(' %s %s' % (name.ljust(12), description))
print '\nCommands:\n' + '\n'.join(sorted(commands))
-def print_dict(d, header='name', f=stdout):
+def print_dict(d, header='name', f=stdout, detail=True):
header = header in d and header or 'subdir'
if header and header in d:
f.write('%s\n' %d.pop(header))
- patterns = ['^x_(account|container|object)_meta_(\w+)$']
- patterns.append(patterns[0].replace('_', '-'))
- for key, val in sorted(d.items()):
- for p in patterns:
- p = re.compile(p)
- m = p.match(key)
- if m:
- key = m.group(2)
- f.write('%s: %s\n' % (key.rjust(30), val))
-
-def print_list(l, verbose=False, f=stdout):
+ if detail:
+ patterns = ['^x_(account|container|object)_meta_(\w+)$']
+ patterns.append(patterns[0].replace('_', '-'))
+ for key, val in sorted(d.items()):
+ for p in patterns:
+ p = re.compile(p)
+ m = p.match(key)
+ if m:
+ key = m.group(2)
+ f.write('%s: %s\n' % (key.rjust(30), val))
+
+def print_list(l, verbose=False, f=stdout, detail=True):
for elem in l:
#if it's empty string continue
if not elem:
continue
if type(elem) == types.DictionaryType:
- print_dict(elem, f=f)
+ print_dict(elem, f=f, detail=detail)
elif type(elem) == types.StringType:
if not verbose:
elem = elem.split('Traceback')[0]
else:
f.write('%s\n' % elem)
+def print_versions(data, f=stdout):
+ if 'versions' not in data:
+ f.write('%s\n' %data)
+ return
+ f.write('versions:\n')
+ for id, t in data['versions']:
+ f.write('%s @ %s\n' % (str(id).rjust(30), datetime.fromtimestamp(t)))
+
+def _get_user():
+ try:
+ return os.environ['PITHOS_USER']
+ except KeyError:
+ return getuser()
+
+def _get_auth():
+ try:
+ return os.environ['PITHOS_AUTH']
+ except KeyError:
+ return '0000'
+
+
def main():
try:
name = argv[1]
print_usage()
exit(1)
- cmd = cls(argv[2:])
+ cmd = cls(name, argv[2:])
try:
cmd.execute(*cmd.args)
- except TypeError:
- cmd.parser.usage = '%%prog %s [options] %s' % (name, cmd.syntax)
+ except TypeError, e:
cmd.parser.print_help()
exit(1)
except Fault, f:
- print f.data
+ status = f.status and '%s ' % f.status or ''
+ print '%s%s' % (status, f.data)
if __name__ == '__main__':
main()
projects / pithos / blobdiff