3 # Copyright 2011 GRNET S.A. All rights reserved.
5 # Redistribution and use in source and binary forms, with or
6 # without modification, are permitted provided that the following
9 # 1. Redistributions of source code must retain the above
10 # copyright notice, this list of conditions and the following
13 # 2. Redistributions in binary form must reproduce the above
14 # copyright notice, this list of conditions and the following
15 # disclaimer in the documentation and/or other materials
16 # provided with the distribution.
18 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 # POSSIBILITY OF SUCH DAMAGE.
31 # The views and conclusions contained in the software and
32 # documentation are those of the authors and should not be
33 # interpreted as representing official policies, either expressed
34 # or implied, of GRNET S.A.
36 from getpass import getuser
37 from optparse import OptionParser
38 from os import environ
39 from sys import argv, exit, stdin, stdout
40 from datetime import datetime
41 from lib.client import Pithos_Client, Fault
42 from lib.util import get_user, get_auth, get_server, get_api
53 def cli_command(*args):
57 _cli_commands[name] = cls
61 def class_for_cli_command(name):
62 return _cli_commands[name]
64 class Command(object):
67 def __init__(self, name, argv):
68 parser = OptionParser('%%prog %s [options] %s' % (name, self.syntax))
69 parser.add_option('--host', dest='host', metavar='HOST',
70 default=get_server(), help='use server HOST')
71 parser.add_option('--user', dest='user', metavar='USERNAME',
73 help='use account USERNAME')
74 parser.add_option('--token', dest='token', metavar='AUTH',
76 help='use account AUTH')
77 parser.add_option('--api', dest='api', metavar='API',
78 default=get_api(), help='use api API')
79 parser.add_option('-v', action='store_true', dest='verbose',
80 default=False, help='use verbose output')
81 parser.add_option('-d', action='store_true', dest='debug',
82 default=False, help='use debug output')
83 self.add_options(parser)
84 options, args = parser.parse_args(argv)
87 for opt in parser.option_list:
90 val = getattr(options, key)
91 setattr(self, key, val)
93 self.client = Pithos_Client(self.host, self.token, self.user, self.api, self.verbose,
99 def _build_args(self, attrs):
101 for a in [a for a in attrs if getattr(self, a)]:
102 args[a] = getattr(self, a)
105 def add_options(self, parser):
108 def execute(self, *args):
111 @cli_command('list', 'ls')
113 syntax = '[<container>[/<object>]]'
114 description = 'list containers or objects'
116 def add_options(self, parser):
117 parser.add_option('-l', action='store_true', dest='detail',
118 default=False, help='show detailed output')
119 parser.add_option('-n', action='store', type='int', dest='limit',
120 default=10000, help='show limited output')
121 parser.add_option('--marker', action='store', type='str',
122 dest='marker', default=None,
123 help='show output greater then marker')
124 parser.add_option('--prefix', action='store', type='str',
125 dest='prefix', default=None,
126 help='show output starting with prefix')
127 parser.add_option('--delimiter', action='store', type='str',
128 dest='delimiter', default=None,
129 help='show output up to the delimiter')
130 parser.add_option('--path', action='store', type='str',
131 dest='path', default=None,
132 help='show output starting with prefix up to /')
133 parser.add_option('--meta', action='store', type='str',
134 dest='meta', default=None,
135 help='show output having the specified meta keys')
136 parser.add_option('--if-modified-since', action='store', type='str',
137 dest='if_modified_since', default=None,
138 help='show output if modified since then')
139 parser.add_option('--if-unmodified-since', action='store', type='str',
140 dest='if_unmodified_since', default=None,
141 help='show output if not modified since then')
142 parser.add_option('--until', action='store', dest='until',
143 default=None, help='show metadata until that date')
144 parser.add_option('--format', action='store', dest='format',
145 default='%d/%m/%Y', help='format to parse until date')
147 def execute(self, container=None):
149 self.list_objects(container)
151 self.list_containers()
153 def list_containers(self):
154 attrs = ['limit', 'marker', 'if_modified_since',
155 'if_unmodified_since']
156 args = self._build_args(attrs)
157 args['format'] = 'json' if self.detail else 'text'
159 if getattr(self, 'until'):
160 t = _time.strptime(self.until, self.format)
161 args['until'] = int(_time.mktime(t))
163 l = self.client.list_containers(**args)
166 def list_objects(self, container):
169 attrs = ['limit', 'marker', 'prefix', 'delimiter', 'path',
170 'meta', 'if_modified_since', 'if_unmodified_since']
171 args = self._build_args(attrs)
172 args['format'] = 'json' if self.detail else 'text'
175 t = _time.strptime(self.until, self.format)
176 args['until'] = int(_time.mktime(t))
178 container, sep, object = container.partition('/')
183 #if request with meta quering disable trash filtering
184 show_trashed = True if self.meta else False
185 l = self.client.list_objects(container, **args)
186 print_list(l, detail=self.detail)
190 syntax = '[<container>[/<object>]]'
191 description = 'get account/container/object metadata'
193 def add_options(self, parser):
194 parser.add_option('-r', action='store_true', dest='restricted',
195 default=False, help='show only user defined metadata')
196 parser.add_option('--until', action='store', dest='until',
197 default=None, help='show metadata until that date')
198 parser.add_option('--format', action='store', dest='format',
199 default='%d/%m/%Y', help='format to parse until date')
200 parser.add_option('--version', action='store', dest='version',
201 default=None, help='show specific version \
202 (applies only for objects)')
204 def execute(self, path=''):
205 container, sep, object = path.partition('/')
206 args = {'restricted':self.restricted}
207 if getattr(self, 'until'):
208 t = _time.strptime(self.until, self.format)
209 args['until'] = int(_time.mktime(t))
212 meta = self.client.retrieve_object_metadata(container, object,
216 meta = self.client.retrieve_container_metadata(container, **args)
218 meta = self.client.retrieve_account_metadata(**args)
220 print 'Entity does not exist'
222 print_dict(meta, header=None)
224 @cli_command('create')
225 class CreateContainer(Command):
226 syntax = '<container> [key=val] [...]'
227 description = 'create a container'
229 def execute(self, container, *args):
232 key, sep, val = arg.partition('=')
234 ret = self.client.create_container(container, **meta)
236 print 'Container already exists'
238 @cli_command('delete', 'rm')
239 class Delete(Command):
240 syntax = '<container>[/<object>]'
241 description = 'delete a container or an object'
243 def add_options(self, parser):
244 parser.add_option('--until', action='store', dest='until',
245 default=None, help='remove history until that date')
246 parser.add_option('--format', action='store', dest='format',
247 default='%d/%m/%Y', help='format to parse until date')
249 def execute(self, path):
250 container, sep, object = path.partition('/')
252 if getattr(self, 'until'):
253 t = _time.strptime(self.until, self.format)
254 until = int(_time.mktime(t))
257 self.client.delete_object(container, object, until)
259 self.client.delete_container(container, until)
262 class GetObject(Command):
263 syntax = '<container>/<object>'
264 description = 'get the data of an object'
266 def add_options(self, parser):
267 parser.add_option('-l', action='store_true', dest='detail',
268 default=False, help='show detailed output')
269 parser.add_option('--range', action='store', dest='range',
270 default=None, help='show range of data')
271 parser.add_option('--if-range', action='store', dest='if_range',
272 default=None, help='show range of data')
273 parser.add_option('--if-match', action='store', dest='if_match',
274 default=None, help='show output if ETags match')
275 parser.add_option('--if-none-match', action='store',
276 dest='if_none_match', default=None,
277 help='show output if ETags don\'t match')
278 parser.add_option('--if-modified-since', action='store', type='str',
279 dest='if_modified_since', default=None,
280 help='show output if modified since then')
281 parser.add_option('--if-unmodified-since', action='store', type='str',
282 dest='if_unmodified_since', default=None,
283 help='show output if not modified since then')
284 parser.add_option('-o', action='store', type='str',
285 dest='file', default=None,
286 help='save output in file')
287 parser.add_option('--version', action='store', type='str',
288 dest='version', default=None,
289 help='get the specific \
291 parser.add_option('--versionlist', action='store_true',
292 dest='versionlist', default=False,
293 help='get the full object version list')
295 def execute(self, path):
296 attrs = ['if_match', 'if_none_match', 'if_modified_since',
297 'if_unmodified_since']
298 args = self._build_args(attrs)
299 args['format'] = 'json' if self.detail else 'text'
301 args['range'] = 'bytes=%s' %self.range
302 if getattr(self, 'if_range'):
303 args['if-range'] = 'If-Range:%s' % getattr(self, 'if_range')
305 container, sep, object = path.partition('/')
308 if 'detail' in args.keys():
312 data = self.client.retrieve_object_versionlist(container, object, **args)
314 data = self.client.retrieve_object_version(container, object,
315 self.version, **args)
317 data = self.client.retrieve_object(container, object, **args)
319 f = self.file and open(self.file, 'w') or stdout
322 print_versions(data, f=f)
324 print_dict(data, f=f)
329 @cli_command('mkdir')
330 class PutMarker(Command):
331 syntax = '<container>/<directory marker>'
332 description = 'create a directory marker'
334 def execute(self, path):
335 container, sep, object = path.partition('/')
336 self.client.create_directory_marker(container, object)
339 class PutObject(Command):
340 syntax = '<container>/<object> [key=val] [...]'
341 description = 'create/override object'
343 def add_options(self, parser):
344 parser.add_option('--use_hashes', action='store_true', dest='use_hashes',
345 default=False, help='provide hashmap instead of data')
346 parser.add_option('--chunked', action='store_true', dest='chunked',
347 default=False, help='set chunked transfer mode')
348 parser.add_option('--etag', action='store', dest='etag',
349 default=None, help='check written data')
350 parser.add_option('--content-encoding', action='store',
351 dest='content_encoding', default=None,
352 help='provide the object MIME content type')
353 parser.add_option('--content-disposition', action='store', type='str',
354 dest='content_disposition', default=None,
355 help='provide the presentation style of the object')
356 #parser.add_option('-S', action='store',
357 # dest='segment_size', default=False,
358 # help='use for large file support')
359 parser.add_option('--manifest', action='store',
360 dest='x_object_manifest', default=None,
361 help='upload a manifestation file')
362 parser.add_option('--content-type', action='store',
363 dest='content_type', default=None,
364 help='create object with specific content type')
365 parser.add_option('--sharing', action='store',
366 dest='x_object_sharing', default=None,
367 help='define sharing object policy')
368 parser.add_option('-f', action='store',
369 dest='srcpath', default=None,
370 help='file descriptor to read from (pass - for standard input)')
371 parser.add_option('--public', action='store_true',
372 dest='x_object_public', default=False,
373 help='make object publicly accessible')
375 def execute(self, path, *args):
376 if path.find('=') != -1:
377 raise Fault('Missing path argument')
379 #prepare user defined meta
382 key, sep, val = arg.partition('=')
385 attrs = ['etag', 'content_encoding', 'content_disposition',
386 'content_type', 'x_object_sharing', 'x_object_public']
387 args = self._build_args(attrs)
389 container, sep, object = path.partition('/')
393 f = open(self.srcpath) if self.srcpath != '-' else stdin
395 if self.use_hashes and not f:
396 raise Fault('Illegal option combination')
399 self.client.create_object_using_chunks(container, object, f,
401 elif self.use_hashes:
404 hashmap = json.loads()
405 self.client.create_object_by_hashmap(container, object, hashmap,
408 print "Expected object"
409 elif self.x_object_manifest:
410 self.client.create_manifestation(container, object, self.x_object_manifest)
412 self.client.create_zero_length_object(container, object, meta=meta, **args)
414 self.client.create_object(container, object, f, meta=meta, **args)
418 @cli_command('copy', 'cp')
419 class CopyObject(Command):
420 syntax = '<src container>/<src object> [<dst container>/]<dst object> [key=val] [...]'
421 description = 'copy an object to a different location'
423 def add_options(self, parser):
424 parser.add_option('--version', action='store',
425 dest='version', default=False,
426 help='copy specific version')
427 parser.add_option('--public', action='store_true',
428 dest='public', default=False,
429 help='make object publicly accessible')
430 parser.add_option('--content-type', action='store',
431 dest='content_type', default=None,
432 help='change object\'s content type')
434 def execute(self, src, dst, *args):
435 src_container, sep, src_object = src.partition('/')
436 dst_container, sep, dst_object = dst.partition('/')
438 #prepare user defined meta
441 key, sep, val = arg.partition('=')
445 dst_container = src_container
448 args = {'content_type':self.content_type} if self.content_type else {}
449 self.client.copy_object(src_container, src_object, dst_container,
450 dst_object, meta, self.public, self.version,
454 class SetMeta(Command):
455 syntax = '[<container>[/<object>]] key=val [key=val] [...]'
456 description = 'set account/container/object metadata'
458 def execute(self, path, *args):
459 #in case of account fix the args
460 if path.find('=') != -1:
467 key, sep, val = arg.partition('=')
468 meta[key.strip()] = val.strip()
469 container, sep, object = path.partition('/')
471 self.client.update_object_metadata(container, object, **meta)
473 self.client.update_container_metadata(container, **meta)
475 self.client.update_account_metadata(**meta)
477 @cli_command('update')
478 class UpdateObject(Command):
479 syntax = '<container>/<object> path [key=val] [...]'
480 description = 'update object metadata/data (default mode: append)'
482 def add_options(self, parser):
483 parser.add_option('-a', action='store_true', dest='append',
484 default=True, help='append data')
485 parser.add_option('--offset', action='store',
487 default=None, help='starting offest to be updated')
488 parser.add_option('--range', action='store', dest='content-range',
489 default=None, help='range of data to be updated')
490 parser.add_option('--chunked', action='store_true', dest='chunked',
491 default=False, help='set chunked transfer mode')
492 parser.add_option('--content-encoding', action='store',
493 dest='content_encoding', default=None,
494 help='provide the object MIME content type')
495 parser.add_option('--content-disposition', action='store', type='str',
496 dest='content_disposition', default=None,
497 help='provide the presentation style of the object')
498 parser.add_option('--manifest', action='store', type='str',
499 dest='x_object_manifest', default=None,
500 help='use for large file support')
501 parser.add_option('--sharing', action='store',
502 dest='x_object_sharing', default=None,
503 help='define sharing object policy')
504 parser.add_option('--nosharing', action='store_true',
505 dest='no_sharing', default=None,
506 help='clear object sharing policy')
507 parser.add_option('-f', action='store',
508 dest='srcpath', default=None,
509 help='file descriptor to read from: pass - for standard input')
510 parser.add_option('--public', action='store_true',
511 dest='x_object_public', default=False,
512 help='make object publicly accessible')
513 parser.add_option('--replace', action='store_true',
514 dest='replace', default=False,
515 help='override metadata')
517 def execute(self, path, *args):
518 if path.find('=') != -1:
519 raise Fault('Missing path argument')
521 #prepare user defined meta
524 key, sep, val = arg.partition('=')
528 self.x_object_sharing = ''
530 attrs = ['content_encoding', 'content_disposition', 'x_object_sharing',
531 'x_object_public', 'replace']
532 args = self._build_args(attrs)
534 container, sep, object = path.partition('/')
538 f = open(self.srcpath) if self.srcpath != '-' else stdin
541 self.client.update_object_using_chunks(container, object, f,
544 self.client.update_object(container, object, f, meta=meta, **args)
548 @cli_command('move', 'mv')
549 class MoveObject(Command):
550 syntax = '<src container>/<src object> [<dst container>/]<dst object>'
551 description = 'move an object to a different location'
553 def add_options(self, parser):
554 parser.add_option('--version', action='store',
555 dest='version', default=None,
556 help='move a specific object version')
557 parser.add_option('--public', action='store_true',
558 dest='public', default=False,
559 help='make object publicly accessible')
560 parser.add_option('--content-type', action='store',
561 dest='content_type', default=None,
562 help='change object\'s content type')
564 def execute(self, src, dst, *args):
565 src_container, sep, src_object = src.partition('/')
566 dst_container, sep, dst_object = dst.partition('/')
568 dst_container = src_container
571 #prepare user defined meta
574 key, sep, val = arg.partition('=')
577 args = {'content_type':self.content_type} if self.content_type else {}
578 self.client.move_object(src_container, src_object, dst_container,
579 dst_object, meta, self.public, self.version, **args)
581 @cli_command('unset')
582 class UnsetObject(Command):
583 syntax = '<container>/[<object>] key [key] [...]'
584 description = 'delete metadata info'
586 def execute(self, path, *args):
587 #in case of account fix the args
596 container, sep, object = path.partition('/')
598 self.client.delete_object_metadata(container, object, meta)
600 self.client.delete_container_metadata(container, meta)
602 self.client.delete_account_metadata(meta)
604 @cli_command('group')
605 class CreateGroup(Command):
606 syntax = 'key=val [key=val] [...]'
607 description = 'create account groups'
609 def execute(self, *args):
612 key, sep, val = arg.partition('=')
614 self.client.set_account_groups(**groups)
616 @cli_command('ungroup')
617 class DeleteGroup(Command):
618 syntax = 'key [key] [...]'
619 description = 'delete account groups'
621 def execute(self, *args):
625 self.client.unset_account_groups(groups)
627 @cli_command('policy')
628 class SetPolicy(Command):
629 syntax = 'container key=val [key=val] [...]'
630 description = 'set container policies'
632 def execute(self, path, *args):
633 if path.find('=') != -1:
634 raise Fault('Missing container argument')
636 container, sep, object = path.partition('/')
639 raise Fault('Only containers have policies')
643 key, sep, val = arg.partition('=')
646 self.client.set_container_policies(container, **policies)
648 @cli_command('publish')
649 class PublishObject(Command):
650 syntax = '<container>/<object>'
651 description = 'publish an object'
653 def execute(self, src):
654 src_container, sep, src_object = src.partition('/')
656 self.client.publish_object(src_container, src_object)
658 @cli_command('unpublish')
659 class UnpublishObject(Command):
660 syntax = '<container>/<object>'
661 description = 'unpublish an object'
663 def execute(self, src):
664 src_container, sep, src_object = src.partition('/')
666 self.client.unpublish_object(src_container, src_object)
668 @cli_command('sharing')
669 class SharingObject(Command):
670 syntax = 'list users sharing objects with the user'
671 description = 'list user accounts sharing objects with the user'
673 def add_options(self, parser):
674 parser.add_option('-l', action='store_true', dest='detail',
675 default=False, help='show detailed output')
676 parser.add_option('-n', action='store', type='int', dest='limit',
677 default=10000, help='show limited output')
678 parser.add_option('--marker', action='store', type='str',
679 dest='marker', default=None,
680 help='show output greater then marker')
684 attrs = ['limit', 'marker']
685 args = self._build_args(attrs)
686 args['format'] = 'json' if self.detail else 'text'
688 print_list(self.client.list_shared_by_others(**args))
691 cmd = Command('', [])
693 parser.usage = '%prog <command> [options]'
697 for cls in set(_cli_commands.values()):
698 name = ', '.join(cls.commands)
699 description = getattr(cls, 'description', '')
700 commands.append(' %s %s' % (name.ljust(12), description))
701 print '\nCommands:\n' + '\n'.join(sorted(commands))
703 def print_dict(d, header='name', f=stdout, detail=True):
704 header = header if header in d else 'subdir'
705 if header and header in d:
706 f.write('%s\n' %d.pop(header).encode('utf8'))
708 patterns = ['^x_(account|container|object)_meta_(\w+)$']
709 patterns.append(patterns[0].replace('_', '-'))
710 for key, val in sorted(d.items()):
711 f.write('%s: %s\n' % (key.rjust(30), val))
713 def print_list(l, verbose=False, f=stdout, detail=True):
715 #if it's empty string continue
718 if type(elem) == types.DictionaryType:
719 print_dict(elem, f=f, detail=detail)
720 elif type(elem) == types.StringType:
722 elem = elem.split('Traceback')[0]
723 f.write('%s\n' % elem)
725 f.write('%s\n' % elem)
727 def print_versions(data, f=stdout):
728 if 'versions' not in data:
729 f.write('%s\n' %data)
731 f.write('versions:\n')
732 for id, t in data['versions']:
733 f.write('%s @ %s\n' % (str(id).rjust(30), datetime.fromtimestamp(t)))
738 cls = class_for_cli_command(name)
739 except (IndexError, KeyError):
743 cmd = cls(name, argv[2:])
746 cmd.execute(*cmd.args)
748 cmd.parser.print_help()
751 status = f.status and '%s ' % f.status or ''
752 print '%s%s' % (status, f.data)
754 if __name__ == '__main__':