3 # Copyright 2011-2012 GRNET S.A. All rights reserved.
5 # Redistribution and use in source and binary forms, with or
6 # without modification, are permitted provided that the following
9 # 1. Redistributions of source code must retain the above
10 # copyright notice, this list of conditions and the following
13 # 2. Redistributions in binary form must reproduce the above
14 # copyright notice, this list of conditions and the following
15 # disclaimer in the documentation and/or other materials
16 # provided with the distribution.
18 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 # POSSIBILITY OF SUCH DAMAGE.
31 # The views and conclusions contained in the software and
32 # documentation are those of the authors and should not be
33 # interpreted as representing official policies, either expressed
34 # or implied, of GRNET S.A.
36 from getpass import getuser
37 from optparse import OptionParser
38 from os import environ
39 from sys import argv, exit, stdin, stdout
40 from datetime import datetime
42 from kamaki.pithos_sh.client import Pithos_Client, Fault
43 from kamaki.pithos_sh.util import get_user, get_auth, get_url
44 from kamaki.pithos_sh.transfer import upload, download
55 def cli_command(*args):
59 _cli_commands[name] = cls
63 def class_for_cli_command(name):
64 return _cli_commands[name]
66 class Command(object):
69 def __init__(self, name, argv):
70 parser = OptionParser('%%prog %s [options] %s' % (name, self.syntax))
71 parser.add_option('--url', dest='url', metavar='URL',
72 default=get_url(), help='server URL (currently: %s)' % get_url())
73 parser.add_option('--user', dest='user', metavar='USER',
75 help='account USER (currently: %s)' % get_user())
76 parser.add_option('--token', dest='token', metavar='TOKEN',
78 help='account TOKEN (currently: %s)' % get_auth())
79 parser.add_option('-v', action='store_true', dest='verbose',
80 default=False, help='verbose output')
81 parser.add_option('-d', action='store_true', dest='debug',
82 default=False, help='debug output')
83 self.add_options(parser)
84 options, args = parser.parse_args(argv)
87 for opt in parser.option_list:
90 val = getattr(options, key)
91 setattr(self, key, val)
93 self.client = Pithos_Client(self.url, self.token, self.user, self.verbose,
99 def _build_args(self, attrs):
101 for a in [a for a in attrs if getattr(self, a)]:
102 args[a] = getattr(self, a)
105 def add_options(self, parser):
108 def execute(self, *args):
111 @cli_command('list', 'ls')
113 syntax = '[<container>[/<object>]]'
114 description = 'list containers or objects'
116 def add_options(self, parser):
117 parser.add_option('-l', action='store_true', dest='detail',
118 default=False, help='show detailed output')
119 parser.add_option('-n', action='store', type='int', dest='limit',
120 default=10000, help='show limited output')
121 parser.add_option('--marker', action='store', type='str',
122 dest='marker', default=None,
123 help='show output greater then marker')
124 parser.add_option('--prefix', action='store', type='str',
125 dest='prefix', default=None,
126 help='show output starting with prefix')
127 parser.add_option('--delimiter', action='store', type='str',
128 dest='delimiter', default=None,
129 help='show output up to the delimiter')
130 parser.add_option('--path', action='store', type='str',
131 dest='path', default=None,
132 help='show output starting with prefix up to /')
133 parser.add_option('--meta', action='store', type='str',
134 dest='meta', default=None,
135 help='show output having the specified meta keys')
136 parser.add_option('--if-modified-since', action='store', type='str',
137 dest='if_modified_since', default=None,
138 help='show output if modified since then')
139 parser.add_option('--if-unmodified-since', action='store', type='str',
140 dest='if_unmodified_since', default=None,
141 help='show output if not modified since then')
142 parser.add_option('--until', action='store', dest='until',
143 default=None, help='show metadata until that date')
144 parser.add_option('--format', action='store', dest='format',
145 default='%d/%m/%Y %H:%M:%S', help='format to parse until date (default: %d/%m/%Y %H:%M:%S)')
146 parser.add_option('--shared', action='store_true', dest='shared',
147 default=False, help='show only shared')
148 parser.add_option('--public', action='store_true', dest='public',
149 default=False, help='show only public')
152 def execute(self, container=None):
154 self.list_objects(container)
156 self.list_containers()
158 def list_containers(self):
159 attrs = ['limit', 'marker', 'if_modified_since',
160 'if_unmodified_since', 'shared', 'public']
161 args = self._build_args(attrs)
162 args['format'] = 'json' if self.detail else 'text'
164 if getattr(self, 'until'):
165 t = _time.strptime(self.until, self.format)
166 args['until'] = int(_time.mktime(t))
168 l = self.client.list_containers(**args)
171 def list_objects(self, container):
174 attrs = ['limit', 'marker', 'prefix', 'delimiter', 'path',
175 'meta', 'if_modified_since', 'if_unmodified_since',
177 args = self._build_args(attrs)
178 args['format'] = 'json' if self.detail else 'text'
181 t = _time.strptime(self.until, self.format)
182 args['until'] = int(_time.mktime(t))
184 container, sep, object = container.partition('/')
189 #if request with meta quering disable trash filtering
190 show_trashed = True if self.meta else False
191 l = self.client.list_objects(container, **args)
192 print_list(l, detail=self.detail)
196 syntax = '[<container>[/<object>]]'
197 description = 'get account/container/object metadata'
199 def add_options(self, parser):
200 parser.add_option('-r', action='store_true', dest='restricted',
201 default=False, help='show only user defined metadata')
202 parser.add_option('--until', action='store', dest='until',
203 default=None, help='show metadata until that date')
204 parser.add_option('--format', action='store', dest='format',
205 default='%d/%m/%Y %H:%M:%S', help='format to parse until date (default: %d/%m/%Y %H:%M:%S)')
206 parser.add_option('--version', action='store', dest='version',
207 default=None, help='show specific version \
208 (applies only for objects)')
210 def execute(self, path=''):
211 container, sep, object = path.partition('/')
212 args = {'restricted': self.restricted}
213 if getattr(self, 'until'):
214 t = _time.strptime(self.until, self.format)
215 args['until'] = int(_time.mktime(t))
218 meta = self.client.retrieve_object_metadata(container, object,
222 meta = self.client.retrieve_container_metadata(container, **args)
224 meta = self.client.retrieve_account_metadata(**args)
226 print 'Entity does not exist'
228 print_dict(meta, header=None)
230 @cli_command('create')
231 class CreateContainer(Command):
232 syntax = '<container> [key=val] [...]'
233 description = 'create a container'
235 def add_options(self, parser):
236 parser.add_option('--versioning', action='store', dest='versioning',
237 default=None, help='set container versioning (auto/none)')
238 parser.add_option('--quota', action='store', dest='quota',
239 default=None, help='set default container quota')
241 def execute(self, container, *args):
244 key, sep, val = arg.partition('=')
247 if getattr(self, 'versioning'):
248 policy['versioning'] = self.versioning
249 if getattr(self, 'quota'):
250 policy['quota'] = self.quota
251 ret = self.client.create_container(container, meta=meta, policies=policy)
253 print 'Container already exists'
255 @cli_command('delete', 'rm')
256 class Delete(Command):
257 syntax = '<container>[/<object>]'
258 description = 'delete a container or an object'
260 def add_options(self, parser):
261 parser.add_option('--until', action='store', dest='until',
262 default=None, help='remove history until that date')
263 parser.add_option('--format', action='store', dest='format',
264 default='%d/%m/%Y %H:%M:%S', help='format to parse until date (default: %d/%m/%Y %H:%M:%S)')
265 parser.add_option('--delimiter', action='store', type='str',
266 dest='delimiter', default=None,
267 help='mass delete objects with path staring with <src object> + delimiter')
268 parser.add_option('-r', action='store_true',
269 dest='recursive', default=False,
270 help='mass delimiter objects with delimiter /')
272 def execute(self, path):
273 container, sep, object = path.partition('/')
275 if getattr(self, 'until'):
276 t = _time.strptime(self.until, self.format)
277 until = int(_time.mktime(t))
281 kwargs['delimiter'] = self.delimiter
283 kwargs['delimiter'] = '/'
286 self.client.delete_object(container, object, until, **kwargs)
288 self.client.delete_container(container, until, **kwargs)
291 class GetObject(Command):
292 syntax = '<container>/<object>'
293 description = 'get the data of an object'
295 def add_options(self, parser):
296 parser.add_option('-l', action='store_true', dest='detail',
297 default=False, help='show detailed output')
298 parser.add_option('--range', action='store', dest='range',
299 default=None, help='show range of data')
300 parser.add_option('--if-range', action='store', dest='if_range',
301 default=None, help='show range of data')
302 parser.add_option('--if-match', action='store', dest='if_match',
303 default=None, help='show output if ETags match')
304 parser.add_option('--if-none-match', action='store',
305 dest='if_none_match', default=None,
306 help='show output if ETags don\'t match')
307 parser.add_option('--if-modified-since', action='store', type='str',
308 dest='if_modified_since', default=None,
309 help='show output if modified since then')
310 parser.add_option('--if-unmodified-since', action='store', type='str',
311 dest='if_unmodified_since', default=None,
312 help='show output if not modified since then')
313 parser.add_option('-o', action='store', type='str',
314 dest='file', default=None,
315 help='save output in file')
316 parser.add_option('--version', action='store', type='str',
317 dest='version', default=None,
318 help='get the specific \
320 parser.add_option('--versionlist', action='store_true',
321 dest='versionlist', default=False,
322 help='get the full object version list')
323 parser.add_option('--hashmap', action='store_true',
324 dest='hashmap', default=False,
325 help='get the object hashmap instead')
327 def execute(self, path):
328 attrs = ['if_match', 'if_none_match', 'if_modified_since',
329 'if_unmodified_since', 'hashmap']
330 args = self._build_args(attrs)
331 args['format'] = 'json' if self.detail else 'text'
333 args['range'] = 'bytes=%s' % self.range
334 if getattr(self, 'if_range'):
335 args['if-range'] = 'If-Range:%s' % getattr(self, 'if_range')
337 container, sep, object = path.partition('/')
340 if 'detail' in args.keys():
344 data = self.client.retrieve_object_versionlist(container, object, **args)
346 data = self.client.retrieve_object_version(container, object,
347 self.version, **args)
349 if 'detail' in args.keys():
353 data = self.client.retrieve_object_hashmap(container, object, **args)
355 data = self.client.retrieve_object(container, object, **args)
357 f = open(self.file, 'w') if self.file else stdout
358 if self.detail or type(data) == types.DictionaryType:
360 print_versions(data, f=f)
362 print_dict(data, f=f)
367 @cli_command('mkdir')
368 class PutMarker(Command):
369 syntax = '<container>/<directory marker>'
370 description = 'create a directory marker'
372 def execute(self, path):
373 container, sep, object = path.partition('/')
374 self.client.create_directory_marker(container, object)
377 class PutObject(Command):
378 syntax = '<container>/<object> [key=val] [...]'
379 description = 'create/override object'
381 def add_options(self, parser):
382 parser.add_option('--use_hashes', action='store_true', dest='use_hashes',
383 default=False, help='provide hashmap instead of data')
384 parser.add_option('--chunked', action='store_true', dest='chunked',
385 default=False, help='set chunked transfer mode')
386 parser.add_option('--etag', action='store', dest='etag',
387 default=None, help='check written data')
388 parser.add_option('--content-encoding', action='store',
389 dest='content_encoding', default=None,
390 help='provide the object MIME content type')
391 parser.add_option('--content-disposition', action='store', type='str',
392 dest='content_disposition', default=None,
393 help='provide the presentation style of the object')
394 #parser.add_option('-S', action='store',
395 # dest='segment_size', default=False,
396 # help='use for large file support')
397 parser.add_option('--manifest', action='store',
398 dest='x_object_manifest', default=None,
399 help='provide object parts prefix in <container>/<object> form')
400 parser.add_option('--content-type', action='store',
401 dest='content_type', default=None,
402 help='create object with specific content type')
403 parser.add_option('--sharing', action='store',
404 dest='x_object_sharing', default=None,
405 help='define sharing object policy')
406 parser.add_option('-f', action='store',
407 dest='srcpath', default=None,
408 help='file descriptor to read from (pass - for standard input)')
409 parser.add_option('--public', action='store_true',
410 dest='x_object_public', default=False,
411 help='make object publicly accessible')
413 def execute(self, path, *args):
414 if path.find('=') != -1:
415 raise Fault('Missing path argument')
417 #prepare user defined meta
420 key, sep, val = arg.partition('=')
423 attrs = ['etag', 'content_encoding', 'content_disposition',
424 'content_type', 'x_object_sharing', 'x_object_public']
425 args = self._build_args(attrs)
427 container, sep, object = path.partition('/')
431 f = open(self.srcpath) if self.srcpath != '-' else stdin
433 if self.use_hashes and not f:
434 raise Fault('Illegal option combination')
437 self.client.create_object_using_chunks(container, object, f,
439 elif self.use_hashes:
441 hashmap = json.loads(data)
442 self.client.create_object_by_hashmap(container, object, hashmap,
444 elif self.x_object_manifest:
445 self.client.create_manifestation(container, object, self.x_object_manifest)
447 self.client.create_zero_length_object(container, object, meta=meta, **args)
449 self.client.create_object(container, object, f, meta=meta, **args)
453 @cli_command('copy', 'cp')
454 class CopyObject(Command):
455 syntax = '<src container>/<src object> [<dst container>/]<dst object> [key=val] [...]'
456 description = 'copy an object to a different location'
458 def add_options(self, parser):
459 parser.add_option('--version', action='store',
460 dest='version', default=False,
461 help='copy specific version')
462 parser.add_option('--public', action='store_true',
463 dest='public', default=False,
464 help='make object publicly accessible')
465 parser.add_option('--content-type', action='store',
466 dest='content_type', default=None,
467 help='change object\'s content type')
468 parser.add_option('--delimiter', action='store', type='str',
469 dest='delimiter', default=None,
470 help='mass copy objects with path staring with <src object> + delimiter')
471 parser.add_option('-r', action='store_true',
472 dest='recursive', default=False,
473 help='mass copy with delimiter /')
475 def execute(self, src, dst, *args):
476 src_container, sep, src_object = src.partition('/')
477 dst_container, sep, dst_object = dst.partition('/')
479 #prepare user defined meta
482 key, sep, val = arg.partition('=')
486 dst_container = src_container
489 args = {'content_type':self.content_type} if self.content_type else {}
491 args['delimiter'] = self.delimiter
493 args['delimiter'] = '/'
494 self.client.copy_object(src_container, src_object, dst_container,
495 dst_object, meta, self.public, self.version,
499 class SetMeta(Command):
500 syntax = '[<container>[/<object>]] key=val [key=val] [...]'
501 description = 'set account/container/object metadata'
503 def execute(self, path, *args):
504 #in case of account fix the args
505 if path.find('=') != -1:
512 key, sep, val = arg.partition('=')
513 meta[key.strip()] = val.strip()
514 container, sep, object = path.partition('/')
516 self.client.update_object_metadata(container, object, **meta)
518 self.client.update_container_metadata(container, **meta)
520 self.client.update_account_metadata(**meta)
522 @cli_command('update')
523 class UpdateObject(Command):
524 syntax = '<container>/<object> path [key=val] [...]'
525 description = 'update object metadata/data (default mode: append)'
527 def add_options(self, parser):
528 parser.add_option('-a', action='store_true', dest='append',
529 default=True, help='append data')
530 parser.add_option('--offset', action='store',
532 default=None, help='starting offest to be updated')
533 parser.add_option('--range', action='store', dest='content_range',
534 default=None, help='range of data to be updated')
535 parser.add_option('--chunked', action='store_true', dest='chunked',
536 default=False, help='set chunked transfer mode')
537 parser.add_option('--content-encoding', action='store',
538 dest='content_encoding', default=None,
539 help='provide the object MIME content type')
540 parser.add_option('--content-disposition', action='store', type='str',
541 dest='content_disposition', default=None,
542 help='provide the presentation style of the object')
543 parser.add_option('--manifest', action='store', type='str',
544 dest='x_object_manifest', default=None,
545 help='use for large file support')
546 parser.add_option('--sharing', action='store',
547 dest='x_object_sharing', default=None,
548 help='define sharing object policy')
549 parser.add_option('--nosharing', action='store_true',
550 dest='no_sharing', default=None,
551 help='clear object sharing policy')
552 parser.add_option('-f', action='store',
553 dest='srcpath', default=None,
554 help='file descriptor to read from: pass - for standard input')
555 parser.add_option('--public', action='store_true',
556 dest='x_object_public', default=False,
557 help='make object publicly accessible')
558 parser.add_option('--replace', action='store_true',
559 dest='replace', default=False,
560 help='override metadata')
562 def execute(self, path, *args):
563 if path.find('=') != -1:
564 raise Fault('Missing path argument')
566 #prepare user defined meta
569 key, sep, val = arg.partition('=')
573 attrs = ['content_encoding', 'content_disposition', 'x_object_sharing',
574 'x_object_public', 'x_object_manifest', 'replace', 'offset',
576 args = self._build_args(attrs)
579 args['x_object_sharing'] = ''
581 container, sep, object = path.partition('/')
585 f = open(self.srcpath) if self.srcpath != '-' else stdin
588 self.client.update_object_using_chunks(container, object, f,
591 self.client.update_object(container, object, f, meta=meta, **args)
595 @cli_command('move', 'mv')
596 class MoveObject(Command):
597 syntax = '<src container>/<src object> [<dst container>/]<dst object>'
598 description = 'move an object to a different location'
600 def add_options(self, parser):
601 parser.add_option('--public', action='store_true',
602 dest='public', default=False,
603 help='make object publicly accessible')
604 parser.add_option('--content-type', action='store',
605 dest='content_type', default=None,
606 help='change object\'s content type')
607 parser.add_option('--delimiter', action='store', type='str',
608 dest='delimiter', default=None,
609 help='mass move objects with path staring with <src object> + delimiter')
610 parser.add_option('-r', action='store_true',
611 dest='recursive', default=False,
612 help='mass move objects with delimiter /')
614 def execute(self, src, dst, *args):
615 src_container, sep, src_object = src.partition('/')
616 dst_container, sep, dst_object = dst.partition('/')
618 dst_container = src_container
621 #prepare user defined meta
624 key, sep, val = arg.partition('=')
627 args = {'content_type':self.content_type} if self.content_type else {}
629 args['delimiter'] = self.delimiter
631 args['delimiter'] = '/'
632 self.client.move_object(src_container, src_object, dst_container,
633 dst_object, meta, self.public, **args)
635 @cli_command('unset')
636 class UnsetObject(Command):
637 syntax = '<container>/[<object>] key [key] [...]'
638 description = 'delete metadata info'
640 def execute(self, path, *args):
641 #in case of account fix the args
650 container, sep, object = path.partition('/')
652 self.client.delete_object_metadata(container, object, meta)
654 self.client.delete_container_metadata(container, meta)
656 self.client.delete_account_metadata(meta)
658 @cli_command('group')
659 class CreateGroup(Command):
660 syntax = 'key=val [key=val] [...]'
661 description = 'create account groups'
663 def execute(self, *args):
666 key, sep, val = arg.partition('=')
668 self.client.set_account_groups(**groups)
670 @cli_command('ungroup')
671 class DeleteGroup(Command):
672 syntax = 'key [key] [...]'
673 description = 'delete account groups'
675 def execute(self, *args):
679 self.client.unset_account_groups(groups)
681 @cli_command('policy')
682 class SetPolicy(Command):
683 syntax = 'container key=val [key=val] [...]'
684 description = 'set container policies'
686 def execute(self, path, *args):
687 if path.find('=') != -1:
688 raise Fault('Missing container argument')
690 container, sep, object = path.partition('/')
693 raise Fault('Only containers have policies')
697 key, sep, val = arg.partition('=')
700 self.client.set_container_policies(container, **policies)
702 @cli_command('publish')
703 class PublishObject(Command):
704 syntax = '<container>/<object>'
705 description = 'publish an object'
707 def execute(self, src):
708 src_container, sep, src_object = src.partition('/')
710 self.client.publish_object(src_container, src_object)
712 @cli_command('unpublish')
713 class UnpublishObject(Command):
714 syntax = '<container>/<object>'
715 description = 'unpublish an object'
717 def execute(self, src):
718 src_container, sep, src_object = src.partition('/')
720 self.client.unpublish_object(src_container, src_object)
722 @cli_command('sharing')
723 class SharingObject(Command):
724 syntax = 'list users sharing objects with the user'
725 description = 'list user accounts sharing objects with the user'
727 def add_options(self, parser):
728 parser.add_option('-l', action='store_true', dest='detail',
729 default=False, help='show detailed output')
730 parser.add_option('-n', action='store', type='int', dest='limit',
731 default=10000, help='show limited output')
732 parser.add_option('--marker', action='store', type='str',
733 dest='marker', default=None,
734 help='show output greater then marker')
738 attrs = ['limit', 'marker']
739 args = self._build_args(attrs)
740 args['format'] = 'json' if self.detail else 'text'
742 print_list(self.client.list_shared_by_others(**args))
746 syntax = '<file> <container>[/<prefix>]'
747 description = 'upload file to container (using prefix)'
749 def execute(self, file, path):
750 container, sep, prefix = path.partition('/')
751 upload(self.client, file, container, prefix)
753 @cli_command('receive')
754 class Receive(Command):
755 syntax = '<container>/<object> <file>'
756 description = 'download object to file'
758 def execute(self, path, file):
759 container, sep, object = path.partition('/')
760 download(self.client, container, object, file)
763 cmd = Command('', [])
765 parser.usage = '%prog <command> [options]'
769 for cls in set(_cli_commands.values()):
770 name = ', '.join(cls.commands)
771 description = getattr(cls, 'description', '')
772 commands.append(' %s %s' % (name.ljust(12), description))
773 print '\nCommands:\n' + '\n'.join(sorted(commands))
775 def print_dict(d, header='name', f=stdout, detail=True):
776 header = header if header in d else 'subdir'
777 if header and header in d:
778 f.write('%s\n' %d.pop(header).encode('utf8'))
780 patterns = ['^x_(account|container|object)_meta_(\w+)$']
781 patterns.append(patterns[0].replace('_', '-'))
782 for key, val in sorted(d.items()):
783 f.write('%s: %s\n' % (key.rjust(30), val))
785 def print_list(l, verbose=False, f=stdout, detail=True):
787 #if it's empty string continue
790 if type(elem) == types.DictionaryType:
791 print_dict(elem, f=f, detail=detail)
792 elif type(elem) == types.StringType:
794 elem = elem.split('Traceback')[0]
795 f.write('%s\n' % elem)
797 f.write('%s\n' % elem)
799 def print_versions(data, f=stdout):
800 if 'versions' not in data:
801 f.write('%s\n' %data)
803 f.write('versions:\n')
804 for id, t in data['versions']:
805 f.write('%s @ %s\n' % (str(id).rjust(30), datetime.fromtimestamp(float(t))))
811 cls = class_for_cli_command(name)
812 except (IndexError, KeyError):
816 cmd = cls(name, argv[2:])
819 cmd.execute(*cmd.args)
821 cmd.parser.print_help()
824 status = '%s ' % f.status if f.status else ''
825 print '%s%s' % (status, f.data)
828 if __name__ == '__main__':