3 # Copyright 2011-2012 GRNET S.A. All rights reserved.
5 # Redistribution and use in source and binary forms, with or
6 # without modification, are permitted provided that the following
9 # 1. Redistributions of source code must retain the above
10 # copyright notice, this list of conditions and the following
13 # 2. Redistributions in binary form must reproduce the above
14 # copyright notice, this list of conditions and the following
15 # disclaimer in the documentation and/or other materials
16 # provided with the distribution.
18 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 # POSSIBILITY OF SUCH DAMAGE.
31 # The views and conclusions contained in the software and
32 # documentation are those of the authors and should not be
33 # interpreted as representing official policies, either expressed
34 # or implied, of GRNET S.A.
36 from getpass import getuser
37 from optparse import OptionParser
38 from os import environ
39 from sys import argv, exit, stdin, stdout
40 from datetime import datetime
42 from pithos.lib.client import Pithos_Client, Fault
43 from pithos.lib.util import get_user, get_auth, get_url
44 from pithos.lib.transfer import upload, download
55 def cli_command(*args):
59 _cli_commands[name] = cls
63 def class_for_cli_command(name):
64 return _cli_commands[name]
66 class Command(object):
69 def __init__(self, name, argv):
70 parser = OptionParser('%%prog %s [options] %s' % (name, self.syntax))
71 parser.add_option('--url', dest='url', metavar='URL',
72 default=get_url(), help='server URL (currently: %s)' % get_url())
73 parser.add_option('--user', dest='user', metavar='USER',
75 help='account USER (currently: %s)' % get_user())
76 parser.add_option('--token', dest='token', metavar='TOKEN',
78 help='account TOKEN (currently: %s)' % get_auth())
79 parser.add_option('-v', action='store_true', dest='verbose',
80 default=False, help='verbose output')
81 parser.add_option('-d', action='store_true', dest='debug',
82 default=False, help='debug output')
83 self.add_options(parser)
84 options, args = parser.parse_args(argv)
87 for opt in parser.option_list:
90 val = getattr(options, key)
91 setattr(self, key, val)
93 self.client = Pithos_Client(self.url, self.token, self.user, self.verbose,
99 def _build_args(self, attrs):
101 for a in [a for a in attrs if getattr(self, a)]:
102 args[a] = getattr(self, a)
105 def add_options(self, parser):
108 def execute(self, *args):
111 @cli_command('list', 'ls')
113 syntax = '[<container>[/<object>]]'
114 description = 'list containers or objects'
116 def add_options(self, parser):
117 parser.add_option('-l', action='store_true', dest='detail',
118 default=False, help='show detailed output')
119 parser.add_option('-n', action='store', type='int', dest='limit',
120 default=10000, help='show limited output')
121 parser.add_option('--marker', action='store', type='str',
122 dest='marker', default=None,
123 help='show output greater then marker')
124 parser.add_option('--prefix', action='store', type='str',
125 dest='prefix', default=None,
126 help='show output starting with prefix')
127 parser.add_option('--delimiter', action='store', type='str',
128 dest='delimiter', default=None,
129 help='show output up to the delimiter')
130 parser.add_option('--path', action='store', type='str',
131 dest='path', default=None,
132 help='show output starting with prefix up to /')
133 parser.add_option('--meta', action='store', type='str',
134 dest='meta', default=None,
135 help='show output having the specified meta keys')
136 parser.add_option('--if-modified-since', action='store', type='str',
137 dest='if_modified_since', default=None,
138 help='show output if modified since then')
139 parser.add_option('--if-unmodified-since', action='store', type='str',
140 dest='if_unmodified_since', default=None,
141 help='show output if not modified since then')
142 parser.add_option('--until', action='store', dest='until',
143 default=None, help='show metadata until that date')
144 parser.add_option('--format', action='store', dest='format',
145 default='%d/%m/%Y', help='format to parse until date')
147 def execute(self, container=None):
149 self.list_objects(container)
151 self.list_containers()
153 def list_containers(self):
154 attrs = ['limit', 'marker', 'if_modified_since',
155 'if_unmodified_since']
156 args = self._build_args(attrs)
157 args['format'] = 'json' if self.detail else 'text'
159 if getattr(self, 'until'):
160 t = _time.strptime(self.until, self.format)
161 args['until'] = int(_time.mktime(t))
163 l = self.client.list_containers(**args)
166 def list_objects(self, container):
169 attrs = ['limit', 'marker', 'prefix', 'delimiter', 'path',
170 'meta', 'if_modified_since', 'if_unmodified_since']
171 args = self._build_args(attrs)
172 args['format'] = 'json' if self.detail else 'text'
175 t = _time.strptime(self.until, self.format)
176 args['until'] = int(_time.mktime(t))
178 container, sep, object = container.partition('/')
183 #if request with meta quering disable trash filtering
184 show_trashed = True if self.meta else False
185 l = self.client.list_objects(container, **args)
186 print_list(l, detail=self.detail)
190 syntax = '[<container>[/<object>]]'
191 description = 'get account/container/object metadata'
193 def add_options(self, parser):
194 parser.add_option('-r', action='store_true', dest='restricted',
195 default=False, help='show only user defined metadata')
196 parser.add_option('--until', action='store', dest='until',
197 default=None, help='show metadata until that date')
198 parser.add_option('--format', action='store', dest='format',
199 default='%d/%m/%Y', help='format to parse until date')
200 parser.add_option('--version', action='store', dest='version',
201 default=None, help='show specific version \
202 (applies only for objects)')
204 def execute(self, path=''):
205 container, sep, object = path.partition('/')
206 args = {'restricted': self.restricted}
207 if getattr(self, 'until'):
208 t = _time.strptime(self.until, self.format)
209 args['until'] = int(_time.mktime(t))
212 meta = self.client.retrieve_object_metadata(container, object,
216 meta = self.client.retrieve_container_metadata(container, **args)
218 meta = self.client.retrieve_account_metadata(**args)
220 print 'Entity does not exist'
222 print_dict(meta, header=None)
224 @cli_command('create')
225 class CreateContainer(Command):
226 syntax = '<container> [key=val] [...]'
227 description = 'create a container'
229 def add_options(self, parser):
230 parser.add_option('--versioning', action='store', dest='versioning',
231 default=None, help='set container versioning (auto/none)')
232 parser.add_option('--quota', action='store', dest='quota',
233 default=None, help='set default container quota')
235 def execute(self, container, *args):
238 key, sep, val = arg.partition('=')
241 if getattr(self, 'versioning'):
242 policy['versioning'] = self.versioning
243 if getattr(self, 'quota'):
244 policy['quota'] = self.quota
245 ret = self.client.create_container(container, meta=meta, policies=policy)
247 print 'Container already exists'
249 @cli_command('delete', 'rm')
250 class Delete(Command):
251 syntax = '<container>[/<object>]'
252 description = 'delete a container or an object'
254 def add_options(self, parser):
255 parser.add_option('--until', action='store', dest='until',
256 default=None, help='remove history until that date')
257 parser.add_option('--format', action='store', dest='format',
258 default='%d/%m/%Y', help='format to parse until date')
260 def execute(self, path):
261 container, sep, object = path.partition('/')
263 if getattr(self, 'until'):
264 t = _time.strptime(self.until, self.format)
265 until = int(_time.mktime(t))
268 self.client.delete_object(container, object, until)
270 self.client.delete_container(container, until)
273 class GetObject(Command):
274 syntax = '<container>/<object>'
275 description = 'get the data of an object'
277 def add_options(self, parser):
278 parser.add_option('-l', action='store_true', dest='detail',
279 default=False, help='show detailed output')
280 parser.add_option('--range', action='store', dest='range',
281 default=None, help='show range of data')
282 parser.add_option('--if-range', action='store', dest='if_range',
283 default=None, help='show range of data')
284 parser.add_option('--if-match', action='store', dest='if_match',
285 default=None, help='show output if ETags match')
286 parser.add_option('--if-none-match', action='store',
287 dest='if_none_match', default=None,
288 help='show output if ETags don\'t match')
289 parser.add_option('--if-modified-since', action='store', type='str',
290 dest='if_modified_since', default=None,
291 help='show output if modified since then')
292 parser.add_option('--if-unmodified-since', action='store', type='str',
293 dest='if_unmodified_since', default=None,
294 help='show output if not modified since then')
295 parser.add_option('-o', action='store', type='str',
296 dest='file', default=None,
297 help='save output in file')
298 parser.add_option('--version', action='store', type='str',
299 dest='version', default=None,
300 help='get the specific \
302 parser.add_option('--versionlist', action='store_true',
303 dest='versionlist', default=False,
304 help='get the full object version list')
305 parser.add_option('--hashmap', action='store_true',
306 dest='hashmap', default=False,
307 help='get the object hashmap instead')
309 def execute(self, path):
310 attrs = ['if_match', 'if_none_match', 'if_modified_since',
311 'if_unmodified_since', 'hashmap']
312 args = self._build_args(attrs)
313 args['format'] = 'json' if self.detail else 'text'
315 args['range'] = 'bytes=%s' % self.range
316 if getattr(self, 'if_range'):
317 args['if-range'] = 'If-Range:%s' % getattr(self, 'if_range')
319 container, sep, object = path.partition('/')
322 if 'detail' in args.keys():
326 data = self.client.retrieve_object_versionlist(container, object, **args)
328 data = self.client.retrieve_object_version(container, object,
329 self.version, **args)
331 if 'detail' in args.keys():
335 data = self.client.retrieve_object_hashmap(container, object, **args)
337 data = self.client.retrieve_object(container, object, **args)
339 f = open(self.file, 'w') if self.file else stdout
340 if self.detail or type(data) == types.DictionaryType:
342 print_versions(data, f=f)
344 print_dict(data, f=f)
349 @cli_command('mkdir')
350 class PutMarker(Command):
351 syntax = '<container>/<directory marker>'
352 description = 'create a directory marker'
354 def execute(self, path):
355 container, sep, object = path.partition('/')
356 self.client.create_directory_marker(container, object)
359 class PutObject(Command):
360 syntax = '<container>/<object> [key=val] [...]'
361 description = 'create/override object'
363 def add_options(self, parser):
364 parser.add_option('--use_hashes', action='store_true', dest='use_hashes',
365 default=False, help='provide hashmap instead of data')
366 parser.add_option('--chunked', action='store_true', dest='chunked',
367 default=False, help='set chunked transfer mode')
368 parser.add_option('--etag', action='store', dest='etag',
369 default=None, help='check written data')
370 parser.add_option('--content-encoding', action='store',
371 dest='content_encoding', default=None,
372 help='provide the object MIME content type')
373 parser.add_option('--content-disposition', action='store', type='str',
374 dest='content_disposition', default=None,
375 help='provide the presentation style of the object')
376 #parser.add_option('-S', action='store',
377 # dest='segment_size', default=False,
378 # help='use for large file support')
379 parser.add_option('--manifest', action='store',
380 dest='x_object_manifest', default=None,
381 help='provide object parts prefix in <container>/<object> form')
382 parser.add_option('--content-type', action='store',
383 dest='content_type', default=None,
384 help='create object with specific content type')
385 parser.add_option('--sharing', action='store',
386 dest='x_object_sharing', default=None,
387 help='define sharing object policy')
388 parser.add_option('-f', action='store',
389 dest='srcpath', default=None,
390 help='file descriptor to read from (pass - for standard input)')
391 parser.add_option('--public', action='store_true',
392 dest='x_object_public', default=False,
393 help='make object publicly accessible')
395 def execute(self, path, *args):
396 if path.find('=') != -1:
397 raise Fault('Missing path argument')
399 #prepare user defined meta
402 key, sep, val = arg.partition('=')
405 attrs = ['etag', 'content_encoding', 'content_disposition',
406 'content_type', 'x_object_sharing', 'x_object_public']
407 args = self._build_args(attrs)
409 container, sep, object = path.partition('/')
413 f = open(self.srcpath) if self.srcpath != '-' else stdin
415 if self.use_hashes and not f:
416 raise Fault('Illegal option combination')
419 self.client.create_object_using_chunks(container, object, f,
421 elif self.use_hashes:
424 hashmap = json.loads()
425 self.client.create_object_by_hashmap(container, object, hashmap,
428 print "Expected object"
429 elif self.x_object_manifest:
430 self.client.create_manifestation(container, object, self.x_object_manifest)
432 self.client.create_zero_length_object(container, object, meta=meta, **args)
434 self.client.create_object(container, object, f, meta=meta, **args)
438 @cli_command('copy', 'cp')
439 class CopyObject(Command):
440 syntax = '<src container>/<src object> [<dst container>/]<dst object> [key=val] [...]'
441 description = 'copy an object to a different location'
443 def add_options(self, parser):
444 parser.add_option('--version', action='store',
445 dest='version', default=False,
446 help='copy specific version')
447 parser.add_option('--public', action='store_true',
448 dest='public', default=False,
449 help='make object publicly accessible')
450 parser.add_option('--content-type', action='store',
451 dest='content_type', default=None,
452 help='change object\'s content type')
454 def execute(self, src, dst, *args):
455 src_container, sep, src_object = src.partition('/')
456 dst_container, sep, dst_object = dst.partition('/')
458 #prepare user defined meta
461 key, sep, val = arg.partition('=')
465 dst_container = src_container
468 args = {'content_type':self.content_type} if self.content_type else {}
469 self.client.copy_object(src_container, src_object, dst_container,
470 dst_object, meta, self.public, self.version,
474 class SetMeta(Command):
475 syntax = '[<container>[/<object>]] key=val [key=val] [...]'
476 description = 'set account/container/object metadata'
478 def execute(self, path, *args):
479 #in case of account fix the args
480 if path.find('=') != -1:
487 key, sep, val = arg.partition('=')
488 meta[key.strip()] = val.strip()
489 container, sep, object = path.partition('/')
491 self.client.update_object_metadata(container, object, **meta)
493 self.client.update_container_metadata(container, **meta)
495 self.client.update_account_metadata(**meta)
497 @cli_command('update')
498 class UpdateObject(Command):
499 syntax = '<container>/<object> path [key=val] [...]'
500 description = 'update object metadata/data (default mode: append)'
502 def add_options(self, parser):
503 parser.add_option('-a', action='store_true', dest='append',
504 default=True, help='append data')
505 parser.add_option('--offset', action='store',
507 default=None, help='starting offest to be updated')
508 parser.add_option('--range', action='store', dest='content_range',
509 default=None, help='range of data to be updated')
510 parser.add_option('--chunked', action='store_true', dest='chunked',
511 default=False, help='set chunked transfer mode')
512 parser.add_option('--content-encoding', action='store',
513 dest='content_encoding', default=None,
514 help='provide the object MIME content type')
515 parser.add_option('--content-disposition', action='store', type='str',
516 dest='content_disposition', default=None,
517 help='provide the presentation style of the object')
518 parser.add_option('--manifest', action='store', type='str',
519 dest='x_object_manifest', default=None,
520 help='use for large file support')
521 parser.add_option('--sharing', action='store',
522 dest='x_object_sharing', default=None,
523 help='define sharing object policy')
524 parser.add_option('--nosharing', action='store_true',
525 dest='no_sharing', default=None,
526 help='clear object sharing policy')
527 parser.add_option('-f', action='store',
528 dest='srcpath', default=None,
529 help='file descriptor to read from: pass - for standard input')
530 parser.add_option('--public', action='store_true',
531 dest='x_object_public', default=False,
532 help='make object publicly accessible')
533 parser.add_option('--replace', action='store_true',
534 dest='replace', default=False,
535 help='override metadata')
537 def execute(self, path, *args):
538 if path.find('=') != -1:
539 raise Fault('Missing path argument')
541 #prepare user defined meta
544 key, sep, val = arg.partition('=')
548 attrs = ['content_encoding', 'content_disposition', 'x_object_sharing',
549 'x_object_public', 'x_object_manifest', 'replace', 'offset',
551 args = self._build_args(attrs)
554 args['x_object_sharing'] = ''
556 container, sep, object = path.partition('/')
560 f = open(self.srcpath) if self.srcpath != '-' else stdin
563 self.client.update_object_using_chunks(container, object, f,
566 self.client.update_object(container, object, f, meta=meta, **args)
570 @cli_command('move', 'mv')
571 class MoveObject(Command):
572 syntax = '<src container>/<src object> [<dst container>/]<dst object>'
573 description = 'move an object to a different location'
575 def add_options(self, parser):
576 parser.add_option('--public', action='store_true',
577 dest='public', default=False,
578 help='make object publicly accessible')
579 parser.add_option('--content-type', action='store',
580 dest='content_type', default=None,
581 help='change object\'s content type')
583 def execute(self, src, dst, *args):
584 src_container, sep, src_object = src.partition('/')
585 dst_container, sep, dst_object = dst.partition('/')
587 dst_container = src_container
590 #prepare user defined meta
593 key, sep, val = arg.partition('=')
596 args = {'content_type':self.content_type} if self.content_type else {}
597 self.client.move_object(src_container, src_object, dst_container,
598 dst_object, meta, self.public, **args)
600 @cli_command('unset')
601 class UnsetObject(Command):
602 syntax = '<container>/[<object>] key [key] [...]'
603 description = 'delete metadata info'
605 def execute(self, path, *args):
606 #in case of account fix the args
615 container, sep, object = path.partition('/')
617 self.client.delete_object_metadata(container, object, meta)
619 self.client.delete_container_metadata(container, meta)
621 self.client.delete_account_metadata(meta)
623 @cli_command('group')
624 class CreateGroup(Command):
625 syntax = 'key=val [key=val] [...]'
626 description = 'create account groups'
628 def execute(self, *args):
631 key, sep, val = arg.partition('=')
633 self.client.set_account_groups(**groups)
635 @cli_command('ungroup')
636 class DeleteGroup(Command):
637 syntax = 'key [key] [...]'
638 description = 'delete account groups'
640 def execute(self, *args):
644 self.client.unset_account_groups(groups)
646 @cli_command('policy')
647 class SetPolicy(Command):
648 syntax = 'container key=val [key=val] [...]'
649 description = 'set container policies'
651 def execute(self, path, *args):
652 if path.find('=') != -1:
653 raise Fault('Missing container argument')
655 container, sep, object = path.partition('/')
658 raise Fault('Only containers have policies')
662 key, sep, val = arg.partition('=')
665 self.client.set_container_policies(container, **policies)
667 @cli_command('publish')
668 class PublishObject(Command):
669 syntax = '<container>/<object>'
670 description = 'publish an object'
672 def execute(self, src):
673 src_container, sep, src_object = src.partition('/')
675 self.client.publish_object(src_container, src_object)
677 @cli_command('unpublish')
678 class UnpublishObject(Command):
679 syntax = '<container>/<object>'
680 description = 'unpublish an object'
682 def execute(self, src):
683 src_container, sep, src_object = src.partition('/')
685 self.client.unpublish_object(src_container, src_object)
687 @cli_command('sharing')
688 class SharingObject(Command):
689 syntax = 'list users sharing objects with the user'
690 description = 'list user accounts sharing objects with the user'
692 def add_options(self, parser):
693 parser.add_option('-l', action='store_true', dest='detail',
694 default=False, help='show detailed output')
695 parser.add_option('-n', action='store', type='int', dest='limit',
696 default=10000, help='show limited output')
697 parser.add_option('--marker', action='store', type='str',
698 dest='marker', default=None,
699 help='show output greater then marker')
703 attrs = ['limit', 'marker']
704 args = self._build_args(attrs)
705 args['format'] = 'json' if self.detail else 'text'
707 print_list(self.client.list_shared_by_others(**args))
711 syntax = '<file> <container>[/<prefix>]'
712 description = 'upload file to container (using prefix)'
714 def execute(self, file, path):
715 container, sep, prefix = path.partition('/')
716 upload(self.client, file, container, prefix)
718 @cli_command('receive')
719 class Receive(Command):
720 syntax = '<container>/<object> <file>'
721 description = 'download object to file'
723 def execute(self, path, file):
724 container, sep, object = path.partition('/')
725 download(self.client, container, object, file)
728 cmd = Command('', [])
730 parser.usage = '%prog <command> [options]'
734 for cls in set(_cli_commands.values()):
735 name = ', '.join(cls.commands)
736 description = getattr(cls, 'description', '')
737 commands.append(' %s %s' % (name.ljust(12), description))
738 print '\nCommands:\n' + '\n'.join(sorted(commands))
740 def print_dict(d, header='name', f=stdout, detail=True):
741 header = header if header in d else 'subdir'
742 if header and header in d:
743 f.write('%s\n' %d.pop(header).encode('utf8'))
745 patterns = ['^x_(account|container|object)_meta_(\w+)$']
746 patterns.append(patterns[0].replace('_', '-'))
747 for key, val in sorted(d.items()):
748 f.write('%s: %s\n' % (key.rjust(30), val))
750 def print_list(l, verbose=False, f=stdout, detail=True):
752 #if it's empty string continue
755 if type(elem) == types.DictionaryType:
756 print_dict(elem, f=f, detail=detail)
757 elif type(elem) == types.StringType:
759 elem = elem.split('Traceback')[0]
760 f.write('%s\n' % elem)
762 f.write('%s\n' % elem)
764 def print_versions(data, f=stdout):
765 if 'versions' not in data:
766 f.write('%s\n' %data)
768 f.write('versions:\n')
769 for id, t in data['versions']:
770 f.write('%s @ %s\n' % (str(id).rjust(30), datetime.fromtimestamp(float(t))))
775 cls = class_for_cli_command(name)
776 except (IndexError, KeyError):
780 cmd = cls(name, argv[2:])
783 cmd.execute(*cmd.args)
785 cmd.parser.print_help()
788 status = '%s ' % f.status if f.status else ''
789 print '%s%s' % (status, f.data)
791 if __name__ == '__main__':