3 # Copyright 2011-2012 GRNET S.A. All rights reserved.
5 # Redistribution and use in source and binary forms, with or
6 # without modification, are permitted provided that the following
9 # 1. Redistributions of source code must retain the above
10 # copyright notice, this list of conditions and the following
13 # 2. Redistributions in binary form must reproduce the above
14 # copyright notice, this list of conditions and the following
15 # disclaimer in the documentation and/or other materials
16 # provided with the distribution.
18 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 # POSSIBILITY OF SUCH DAMAGE.
31 # The views and conclusions contained in the software and
32 # documentation are those of the authors and should not be
33 # interpreted as representing official policies, either expressed
34 # or implied, of GRNET S.A.
36 from getpass import getuser
37 from optparse import OptionParser
38 from os import environ
39 from sys import argv, exit, stdin, stdout
40 from datetime import datetime
42 from pithos.tools.lib.client import Pithos_Client, Fault
43 from pithos.tools.lib.util import get_user, get_auth, get_url
44 from pithos.tools.lib.transfer import upload, download
55 def cli_command(*args):
59 _cli_commands[name] = cls
63 def class_for_cli_command(name):
64 return _cli_commands[name]
66 class Command(object):
69 def __init__(self, name, argv):
70 parser = OptionParser('%%prog %s [options] %s' % (name, self.syntax))
71 parser.add_option('--url', dest='url', metavar='URL',
72 default=get_url(), help='server URL (currently: %s)' % get_url())
73 parser.add_option('--user', dest='user', metavar='USER',
75 help='account USER (currently: %s)' % get_user())
76 parser.add_option('--token', dest='token', metavar='TOKEN',
78 help='account TOKEN (currently: %s)' % get_auth())
79 parser.add_option('-v', action='store_true', dest='verbose',
80 default=False, help='verbose output')
81 parser.add_option('-d', action='store_true', dest='debug',
82 default=False, help='debug output')
83 self.add_options(parser)
84 options, args = parser.parse_args(argv)
87 for opt in parser.option_list:
90 val = getattr(options, key)
91 setattr(self, key, val)
93 self.client = Pithos_Client(self.url, self.token, self.user, self.verbose,
99 def _build_args(self, attrs):
101 for a in [a for a in attrs if getattr(self, a)]:
102 args[a] = getattr(self, a)
105 def add_options(self, parser):
108 def execute(self, *args):
111 @cli_command('list', 'ls')
113 syntax = '[<container>[/<object>]]'
114 description = 'list containers or objects'
116 def add_options(self, parser):
117 parser.add_option('-l', action='store_true', dest='detail',
118 default=False, help='show detailed output')
119 parser.add_option('-n', action='store', type='int', dest='limit',
120 default=10000, help='show limited output')
121 parser.add_option('--marker', action='store', type='str',
122 dest='marker', default=None,
123 help='show output greater then marker')
124 parser.add_option('--prefix', action='store', type='str',
125 dest='prefix', default=None,
126 help='show output starting with prefix')
127 parser.add_option('--delimiter', action='store', type='str',
128 dest='delimiter', default=None,
129 help='show output up to the delimiter')
130 parser.add_option('--path', action='store', type='str',
131 dest='path', default=None,
132 help='show output starting with prefix up to /')
133 parser.add_option('--meta', action='store', type='str',
134 dest='meta', default=None,
135 help='show output having the specified meta keys')
136 parser.add_option('--if-modified-since', action='store', type='str',
137 dest='if_modified_since', default=None,
138 help='show output if modified since then')
139 parser.add_option('--if-unmodified-since', action='store', type='str',
140 dest='if_unmodified_since', default=None,
141 help='show output if not modified since then')
142 parser.add_option('--until', action='store', dest='until',
143 default=None, help='show metadata until that date')
144 parser.add_option('--format', action='store', dest='format',
145 default='%d/%m/%Y', help='format to parse until date')
147 def execute(self, container=None):
149 self.list_objects(container)
151 self.list_containers()
153 def list_containers(self):
154 attrs = ['limit', 'marker', 'if_modified_since',
155 'if_unmodified_since']
156 args = self._build_args(attrs)
157 args['format'] = 'json' if self.detail else 'text'
159 if getattr(self, 'until'):
160 t = _time.strptime(self.until, self.format)
161 args['until'] = int(_time.mktime(t))
163 l = self.client.list_containers(**args)
166 def list_objects(self, container):
169 attrs = ['limit', 'marker', 'prefix', 'delimiter', 'path',
170 'meta', 'if_modified_since', 'if_unmodified_since']
171 args = self._build_args(attrs)
172 args['format'] = 'json' if self.detail else 'text'
175 t = _time.strptime(self.until, self.format)
176 args['until'] = int(_time.mktime(t))
178 container, sep, object = container.partition('/')
183 #if request with meta quering disable trash filtering
184 show_trashed = True if self.meta else False
185 l = self.client.list_objects(container, **args)
186 print_list(l, detail=self.detail)
190 syntax = '[<container>[/<object>]]'
191 description = 'get account/container/object metadata'
193 def add_options(self, parser):
194 parser.add_option('-r', action='store_true', dest='restricted',
195 default=False, help='show only user defined metadata')
196 parser.add_option('--until', action='store', dest='until',
197 default=None, help='show metadata until that date')
198 parser.add_option('--format', action='store', dest='format',
199 default='%d/%m/%Y', help='format to parse until date')
200 parser.add_option('--version', action='store', dest='version',
201 default=None, help='show specific version \
202 (applies only for objects)')
204 def execute(self, path=''):
205 container, sep, object = path.partition('/')
206 args = {'restricted': self.restricted}
207 if getattr(self, 'until'):
208 t = _time.strptime(self.until, self.format)
209 args['until'] = int(_time.mktime(t))
212 meta = self.client.retrieve_object_metadata(container, object,
216 meta = self.client.retrieve_container_metadata(container, **args)
218 meta = self.client.retrieve_account_metadata(**args)
220 print 'Entity does not exist'
222 print_dict(meta, header=None)
224 @cli_command('create')
225 class CreateContainer(Command):
226 syntax = '<container> [key=val] [...]'
227 description = 'create a container'
230 def add_options(self, parser):
231 parser.add_option('--versioning', action='store', dest=policy['versioning'],
232 default=None, help='set container versioning (auto/none)')
233 parser.add_option('--quota', action='store', dest=policy['quota'],
234 default=None, help='set default container quota')
236 def execute(self, container, *args):
239 key, sep, val = arg.partition('=')
241 ret = self.client.create_container(container, meta=meta, policies=policy)
243 print 'Container already exists'
245 @cli_command('delete', 'rm')
246 class Delete(Command):
247 syntax = '<container>[/<object>]'
248 description = 'delete a container or an object'
250 def add_options(self, parser):
251 parser.add_option('--until', action='store', dest='until',
252 default=None, help='remove history until that date')
253 parser.add_option('--format', action='store', dest='format',
254 default='%d/%m/%Y', help='format to parse until date')
256 def execute(self, path):
257 container, sep, object = path.partition('/')
259 if getattr(self, 'until'):
260 t = _time.strptime(self.until, self.format)
261 until = int(_time.mktime(t))
264 self.client.delete_object(container, object, until)
266 self.client.delete_container(container, until)
269 class GetObject(Command):
270 syntax = '<container>/<object>'
271 description = 'get the data of an object'
273 def add_options(self, parser):
274 parser.add_option('-l', action='store_true', dest='detail',
275 default=False, help='show detailed output')
276 parser.add_option('--range', action='store', dest='range',
277 default=None, help='show range of data')
278 parser.add_option('--if-range', action='store', dest='if_range',
279 default=None, help='show range of data')
280 parser.add_option('--if-match', action='store', dest='if_match',
281 default=None, help='show output if ETags match')
282 parser.add_option('--if-none-match', action='store',
283 dest='if_none_match', default=None,
284 help='show output if ETags don\'t match')
285 parser.add_option('--if-modified-since', action='store', type='str',
286 dest='if_modified_since', default=None,
287 help='show output if modified since then')
288 parser.add_option('--if-unmodified-since', action='store', type='str',
289 dest='if_unmodified_since', default=None,
290 help='show output if not modified since then')
291 parser.add_option('-o', action='store', type='str',
292 dest='file', default=None,
293 help='save output in file')
294 parser.add_option('--version', action='store', type='str',
295 dest='version', default=None,
296 help='get the specific \
298 parser.add_option('--versionlist', action='store_true',
299 dest='versionlist', default=False,
300 help='get the full object version list')
301 parser.add_option('--hashmap', action='store_true',
302 dest='hashmap', default=False,
303 help='get the object hashmap instead')
305 def execute(self, path):
306 attrs = ['if_match', 'if_none_match', 'if_modified_since',
307 'if_unmodified_since', 'hashmap']
308 args = self._build_args(attrs)
309 args['format'] = 'json' if self.detail else 'text'
311 args['range'] = 'bytes=%s' % self.range
312 if getattr(self, 'if_range'):
313 args['if-range'] = 'If-Range:%s' % getattr(self, 'if_range')
315 container, sep, object = path.partition('/')
318 if 'detail' in args.keys():
322 data = self.client.retrieve_object_versionlist(container, object, **args)
324 data = self.client.retrieve_object_version(container, object,
325 self.version, **args)
327 if 'detail' in args.keys():
331 data = self.client.retrieve_object_hashmap(container, object, **args)
333 data = self.client.retrieve_object(container, object, **args)
335 f = open(self.file, 'w') if self.file else stdout
336 if self.detail or type(data) == types.DictionaryType:
338 print_versions(data, f=f)
340 print_dict(data, f=f)
345 @cli_command('mkdir')
346 class PutMarker(Command):
347 syntax = '<container>/<directory marker>'
348 description = 'create a directory marker'
350 def execute(self, path):
351 container, sep, object = path.partition('/')
352 self.client.create_directory_marker(container, object)
355 class PutObject(Command):
356 syntax = '<container>/<object> [key=val] [...]'
357 description = 'create/override object'
359 def add_options(self, parser):
360 parser.add_option('--use_hashes', action='store_true', dest='use_hashes',
361 default=False, help='provide hashmap instead of data')
362 parser.add_option('--chunked', action='store_true', dest='chunked',
363 default=False, help='set chunked transfer mode')
364 parser.add_option('--etag', action='store', dest='etag',
365 default=None, help='check written data')
366 parser.add_option('--content-encoding', action='store',
367 dest='content_encoding', default=None,
368 help='provide the object MIME content type')
369 parser.add_option('--content-disposition', action='store', type='str',
370 dest='content_disposition', default=None,
371 help='provide the presentation style of the object')
372 #parser.add_option('-S', action='store',
373 # dest='segment_size', default=False,
374 # help='use for large file support')
375 parser.add_option('--manifest', action='store',
376 dest='x_object_manifest', default=None,
377 help='provide object parts prefix in <container>/<object> form')
378 parser.add_option('--content-type', action='store',
379 dest='content_type', default=None,
380 help='create object with specific content type')
381 parser.add_option('--sharing', action='store',
382 dest='x_object_sharing', default=None,
383 help='define sharing object policy')
384 parser.add_option('-f', action='store',
385 dest='srcpath', default=None,
386 help='file descriptor to read from (pass - for standard input)')
387 parser.add_option('--public', action='store_true',
388 dest='x_object_public', default=False,
389 help='make object publicly accessible')
391 def execute(self, path, *args):
392 if path.find('=') != -1:
393 raise Fault('Missing path argument')
395 #prepare user defined meta
398 key, sep, val = arg.partition('=')
401 attrs = ['etag', 'content_encoding', 'content_disposition',
402 'content_type', 'x_object_sharing', 'x_object_public']
403 args = self._build_args(attrs)
405 container, sep, object = path.partition('/')
409 f = open(self.srcpath) if self.srcpath != '-' else stdin
411 if self.use_hashes and not f:
412 raise Fault('Illegal option combination')
415 self.client.create_object_using_chunks(container, object, f,
417 elif self.use_hashes:
420 hashmap = json.loads()
421 self.client.create_object_by_hashmap(container, object, hashmap,
424 print "Expected object"
425 elif self.x_object_manifest:
426 self.client.create_manifestation(container, object, self.x_object_manifest)
428 self.client.create_zero_length_object(container, object, meta=meta, **args)
430 self.client.create_object(container, object, f, meta=meta, **args)
434 @cli_command('copy', 'cp')
435 class CopyObject(Command):
436 syntax = '<src container>/<src object> [<dst container>/]<dst object> [key=val] [...]'
437 description = 'copy an object to a different location'
439 def add_options(self, parser):
440 parser.add_option('--version', action='store',
441 dest='version', default=False,
442 help='copy specific version')
443 parser.add_option('--public', action='store_true',
444 dest='public', default=False,
445 help='make object publicly accessible')
446 parser.add_option('--content-type', action='store',
447 dest='content_type', default=None,
448 help='change object\'s content type')
450 def execute(self, src, dst, *args):
451 src_container, sep, src_object = src.partition('/')
452 dst_container, sep, dst_object = dst.partition('/')
454 #prepare user defined meta
457 key, sep, val = arg.partition('=')
461 dst_container = src_container
464 args = {'content_type':self.content_type} if self.content_type else {}
465 self.client.copy_object(src_container, src_object, dst_container,
466 dst_object, meta, self.public, self.version,
470 class SetMeta(Command):
471 syntax = '[<container>[/<object>]] key=val [key=val] [...]'
472 description = 'set account/container/object metadata'
474 def execute(self, path, *args):
475 #in case of account fix the args
476 if path.find('=') != -1:
483 key, sep, val = arg.partition('=')
484 meta[key.strip()] = val.strip()
485 container, sep, object = path.partition('/')
487 self.client.update_object_metadata(container, object, **meta)
489 self.client.update_container_metadata(container, **meta)
491 self.client.update_account_metadata(**meta)
493 @cli_command('update')
494 class UpdateObject(Command):
495 syntax = '<container>/<object> path [key=val] [...]'
496 description = 'update object metadata/data (default mode: append)'
498 def add_options(self, parser):
499 parser.add_option('-a', action='store_true', dest='append',
500 default=True, help='append data')
501 parser.add_option('--offset', action='store',
503 default=None, help='starting offest to be updated')
504 parser.add_option('--range', action='store', dest='content_range',
505 default=None, help='range of data to be updated')
506 parser.add_option('--chunked', action='store_true', dest='chunked',
507 default=False, help='set chunked transfer mode')
508 parser.add_option('--content-encoding', action='store',
509 dest='content_encoding', default=None,
510 help='provide the object MIME content type')
511 parser.add_option('--content-disposition', action='store', type='str',
512 dest='content_disposition', default=None,
513 help='provide the presentation style of the object')
514 parser.add_option('--manifest', action='store', type='str',
515 dest='x_object_manifest', default=None,
516 help='use for large file support')
517 parser.add_option('--sharing', action='store',
518 dest='x_object_sharing', default=None,
519 help='define sharing object policy')
520 parser.add_option('--nosharing', action='store_true',
521 dest='no_sharing', default=None,
522 help='clear object sharing policy')
523 parser.add_option('-f', action='store',
524 dest='srcpath', default=None,
525 help='file descriptor to read from: pass - for standard input')
526 parser.add_option('--public', action='store_true',
527 dest='x_object_public', default=False,
528 help='make object publicly accessible')
529 parser.add_option('--replace', action='store_true',
530 dest='replace', default=False,
531 help='override metadata')
533 def execute(self, path, *args):
534 if path.find('=') != -1:
535 raise Fault('Missing path argument')
537 #prepare user defined meta
540 key, sep, val = arg.partition('=')
544 attrs = ['content_encoding', 'content_disposition', 'x_object_sharing',
545 'x_object_public', 'x_object_manifest', 'replace', 'offset',
547 args = self._build_args(attrs)
550 args['x_object_sharing'] = ''
552 container, sep, object = path.partition('/')
556 f = open(self.srcpath) if self.srcpath != '-' else stdin
559 self.client.update_object_using_chunks(container, object, f,
562 self.client.update_object(container, object, f, meta=meta, **args)
566 @cli_command('move', 'mv')
567 class MoveObject(Command):
568 syntax = '<src container>/<src object> [<dst container>/]<dst object>'
569 description = 'move an object to a different location'
571 def add_options(self, parser):
572 parser.add_option('--public', action='store_true',
573 dest='public', default=False,
574 help='make object publicly accessible')
575 parser.add_option('--content-type', action='store',
576 dest='content_type', default=None,
577 help='change object\'s content type')
579 def execute(self, src, dst, *args):
580 src_container, sep, src_object = src.partition('/')
581 dst_container, sep, dst_object = dst.partition('/')
583 dst_container = src_container
586 #prepare user defined meta
589 key, sep, val = arg.partition('=')
592 args = {'content_type':self.content_type} if self.content_type else {}
593 self.client.move_object(src_container, src_object, dst_container,
594 dst_object, meta, self.public, **args)
596 @cli_command('unset')
597 class UnsetObject(Command):
598 syntax = '<container>/[<object>] key [key] [...]'
599 description = 'delete metadata info'
601 def execute(self, path, *args):
602 #in case of account fix the args
611 container, sep, object = path.partition('/')
613 self.client.delete_object_metadata(container, object, meta)
615 self.client.delete_container_metadata(container, meta)
617 self.client.delete_account_metadata(meta)
619 @cli_command('group')
620 class CreateGroup(Command):
621 syntax = 'key=val [key=val] [...]'
622 description = 'create account groups'
624 def execute(self, *args):
627 key, sep, val = arg.partition('=')
629 self.client.set_account_groups(**groups)
631 @cli_command('ungroup')
632 class DeleteGroup(Command):
633 syntax = 'key [key] [...]'
634 description = 'delete account groups'
636 def execute(self, *args):
640 self.client.unset_account_groups(groups)
642 @cli_command('policy')
643 class SetPolicy(Command):
644 syntax = 'container key=val [key=val] [...]'
645 description = 'set container policies'
647 def execute(self, path, *args):
648 if path.find('=') != -1:
649 raise Fault('Missing container argument')
651 container, sep, object = path.partition('/')
654 raise Fault('Only containers have policies')
658 key, sep, val = arg.partition('=')
661 self.client.set_container_policies(container, **policies)
663 @cli_command('publish')
664 class PublishObject(Command):
665 syntax = '<container>/<object>'
666 description = 'publish an object'
668 def execute(self, src):
669 src_container, sep, src_object = src.partition('/')
671 self.client.publish_object(src_container, src_object)
673 @cli_command('unpublish')
674 class UnpublishObject(Command):
675 syntax = '<container>/<object>'
676 description = 'unpublish an object'
678 def execute(self, src):
679 src_container, sep, src_object = src.partition('/')
681 self.client.unpublish_object(src_container, src_object)
683 @cli_command('sharing')
684 class SharingObject(Command):
685 syntax = 'list users sharing objects with the user'
686 description = 'list user accounts sharing objects with the user'
688 def add_options(self, parser):
689 parser.add_option('-l', action='store_true', dest='detail',
690 default=False, help='show detailed output')
691 parser.add_option('-n', action='store', type='int', dest='limit',
692 default=10000, help='show limited output')
693 parser.add_option('--marker', action='store', type='str',
694 dest='marker', default=None,
695 help='show output greater then marker')
699 attrs = ['limit', 'marker']
700 args = self._build_args(attrs)
701 args['format'] = 'json' if self.detail else 'text'
703 print_list(self.client.list_shared_by_others(**args))
707 syntax = '<file> <container>[/<prefix>]'
708 description = 'upload file to container (using prefix)'
710 def execute(self, file, path):
711 container, sep, prefix = path.partition('/')
712 upload(self.client, file, container, prefix)
714 @cli_command('receive')
715 class Receive(Command):
716 syntax = '<container>/<object> <file>'
717 description = 'download object to file'
719 def execute(self, path, file):
720 container, sep, object = path.partition('/')
721 download(self.client, container, object, file)
724 cmd = Command('', [])
726 parser.usage = '%prog <command> [options]'
730 for cls in set(_cli_commands.values()):
731 name = ', '.join(cls.commands)
732 description = getattr(cls, 'description', '')
733 commands.append(' %s %s' % (name.ljust(12), description))
734 print '\nCommands:\n' + '\n'.join(sorted(commands))
736 def print_dict(d, header='name', f=stdout, detail=True):
737 header = header if header in d else 'subdir'
738 if header and header in d:
739 f.write('%s\n' %d.pop(header).encode('utf8'))
741 patterns = ['^x_(account|container|object)_meta_(\w+)$']
742 patterns.append(patterns[0].replace('_', '-'))
743 for key, val in sorted(d.items()):
744 f.write('%s: %s\n' % (key.rjust(30), val))
746 def print_list(l, verbose=False, f=stdout, detail=True):
748 #if it's empty string continue
751 if type(elem) == types.DictionaryType:
752 print_dict(elem, f=f, detail=detail)
753 elif type(elem) == types.StringType:
755 elem = elem.split('Traceback')[0]
756 f.write('%s\n' % elem)
758 f.write('%s\n' % elem)
760 def print_versions(data, f=stdout):
761 if 'versions' not in data:
762 f.write('%s\n' %data)
764 f.write('versions:\n')
765 for id, t in data['versions']:
766 f.write('%s @ %s\n' % (str(id).rjust(30), datetime.fromtimestamp(float(t))))
772 cls = class_for_cli_command(name)
773 except (IndexError, KeyError):
777 cmd = cls(name, argv[2:])
780 cmd.execute(*cmd.args)
782 cmd.parser.print_help()
785 status = '%s ' % f.status if f.status else ''
786 print '%s%s' % (status, f.data)
789 if __name__ == '__main__':