3 # Copyright 2011-2012 GRNET S.A. All rights reserved.
5 # Redistribution and use in source and binary forms, with or
6 # without modification, are permitted provided that the following
9 # 1. Redistributions of source code must retain the above
10 # copyright notice, this list of conditions and the following
13 # 2. Redistributions in binary form must reproduce the above
14 # copyright notice, this list of conditions and the following
15 # disclaimer in the documentation and/or other materials
16 # provided with the distribution.
18 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 # POSSIBILITY OF SUCH DAMAGE.
31 # The views and conclusions contained in the software and
32 # documentation are those of the authors and should not be
33 # interpreted as representing official policies, either expressed
34 # or implied, of GRNET S.A.
36 from getpass import getuser
37 from optparse import OptionParser
38 from os import environ
39 from sys import argv, exit, stdin, stdout
40 from datetime import datetime
42 from pithos.tools.lib.client import Pithos_Client, Fault
43 from pithos.tools.lib.util import get_user, get_auth, get_url
44 from pithos.tools.lib.transfer import upload, download
55 def cli_command(*args):
59 _cli_commands[name] = cls
63 def class_for_cli_command(name):
64 return _cli_commands[name]
66 class Command(object):
69 def __init__(self, name, argv):
70 parser = OptionParser('%%prog %s [options] %s' % (name, self.syntax))
71 parser.add_option('--url', dest='url', metavar='URL',
72 default=get_url(), help='server URL (currently: %s)' % get_url())
73 parser.add_option('--user', dest='user', metavar='USER',
75 help='account USER (currently: %s)' % get_user())
76 parser.add_option('--token', dest='token', metavar='TOKEN',
78 help='account TOKEN (currently: %s)' % get_auth())
79 parser.add_option('-v', action='store_true', dest='verbose',
80 default=False, help='verbose output')
81 parser.add_option('-d', action='store_true', dest='debug',
82 default=False, help='debug output')
83 self.add_options(parser)
84 options, args = parser.parse_args(argv)
87 for opt in parser.option_list:
90 val = getattr(options, key)
91 setattr(self, key, val)
93 self.client = Pithos_Client(self.url, self.token, self.user, self.verbose,
99 def _build_args(self, attrs):
101 for a in [a for a in attrs if getattr(self, a)]:
102 args[a] = getattr(self, a)
105 def add_options(self, parser):
108 def execute(self, *args):
111 @cli_command('list', 'ls')
113 syntax = '[<container>[/<object>]]'
114 description = 'list containers or objects'
116 def add_options(self, parser):
117 parser.add_option('-l', action='store_true', dest='detail',
118 default=False, help='show detailed output')
119 parser.add_option('-n', action='store', type='int', dest='limit',
120 default=10000, help='show limited output')
121 parser.add_option('--marker', action='store', type='str',
122 dest='marker', default=None,
123 help='show output greater then marker')
124 parser.add_option('--prefix', action='store', type='str',
125 dest='prefix', default=None,
126 help='show output starting with prefix')
127 parser.add_option('--delimiter', action='store', type='str',
128 dest='delimiter', default=None,
129 help='show output up to the delimiter')
130 parser.add_option('--path', action='store', type='str',
131 dest='path', default=None,
132 help='show output starting with prefix up to /')
133 parser.add_option('--meta', action='store', type='str',
134 dest='meta', default=None,
135 help='show output having the specified meta keys')
136 parser.add_option('--if-modified-since', action='store', type='str',
137 dest='if_modified_since', default=None,
138 help='show output if modified since then')
139 parser.add_option('--if-unmodified-since', action='store', type='str',
140 dest='if_unmodified_since', default=None,
141 help='show output if not modified since then')
142 parser.add_option('--until', action='store', dest='until',
143 default=None, help='show metadata until that date')
144 parser.add_option('--format', action='store', dest='format',
145 default='%d/%m/%Y', help='format to parse until date')
147 def execute(self, container=None):
149 self.list_objects(container)
151 self.list_containers()
153 def list_containers(self):
154 attrs = ['limit', 'marker', 'if_modified_since',
155 'if_unmodified_since']
156 args = self._build_args(attrs)
157 args['format'] = 'json' if self.detail else 'text'
159 if getattr(self, 'until'):
160 t = _time.strptime(self.until, self.format)
161 args['until'] = int(_time.mktime(t))
163 l = self.client.list_containers(**args)
166 def list_objects(self, container):
169 attrs = ['limit', 'marker', 'prefix', 'delimiter', 'path',
170 'meta', 'if_modified_since', 'if_unmodified_since']
171 args = self._build_args(attrs)
172 args['format'] = 'json' if self.detail else 'text'
175 t = _time.strptime(self.until, self.format)
176 args['until'] = int(_time.mktime(t))
178 container, sep, object = container.partition('/')
183 #if request with meta quering disable trash filtering
184 show_trashed = True if self.meta else False
185 l = self.client.list_objects(container, **args)
186 print_list(l, detail=self.detail)
190 syntax = '[<container>[/<object>]]'
191 description = 'get account/container/object metadata'
193 def add_options(self, parser):
194 parser.add_option('-r', action='store_true', dest='restricted',
195 default=False, help='show only user defined metadata')
196 parser.add_option('--until', action='store', dest='until',
197 default=None, help='show metadata until that date')
198 parser.add_option('--format', action='store', dest='format',
199 default='%d/%m/%Y', help='format to parse until date')
200 parser.add_option('--version', action='store', dest='version',
201 default=None, help='show specific version \
202 (applies only for objects)')
204 def execute(self, path=''):
205 container, sep, object = path.partition('/')
206 args = {'restricted': self.restricted}
207 if getattr(self, 'until'):
208 t = _time.strptime(self.until, self.format)
209 args['until'] = int(_time.mktime(t))
212 meta = self.client.retrieve_object_metadata(container, object,
216 meta = self.client.retrieve_container_metadata(container, **args)
218 meta = self.client.retrieve_account_metadata(**args)
220 print 'Entity does not exist'
222 print_dict(meta, header=None)
224 @cli_command('create')
225 class CreateContainer(Command):
226 syntax = '<container> [key=val] [...]'
227 description = 'create a container'
229 def add_options(self, parser):
230 parser.add_option('--versioning', action='store', dest='versioning',
231 default=None, help='set container versioning (auto/none)')
232 parser.add_option('--quota', action='store', dest='quota',
233 default=None, help='set default container quota')
235 def execute(self, container, *args):
238 key, sep, val = arg.partition('=')
241 if getattr(self, 'versioning'):
242 policy['versioning'] = self.versioning
243 if getattr(self, 'quota'):
244 policy['quota'] = self.quota
245 ret = self.client.create_container(container, meta=meta, policies=policy)
247 print 'Container already exists'
249 @cli_command('delete', 'rm')
250 class Delete(Command):
251 syntax = '<container>[/<object>]'
252 description = 'delete a container or an object'
254 def add_options(self, parser):
255 parser.add_option('--until', action='store', dest='until',
256 default=None, help='remove history until that date')
257 parser.add_option('--format', action='store', dest='format',
258 default='%d/%m/%Y', help='format to parse until date')
260 def execute(self, path):
261 container, sep, object = path.partition('/')
263 if getattr(self, 'until'):
264 t = _time.strptime(self.until, self.format)
265 until = int(_time.mktime(t))
268 self.client.delete_object(container, object, until)
270 self.client.delete_container(container, until)
273 class GetObject(Command):
274 syntax = '<container>/<object>'
275 description = 'get the data of an object'
277 def add_options(self, parser):
278 parser.add_option('-l', action='store_true', dest='detail',
279 default=False, help='show detailed output')
280 parser.add_option('--range', action='store', dest='range',
281 default=None, help='show range of data')
282 parser.add_option('--if-range', action='store', dest='if_range',
283 default=None, help='show range of data')
284 parser.add_option('--if-match', action='store', dest='if_match',
285 default=None, help='show output if ETags match')
286 parser.add_option('--if-none-match', action='store',
287 dest='if_none_match', default=None,
288 help='show output if ETags don\'t match')
289 parser.add_option('--if-modified-since', action='store', type='str',
290 dest='if_modified_since', default=None,
291 help='show output if modified since then')
292 parser.add_option('--if-unmodified-since', action='store', type='str',
293 dest='if_unmodified_since', default=None,
294 help='show output if not modified since then')
295 parser.add_option('-o', action='store', type='str',
296 dest='file', default=None,
297 help='save output in file')
298 parser.add_option('--version', action='store', type='str',
299 dest='version', default=None,
300 help='get the specific \
302 parser.add_option('--versionlist', action='store_true',
303 dest='versionlist', default=False,
304 help='get the full object version list')
305 parser.add_option('--hashmap', action='store_true',
306 dest='hashmap', default=False,
307 help='get the object hashmap instead')
309 def execute(self, path):
310 attrs = ['if_match', 'if_none_match', 'if_modified_since',
311 'if_unmodified_since', 'hashmap']
312 args = self._build_args(attrs)
313 args['format'] = 'json' if self.detail else 'text'
315 args['range'] = 'bytes=%s' % self.range
316 if getattr(self, 'if_range'):
317 args['if-range'] = 'If-Range:%s' % getattr(self, 'if_range')
319 container, sep, object = path.partition('/')
322 if 'detail' in args.keys():
326 data = self.client.retrieve_object_versionlist(container, object, **args)
328 data = self.client.retrieve_object_version(container, object,
329 self.version, **args)
331 if 'detail' in args.keys():
335 data = self.client.retrieve_object_hashmap(container, object, **args)
337 data = self.client.retrieve_object(container, object, **args)
339 f = open(self.file, 'w') if self.file else stdout
340 if self.detail or type(data) == types.DictionaryType:
342 print_versions(data, f=f)
344 print_dict(data, f=f)
349 @cli_command('mkdir')
350 class PutMarker(Command):
351 syntax = '<container>/<directory marker>'
352 description = 'create a directory marker'
354 def execute(self, path):
355 container, sep, object = path.partition('/')
356 self.client.create_directory_marker(container, object)
359 class PutObject(Command):
360 syntax = '<container>/<object> [key=val] [...]'
361 description = 'create/override object'
363 def add_options(self, parser):
364 parser.add_option('--use_hashes', action='store_true', dest='use_hashes',
365 default=False, help='provide hashmap instead of data')
366 parser.add_option('--chunked', action='store_true', dest='chunked',
367 default=False, help='set chunked transfer mode')
368 parser.add_option('--etag', action='store', dest='etag',
369 default=None, help='check written data')
370 parser.add_option('--content-encoding', action='store',
371 dest='content_encoding', default=None,
372 help='provide the object MIME content type')
373 parser.add_option('--content-disposition', action='store', type='str',
374 dest='content_disposition', default=None,
375 help='provide the presentation style of the object')
376 #parser.add_option('-S', action='store',
377 # dest='segment_size', default=False,
378 # help='use for large file support')
379 parser.add_option('--manifest', action='store',
380 dest='x_object_manifest', default=None,
381 help='provide object parts prefix in <container>/<object> form')
382 parser.add_option('--content-type', action='store',
383 dest='content_type', default=None,
384 help='create object with specific content type')
385 parser.add_option('--sharing', action='store',
386 dest='x_object_sharing', default=None,
387 help='define sharing object policy')
388 parser.add_option('-f', action='store',
389 dest='srcpath', default=None,
390 help='file descriptor to read from (pass - for standard input)')
391 parser.add_option('--public', action='store_true',
392 dest='x_object_public', default=False,
393 help='make object publicly accessible')
395 def execute(self, path, *args):
396 if path.find('=') != -1:
397 raise Fault('Missing path argument')
399 #prepare user defined meta
402 key, sep, val = arg.partition('=')
405 attrs = ['etag', 'content_encoding', 'content_disposition',
406 'content_type', 'x_object_sharing', 'x_object_public']
407 args = self._build_args(attrs)
409 container, sep, object = path.partition('/')
413 f = open(self.srcpath) if self.srcpath != '-' else stdin
415 if self.use_hashes and not f:
416 raise Fault('Illegal option combination')
419 self.client.create_object_using_chunks(container, object, f,
421 elif self.use_hashes:
423 hashmap = json.loads(data)
424 self.client.create_object_by_hashmap(container, object, hashmap,
426 elif self.x_object_manifest:
427 self.client.create_manifestation(container, object, self.x_object_manifest)
429 self.client.create_zero_length_object(container, object, meta=meta, **args)
431 self.client.create_object(container, object, f, meta=meta, **args)
435 @cli_command('copy', 'cp')
436 class CopyObject(Command):
437 syntax = '<src container>/<src object> [<dst container>/]<dst object> [key=val] [...]'
438 description = 'copy an object to a different location'
440 def add_options(self, parser):
441 parser.add_option('--version', action='store',
442 dest='version', default=False,
443 help='copy specific version')
444 parser.add_option('--public', action='store_true',
445 dest='public', default=False,
446 help='make object publicly accessible')
447 parser.add_option('--content-type', action='store',
448 dest='content_type', default=None,
449 help='change object\'s content type')
451 def execute(self, src, dst, *args):
452 src_container, sep, src_object = src.partition('/')
453 dst_container, sep, dst_object = dst.partition('/')
455 #prepare user defined meta
458 key, sep, val = arg.partition('=')
462 dst_container = src_container
465 args = {'content_type':self.content_type} if self.content_type else {}
466 self.client.copy_object(src_container, src_object, dst_container,
467 dst_object, meta, self.public, self.version,
471 class SetMeta(Command):
472 syntax = '[<container>[/<object>]] key=val [key=val] [...]'
473 description = 'set account/container/object metadata'
475 def execute(self, path, *args):
476 #in case of account fix the args
477 if path.find('=') != -1:
484 key, sep, val = arg.partition('=')
485 meta[key.strip()] = val.strip()
486 container, sep, object = path.partition('/')
488 self.client.update_object_metadata(container, object, **meta)
490 self.client.update_container_metadata(container, **meta)
492 self.client.update_account_metadata(**meta)
494 @cli_command('update')
495 class UpdateObject(Command):
496 syntax = '<container>/<object> path [key=val] [...]'
497 description = 'update object metadata/data (default mode: append)'
499 def add_options(self, parser):
500 parser.add_option('-a', action='store_true', dest='append',
501 default=True, help='append data')
502 parser.add_option('--offset', action='store',
504 default=None, help='starting offest to be updated')
505 parser.add_option('--range', action='store', dest='content_range',
506 default=None, help='range of data to be updated')
507 parser.add_option('--chunked', action='store_true', dest='chunked',
508 default=False, help='set chunked transfer mode')
509 parser.add_option('--content-encoding', action='store',
510 dest='content_encoding', default=None,
511 help='provide the object MIME content type')
512 parser.add_option('--content-disposition', action='store', type='str',
513 dest='content_disposition', default=None,
514 help='provide the presentation style of the object')
515 parser.add_option('--manifest', action='store', type='str',
516 dest='x_object_manifest', default=None,
517 help='use for large file support')
518 parser.add_option('--sharing', action='store',
519 dest='x_object_sharing', default=None,
520 help='define sharing object policy')
521 parser.add_option('--nosharing', action='store_true',
522 dest='no_sharing', default=None,
523 help='clear object sharing policy')
524 parser.add_option('-f', action='store',
525 dest='srcpath', default=None,
526 help='file descriptor to read from: pass - for standard input')
527 parser.add_option('--public', action='store_true',
528 dest='x_object_public', default=False,
529 help='make object publicly accessible')
530 parser.add_option('--replace', action='store_true',
531 dest='replace', default=False,
532 help='override metadata')
534 def execute(self, path, *args):
535 if path.find('=') != -1:
536 raise Fault('Missing path argument')
538 #prepare user defined meta
541 key, sep, val = arg.partition('=')
545 attrs = ['content_encoding', 'content_disposition', 'x_object_sharing',
546 'x_object_public', 'x_object_manifest', 'replace', 'offset',
548 args = self._build_args(attrs)
551 args['x_object_sharing'] = ''
553 container, sep, object = path.partition('/')
557 f = open(self.srcpath) if self.srcpath != '-' else stdin
560 self.client.update_object_using_chunks(container, object, f,
563 self.client.update_object(container, object, f, meta=meta, **args)
567 @cli_command('move', 'mv')
568 class MoveObject(Command):
569 syntax = '<src container>/<src object> [<dst container>/]<dst object>'
570 description = 'move an object to a different location'
572 def add_options(self, parser):
573 parser.add_option('--public', action='store_true',
574 dest='public', default=False,
575 help='make object publicly accessible')
576 parser.add_option('--content-type', action='store',
577 dest='content_type', default=None,
578 help='change object\'s content type')
580 def execute(self, src, dst, *args):
581 src_container, sep, src_object = src.partition('/')
582 dst_container, sep, dst_object = dst.partition('/')
584 dst_container = src_container
587 #prepare user defined meta
590 key, sep, val = arg.partition('=')
593 args = {'content_type':self.content_type} if self.content_type else {}
594 self.client.move_object(src_container, src_object, dst_container,
595 dst_object, meta, self.public, **args)
597 @cli_command('unset')
598 class UnsetObject(Command):
599 syntax = '<container>/[<object>] key [key] [...]'
600 description = 'delete metadata info'
602 def execute(self, path, *args):
603 #in case of account fix the args
612 container, sep, object = path.partition('/')
614 self.client.delete_object_metadata(container, object, meta)
616 self.client.delete_container_metadata(container, meta)
618 self.client.delete_account_metadata(meta)
620 @cli_command('group')
621 class CreateGroup(Command):
622 syntax = 'key=val [key=val] [...]'
623 description = 'create account groups'
625 def execute(self, *args):
628 key, sep, val = arg.partition('=')
630 self.client.set_account_groups(**groups)
632 @cli_command('ungroup')
633 class DeleteGroup(Command):
634 syntax = 'key [key] [...]'
635 description = 'delete account groups'
637 def execute(self, *args):
641 self.client.unset_account_groups(groups)
643 @cli_command('policy')
644 class SetPolicy(Command):
645 syntax = 'container key=val [key=val] [...]'
646 description = 'set container policies'
648 def execute(self, path, *args):
649 if path.find('=') != -1:
650 raise Fault('Missing container argument')
652 container, sep, object = path.partition('/')
655 raise Fault('Only containers have policies')
659 key, sep, val = arg.partition('=')
662 self.client.set_container_policies(container, **policies)
664 @cli_command('publish')
665 class PublishObject(Command):
666 syntax = '<container>/<object>'
667 description = 'publish an object'
669 def execute(self, src):
670 src_container, sep, src_object = src.partition('/')
672 self.client.publish_object(src_container, src_object)
674 @cli_command('unpublish')
675 class UnpublishObject(Command):
676 syntax = '<container>/<object>'
677 description = 'unpublish an object'
679 def execute(self, src):
680 src_container, sep, src_object = src.partition('/')
682 self.client.unpublish_object(src_container, src_object)
684 @cli_command('sharing')
685 class SharingObject(Command):
686 syntax = 'list users sharing objects with the user'
687 description = 'list user accounts sharing objects with the user'
689 def add_options(self, parser):
690 parser.add_option('-l', action='store_true', dest='detail',
691 default=False, help='show detailed output')
692 parser.add_option('-n', action='store', type='int', dest='limit',
693 default=10000, help='show limited output')
694 parser.add_option('--marker', action='store', type='str',
695 dest='marker', default=None,
696 help='show output greater then marker')
700 attrs = ['limit', 'marker']
701 args = self._build_args(attrs)
702 args['format'] = 'json' if self.detail else 'text'
704 print_list(self.client.list_shared_by_others(**args))
708 syntax = '<file> <container>[/<prefix>]'
709 description = 'upload file to container (using prefix)'
711 def execute(self, file, path):
712 container, sep, prefix = path.partition('/')
713 upload(self.client, file, container, prefix)
715 @cli_command('receive')
716 class Receive(Command):
717 syntax = '<container>/<object> <file>'
718 description = 'download object to file'
720 def execute(self, path, file):
721 container, sep, object = path.partition('/')
722 download(self.client, container, object, file)
725 cmd = Command('', [])
727 parser.usage = '%prog <command> [options]'
731 for cls in set(_cli_commands.values()):
732 name = ', '.join(cls.commands)
733 description = getattr(cls, 'description', '')
734 commands.append(' %s %s' % (name.ljust(12), description))
735 print '\nCommands:\n' + '\n'.join(sorted(commands))
737 def print_dict(d, header='name', f=stdout, detail=True):
738 header = header if header in d else 'subdir'
739 if header and header in d:
740 f.write('%s\n' %d.pop(header).encode('utf8'))
742 patterns = ['^x_(account|container|object)_meta_(\w+)$']
743 patterns.append(patterns[0].replace('_', '-'))
744 for key, val in sorted(d.items()):
745 f.write('%s: %s\n' % (key.rjust(30), val))
747 def print_list(l, verbose=False, f=stdout, detail=True):
749 #if it's empty string continue
752 if type(elem) == types.DictionaryType:
753 print_dict(elem, f=f, detail=detail)
754 elif type(elem) == types.StringType:
756 elem = elem.split('Traceback')[0]
757 f.write('%s\n' % elem)
759 f.write('%s\n' % elem)
761 def print_versions(data, f=stdout):
762 if 'versions' not in data:
763 f.write('%s\n' %data)
765 f.write('versions:\n')
766 for id, t in data['versions']:
767 f.write('%s @ %s\n' % (str(id).rjust(30), datetime.fromtimestamp(float(t))))
773 cls = class_for_cli_command(name)
774 except (IndexError, KeyError):
778 cmd = cls(name, argv[2:])
781 cmd.execute(*cmd.args)
783 cmd.parser.print_help()
786 status = '%s ' % f.status if f.status else ''
787 print '%s%s' % (status, f.data)
790 if __name__ == '__main__':