3 # Copyright 2011-2012 GRNET S.A. All rights reserved.
5 # Redistribution and use in source and binary forms, with or
6 # without modification, are permitted provided that the following
9 # 1. Redistributions of source code must retain the above
10 # copyright notice, this list of conditions and the following
13 # 2. Redistributions in binary form must reproduce the above
14 # copyright notice, this list of conditions and the following
15 # disclaimer in the documentation and/or other materials
16 # provided with the distribution.
18 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 # POSSIBILITY OF SUCH DAMAGE.
31 # The views and conclusions contained in the software and
32 # documentation are those of the authors and should not be
33 # interpreted as representing official policies, either expressed
34 # or implied, of GRNET S.A.
36 from getpass import getuser
37 from optparse import OptionParser
38 from os import environ
39 from sys import argv, exit, stdin, stdout
40 from datetime import datetime
42 from pithos.lib.client import Pithos_Client, Fault
43 from pithos.lib.util import get_user, get_auth, get_server, get_api
44 from pithos.lib.transfer import upload, download
55 def cli_command(*args):
59 _cli_commands[name] = cls
63 def class_for_cli_command(name):
64 return _cli_commands[name]
66 class Command(object):
69 def __init__(self, name, argv):
70 parser = OptionParser('%%prog %s [options] %s' % (name, self.syntax))
71 parser.add_option('--host', dest='host', metavar='HOST',
72 default=get_server(), help='use server HOST')
73 parser.add_option('--user', dest='user', metavar='USERNAME',
75 help='use account USERNAME')
76 parser.add_option('--token', dest='token', metavar='AUTH',
78 help='use account AUTH')
79 parser.add_option('--api', dest='api', metavar='API',
80 default=get_api(), help='use api API')
81 parser.add_option('-v', action='store_true', dest='verbose',
82 default=False, help='use verbose output')
83 parser.add_option('-d', action='store_true', dest='debug',
84 default=False, help='use debug output')
85 self.add_options(parser)
86 options, args = parser.parse_args(argv)
89 for opt in parser.option_list:
92 val = getattr(options, key)
93 setattr(self, key, val)
95 self.client = Pithos_Client(self.host, self.token, self.user, self.api, self.verbose,
101 def _build_args(self, attrs):
103 for a in [a for a in attrs if getattr(self, a)]:
104 args[a] = getattr(self, a)
107 def add_options(self, parser):
110 def execute(self, *args):
113 @cli_command('list', 'ls')
115 syntax = '[<container>[/<object>]]'
116 description = 'list containers or objects'
118 def add_options(self, parser):
119 parser.add_option('-l', action='store_true', dest='detail',
120 default=False, help='show detailed output')
121 parser.add_option('-n', action='store', type='int', dest='limit',
122 default=10000, help='show limited output')
123 parser.add_option('--marker', action='store', type='str',
124 dest='marker', default=None,
125 help='show output greater then marker')
126 parser.add_option('--prefix', action='store', type='str',
127 dest='prefix', default=None,
128 help='show output starting with prefix')
129 parser.add_option('--delimiter', action='store', type='str',
130 dest='delimiter', default=None,
131 help='show output up to the delimiter')
132 parser.add_option('--path', action='store', type='str',
133 dest='path', default=None,
134 help='show output starting with prefix up to /')
135 parser.add_option('--meta', action='store', type='str',
136 dest='meta', default=None,
137 help='show output having the specified meta keys')
138 parser.add_option('--if-modified-since', action='store', type='str',
139 dest='if_modified_since', default=None,
140 help='show output if modified since then')
141 parser.add_option('--if-unmodified-since', action='store', type='str',
142 dest='if_unmodified_since', default=None,
143 help='show output if not modified since then')
144 parser.add_option('--until', action='store', dest='until',
145 default=None, help='show metadata until that date')
146 parser.add_option('--format', action='store', dest='format',
147 default='%d/%m/%Y', help='format to parse until date')
149 def execute(self, container=None):
151 self.list_objects(container)
153 self.list_containers()
155 def list_containers(self):
156 attrs = ['limit', 'marker', 'if_modified_since',
157 'if_unmodified_since']
158 args = self._build_args(attrs)
159 args['format'] = 'json' if self.detail else 'text'
161 if getattr(self, 'until'):
162 t = _time.strptime(self.until, self.format)
163 args['until'] = int(_time.mktime(t))
165 l = self.client.list_containers(**args)
168 def list_objects(self, container):
171 attrs = ['limit', 'marker', 'prefix', 'delimiter', 'path',
172 'meta', 'if_modified_since', 'if_unmodified_since']
173 args = self._build_args(attrs)
174 args['format'] = 'json' if self.detail else 'text'
177 t = _time.strptime(self.until, self.format)
178 args['until'] = int(_time.mktime(t))
180 container, sep, object = container.partition('/')
185 #if request with meta quering disable trash filtering
186 show_trashed = True if self.meta else False
187 l = self.client.list_objects(container, **args)
188 print_list(l, detail=self.detail)
192 syntax = '[<container>[/<object>]]'
193 description = 'get account/container/object metadata'
195 def add_options(self, parser):
196 parser.add_option('-r', action='store_true', dest='restricted',
197 default=False, help='show only user defined metadata')
198 parser.add_option('--until', action='store', dest='until',
199 default=None, help='show metadata until that date')
200 parser.add_option('--format', action='store', dest='format',
201 default='%d/%m/%Y', help='format to parse until date')
202 parser.add_option('--version', action='store', dest='version',
203 default=None, help='show specific version \
204 (applies only for objects)')
206 def execute(self, path=''):
207 container, sep, object = path.partition('/')
208 args = {'restricted': self.restricted}
209 if getattr(self, 'until'):
210 t = _time.strptime(self.until, self.format)
211 args['until'] = int(_time.mktime(t))
214 meta = self.client.retrieve_object_metadata(container, object,
218 meta = self.client.retrieve_container_metadata(container, **args)
220 meta = self.client.retrieve_account_metadata(**args)
222 print 'Entity does not exist'
224 print_dict(meta, header=None)
226 @cli_command('create')
227 class CreateContainer(Command):
228 syntax = '<container> [key=val] [...]'
229 description = 'create a container'
231 def execute(self, container, *args):
234 key, sep, val = arg.partition('=')
236 ret = self.client.create_container(container, **meta)
238 print 'Container already exists'
240 @cli_command('delete', 'rm')
241 class Delete(Command):
242 syntax = '<container>[/<object>]'
243 description = 'delete a container or an object'
245 def add_options(self, parser):
246 parser.add_option('--until', action='store', dest='until',
247 default=None, help='remove history until that date')
248 parser.add_option('--format', action='store', dest='format',
249 default='%d/%m/%Y', help='format to parse until date')
251 def execute(self, path):
252 container, sep, object = path.partition('/')
254 if getattr(self, 'until'):
255 t = _time.strptime(self.until, self.format)
256 until = int(_time.mktime(t))
259 self.client.delete_object(container, object, until)
261 self.client.delete_container(container, until)
264 class GetObject(Command):
265 syntax = '<container>/<object>'
266 description = 'get the data of an object'
268 def add_options(self, parser):
269 parser.add_option('-l', action='store_true', dest='detail',
270 default=False, help='show detailed output')
271 parser.add_option('--range', action='store', dest='range',
272 default=None, help='show range of data')
273 parser.add_option('--if-range', action='store', dest='if_range',
274 default=None, help='show range of data')
275 parser.add_option('--if-match', action='store', dest='if_match',
276 default=None, help='show output if ETags match')
277 parser.add_option('--if-none-match', action='store',
278 dest='if_none_match', default=None,
279 help='show output if ETags don\'t match')
280 parser.add_option('--if-modified-since', action='store', type='str',
281 dest='if_modified_since', default=None,
282 help='show output if modified since then')
283 parser.add_option('--if-unmodified-since', action='store', type='str',
284 dest='if_unmodified_since', default=None,
285 help='show output if not modified since then')
286 parser.add_option('-o', action='store', type='str',
287 dest='file', default=None,
288 help='save output in file')
289 parser.add_option('--version', action='store', type='str',
290 dest='version', default=None,
291 help='get the specific \
293 parser.add_option('--versionlist', action='store_true',
294 dest='versionlist', default=False,
295 help='get the full object version list')
297 def execute(self, path):
298 attrs = ['if_match', 'if_none_match', 'if_modified_since',
299 'if_unmodified_since']
300 args = self._build_args(attrs)
301 args['format'] = 'json' if self.detail else 'text'
303 args['range'] = 'bytes=%s' % self.range
304 if getattr(self, 'if_range'):
305 args['if-range'] = 'If-Range:%s' % getattr(self, 'if_range')
307 container, sep, object = path.partition('/')
310 if 'detail' in args.keys():
314 data = self.client.retrieve_object_versionlist(container, object, **args)
316 data = self.client.retrieve_object_version(container, object,
317 self.version, **args)
319 data = self.client.retrieve_object(container, object, **args)
321 f = open(self.file, 'w') if self.file else stdout
324 print_versions(data, f=f)
326 print_dict(data, f=f)
331 @cli_command('mkdir')
332 class PutMarker(Command):
333 syntax = '<container>/<directory marker>'
334 description = 'create a directory marker'
336 def execute(self, path):
337 container, sep, object = path.partition('/')
338 self.client.create_directory_marker(container, object)
341 class PutObject(Command):
342 syntax = '<container>/<object> [key=val] [...]'
343 description = 'create/override object'
345 def add_options(self, parser):
346 parser.add_option('--use_hashes', action='store_true', dest='use_hashes',
347 default=False, help='provide hashmap instead of data')
348 parser.add_option('--chunked', action='store_true', dest='chunked',
349 default=False, help='set chunked transfer mode')
350 parser.add_option('--etag', action='store', dest='etag',
351 default=None, help='check written data')
352 parser.add_option('--content-encoding', action='store',
353 dest='content_encoding', default=None,
354 help='provide the object MIME content type')
355 parser.add_option('--content-disposition', action='store', type='str',
356 dest='content_disposition', default=None,
357 help='provide the presentation style of the object')
358 #parser.add_option('-S', action='store',
359 # dest='segment_size', default=False,
360 # help='use for large file support')
361 parser.add_option('--manifest', action='store',
362 dest='x_object_manifest', default=None,
363 help='upload a manifestation file')
364 parser.add_option('--content-type', action='store',
365 dest='content_type', default=None,
366 help='create object with specific content type')
367 parser.add_option('--sharing', action='store',
368 dest='x_object_sharing', default=None,
369 help='define sharing object policy')
370 parser.add_option('-f', action='store',
371 dest='srcpath', default=None,
372 help='file descriptor to read from (pass - for standard input)')
373 parser.add_option('--public', action='store_true',
374 dest='x_object_public', default=False,
375 help='make object publicly accessible')
377 def execute(self, path, *args):
378 if path.find('=') != -1:
379 raise Fault('Missing path argument')
381 #prepare user defined meta
384 key, sep, val = arg.partition('=')
387 attrs = ['etag', 'content_encoding', 'content_disposition',
388 'content_type', 'x_object_sharing', 'x_object_public']
389 args = self._build_args(attrs)
391 container, sep, object = path.partition('/')
395 f = open(self.srcpath) if self.srcpath != '-' else stdin
397 if self.use_hashes and not f:
398 raise Fault('Illegal option combination')
401 self.client.create_object_using_chunks(container, object, f,
403 elif self.use_hashes:
406 hashmap = json.loads()
407 self.client.create_object_by_hashmap(container, object, hashmap,
410 print "Expected object"
411 elif self.x_object_manifest:
412 self.client.create_manifestation(container, object, self.x_object_manifest)
414 self.client.create_zero_length_object(container, object, meta=meta, **args)
416 self.client.create_object(container, object, f, meta=meta, **args)
420 @cli_command('copy', 'cp')
421 class CopyObject(Command):
422 syntax = '<src container>/<src object> [<dst container>/]<dst object> [key=val] [...]'
423 description = 'copy an object to a different location'
425 def add_options(self, parser):
426 parser.add_option('--version', action='store',
427 dest='version', default=False,
428 help='copy specific version')
429 parser.add_option('--public', action='store_true',
430 dest='public', default=False,
431 help='make object publicly accessible')
432 parser.add_option('--content-type', action='store',
433 dest='content_type', default=None,
434 help='change object\'s content type')
436 def execute(self, src, dst, *args):
437 src_container, sep, src_object = src.partition('/')
438 dst_container, sep, dst_object = dst.partition('/')
440 #prepare user defined meta
443 key, sep, val = arg.partition('=')
447 dst_container = src_container
450 args = {'content_type':self.content_type} if self.content_type else {}
451 self.client.copy_object(src_container, src_object, dst_container,
452 dst_object, meta, self.public, self.version,
456 class SetMeta(Command):
457 syntax = '[<container>[/<object>]] key=val [key=val] [...]'
458 description = 'set account/container/object metadata'
460 def execute(self, path, *args):
461 #in case of account fix the args
462 if path.find('=') != -1:
469 key, sep, val = arg.partition('=')
470 meta[key.strip()] = val.strip()
471 container, sep, object = path.partition('/')
473 self.client.update_object_metadata(container, object, **meta)
475 self.client.update_container_metadata(container, **meta)
477 self.client.update_account_metadata(**meta)
479 @cli_command('update')
480 class UpdateObject(Command):
481 syntax = '<container>/<object> path [key=val] [...]'
482 description = 'update object metadata/data (default mode: append)'
484 def add_options(self, parser):
485 parser.add_option('-a', action='store_true', dest='append',
486 default=True, help='append data')
487 parser.add_option('--offset', action='store',
489 default=None, help='starting offest to be updated')
490 parser.add_option('--range', action='store', dest='content-range',
491 default=None, help='range of data to be updated')
492 parser.add_option('--chunked', action='store_true', dest='chunked',
493 default=False, help='set chunked transfer mode')
494 parser.add_option('--content-encoding', action='store',
495 dest='content_encoding', default=None,
496 help='provide the object MIME content type')
497 parser.add_option('--content-disposition', action='store', type='str',
498 dest='content_disposition', default=None,
499 help='provide the presentation style of the object')
500 parser.add_option('--manifest', action='store', type='str',
501 dest='x_object_manifest', default=None,
502 help='use for large file support')
503 parser.add_option('--sharing', action='store',
504 dest='x_object_sharing', default=None,
505 help='define sharing object policy')
506 parser.add_option('--nosharing', action='store_true',
507 dest='no_sharing', default=None,
508 help='clear object sharing policy')
509 parser.add_option('-f', action='store',
510 dest='srcpath', default=None,
511 help='file descriptor to read from: pass - for standard input')
512 parser.add_option('--public', action='store_true',
513 dest='x_object_public', default=False,
514 help='make object publicly accessible')
515 parser.add_option('--replace', action='store_true',
516 dest='replace', default=False,
517 help='override metadata')
519 def execute(self, path, *args):
520 if path.find('=') != -1:
521 raise Fault('Missing path argument')
523 #prepare user defined meta
526 key, sep, val = arg.partition('=')
530 attrs = ['content_encoding', 'content_disposition', 'x_object_sharing',
531 'x_object_public', 'replace']
532 args = self._build_args(attrs)
535 args['x_object_sharing'] = ''
537 container, sep, object = path.partition('/')
541 f = open(self.srcpath) if self.srcpath != '-' else stdin
544 self.client.update_object_using_chunks(container, object, f,
547 self.client.update_object(container, object, f, meta=meta, **args)
551 @cli_command('move', 'mv')
552 class MoveObject(Command):
553 syntax = '<src container>/<src object> [<dst container>/]<dst object>'
554 description = 'move an object to a different location'
556 def add_options(self, parser):
557 parser.add_option('--public', action='store_true',
558 dest='public', default=False,
559 help='make object publicly accessible')
560 parser.add_option('--content-type', action='store',
561 dest='content_type', default=None,
562 help='change object\'s content type')
564 def execute(self, src, dst, *args):
565 src_container, sep, src_object = src.partition('/')
566 dst_container, sep, dst_object = dst.partition('/')
568 dst_container = src_container
571 #prepare user defined meta
574 key, sep, val = arg.partition('=')
577 args = {'content_type':self.content_type} if self.content_type else {}
578 self.client.move_object(src_container, src_object, dst_container,
579 dst_object, meta, self.public, **args)
581 @cli_command('unset')
582 class UnsetObject(Command):
583 syntax = '<container>/[<object>] key [key] [...]'
584 description = 'delete metadata info'
586 def execute(self, path, *args):
587 #in case of account fix the args
596 container, sep, object = path.partition('/')
598 self.client.delete_object_metadata(container, object, meta)
600 self.client.delete_container_metadata(container, meta)
602 self.client.delete_account_metadata(meta)
604 @cli_command('group')
605 class CreateGroup(Command):
606 syntax = 'key=val [key=val] [...]'
607 description = 'create account groups'
609 def execute(self, *args):
612 key, sep, val = arg.partition('=')
614 self.client.set_account_groups(**groups)
616 @cli_command('ungroup')
617 class DeleteGroup(Command):
618 syntax = 'key [key] [...]'
619 description = 'delete account groups'
621 def execute(self, *args):
625 self.client.unset_account_groups(groups)
627 @cli_command('policy')
628 class SetPolicy(Command):
629 syntax = 'container key=val [key=val] [...]'
630 description = 'set container policies'
632 def execute(self, path, *args):
633 if path.find('=') != -1:
634 raise Fault('Missing container argument')
636 container, sep, object = path.partition('/')
639 raise Fault('Only containers have policies')
643 key, sep, val = arg.partition('=')
646 self.client.set_container_policies(container, **policies)
648 @cli_command('publish')
649 class PublishObject(Command):
650 syntax = '<container>/<object>'
651 description = 'publish an object'
653 def execute(self, src):
654 src_container, sep, src_object = src.partition('/')
656 self.client.publish_object(src_container, src_object)
658 @cli_command('unpublish')
659 class UnpublishObject(Command):
660 syntax = '<container>/<object>'
661 description = 'unpublish an object'
663 def execute(self, src):
664 src_container, sep, src_object = src.partition('/')
666 self.client.unpublish_object(src_container, src_object)
668 @cli_command('sharing')
669 class SharingObject(Command):
670 syntax = 'list users sharing objects with the user'
671 description = 'list user accounts sharing objects with the user'
673 def add_options(self, parser):
674 parser.add_option('-l', action='store_true', dest='detail',
675 default=False, help='show detailed output')
676 parser.add_option('-n', action='store', type='int', dest='limit',
677 default=10000, help='show limited output')
678 parser.add_option('--marker', action='store', type='str',
679 dest='marker', default=None,
680 help='show output greater then marker')
684 attrs = ['limit', 'marker']
685 args = self._build_args(attrs)
686 args['format'] = 'json' if self.detail else 'text'
688 print_list(self.client.list_shared_by_others(**args))
692 syntax = '<file> <container>[/<prefix>]'
693 description = 'upload file to container (using prefix)'
695 def execute(self, file, path):
696 container, sep, prefix = path.partition('/')
697 upload(self.client, file, container, prefix)
699 @cli_command('receive')
700 class Receive(Command):
701 syntax = '<container>/<object> <file>'
702 description = 'download object to file'
704 def execute(self, path, file):
705 container, sep, object = path.partition('/')
706 download(self.client, container, object, file)
709 cmd = Command('', [])
711 parser.usage = '%prog <command> [options]'
715 for cls in set(_cli_commands.values()):
716 name = ', '.join(cls.commands)
717 description = getattr(cls, 'description', '')
718 commands.append(' %s %s' % (name.ljust(12), description))
719 print '\nCommands:\n' + '\n'.join(sorted(commands))
721 def print_dict(d, header='name', f=stdout, detail=True):
722 header = header if header in d else 'subdir'
723 if header and header in d:
724 f.write('%s\n' %d.pop(header).encode('utf8'))
726 patterns = ['^x_(account|container|object)_meta_(\w+)$']
727 patterns.append(patterns[0].replace('_', '-'))
728 for key, val in sorted(d.items()):
729 f.write('%s: %s\n' % (key.rjust(30), val))
731 def print_list(l, verbose=False, f=stdout, detail=True):
733 #if it's empty string continue
736 if type(elem) == types.DictionaryType:
737 print_dict(elem, f=f, detail=detail)
738 elif type(elem) == types.StringType:
740 elem = elem.split('Traceback')[0]
741 f.write('%s\n' % elem)
743 f.write('%s\n' % elem)
745 def print_versions(data, f=stdout):
746 if 'versions' not in data:
747 f.write('%s\n' %data)
749 f.write('versions:\n')
750 for id, t in data['versions']:
751 f.write('%s @ %s\n' % (str(id).rjust(30), datetime.fromtimestamp(float(t))))
756 cls = class_for_cli_command(name)
757 except (IndexError, KeyError):
761 cmd = cls(name, argv[2:])
764 cmd.execute(*cmd.args)
766 cmd.parser.print_help()
769 status = '%s ' % f.status if f.status else ''
770 print '%s%s' % (status, f.data)
772 if __name__ == '__main__':