3 # Copyright 2011 GRNET S.A. All rights reserved.
5 # Redistribution and use in source and binary forms, with or
6 # without modification, are permitted provided that the following
9 # 1. Redistributions of source code must retain the above
10 # copyright notice, this list of conditions and the following
13 # 2. Redistributions in binary form must reproduce the above
14 # copyright notice, this list of conditions and the following
15 # disclaimer in the documentation and/or other materials
16 # provided with the distribution.
18 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 # POSSIBILITY OF SUCH DAMAGE.
31 # The views and conclusions contained in the software and
32 # documentation are those of the authors and should not be
33 # interpreted as representing official policies, either expressed
34 # or implied, of GRNET S.A.
36 from getpass import getuser
37 from optparse import OptionParser
38 from os import environ
39 from sys import argv, exit, stdin, stdout
40 from datetime import datetime
41 from lib.client import Pithos_Client, Fault
42 from lib.util import get_user, get_auth, get_server, get_api
43 from lib.transfer import upload, download
54 def cli_command(*args):
58 _cli_commands[name] = cls
62 def class_for_cli_command(name):
63 return _cli_commands[name]
65 class Command(object):
68 def __init__(self, name, argv):
69 parser = OptionParser('%%prog %s [options] %s' % (name, self.syntax))
70 parser.add_option('--host', dest='host', metavar='HOST',
71 default=get_server(), help='use server HOST')
72 parser.add_option('--user', dest='user', metavar='USERNAME',
74 help='use account USERNAME')
75 parser.add_option('--token', dest='token', metavar='AUTH',
77 help='use account AUTH')
78 parser.add_option('--api', dest='api', metavar='API',
79 default=get_api(), help='use api API')
80 parser.add_option('-v', action='store_true', dest='verbose',
81 default=False, help='use verbose output')
82 parser.add_option('-d', action='store_true', dest='debug',
83 default=False, help='use debug output')
84 self.add_options(parser)
85 options, args = parser.parse_args(argv)
88 for opt in parser.option_list:
91 val = getattr(options, key)
92 setattr(self, key, val)
94 self.client = Pithos_Client(self.host, self.token, self.user, self.api, self.verbose,
100 def _build_args(self, attrs):
102 for a in [a for a in attrs if getattr(self, a)]:
103 args[a] = getattr(self, a)
106 def add_options(self, parser):
109 def execute(self, *args):
112 @cli_command('list', 'ls')
114 syntax = '[<container>[/<object>]]'
115 description = 'list containers or objects'
117 def add_options(self, parser):
118 parser.add_option('-l', action='store_true', dest='detail',
119 default=False, help='show detailed output')
120 parser.add_option('-n', action='store', type='int', dest='limit',
121 default=10000, help='show limited output')
122 parser.add_option('--marker', action='store', type='str',
123 dest='marker', default=None,
124 help='show output greater then marker')
125 parser.add_option('--prefix', action='store', type='str',
126 dest='prefix', default=None,
127 help='show output starting with prefix')
128 parser.add_option('--delimiter', action='store', type='str',
129 dest='delimiter', default=None,
130 help='show output up to the delimiter')
131 parser.add_option('--path', action='store', type='str',
132 dest='path', default=None,
133 help='show output starting with prefix up to /')
134 parser.add_option('--meta', action='store', type='str',
135 dest='meta', default=None,
136 help='show output having the specified meta keys')
137 parser.add_option('--if-modified-since', action='store', type='str',
138 dest='if_modified_since', default=None,
139 help='show output if modified since then')
140 parser.add_option('--if-unmodified-since', action='store', type='str',
141 dest='if_unmodified_since', default=None,
142 help='show output if not modified since then')
143 parser.add_option('--until', action='store', dest='until',
144 default=None, help='show metadata until that date')
145 parser.add_option('--format', action='store', dest='format',
146 default='%d/%m/%Y', help='format to parse until date')
148 def execute(self, container=None):
150 self.list_objects(container)
152 self.list_containers()
154 def list_containers(self):
155 attrs = ['limit', 'marker', 'if_modified_since',
156 'if_unmodified_since']
157 args = self._build_args(attrs)
158 args['format'] = 'json' if self.detail else 'text'
160 if getattr(self, 'until'):
161 t = _time.strptime(self.until, self.format)
162 args['until'] = int(_time.mktime(t))
164 l = self.client.list_containers(**args)
167 def list_objects(self, container):
170 attrs = ['limit', 'marker', 'prefix', 'delimiter', 'path',
171 'meta', 'if_modified_since', 'if_unmodified_since']
172 args = self._build_args(attrs)
173 args['format'] = 'json' if self.detail else 'text'
176 t = _time.strptime(self.until, self.format)
177 args['until'] = int(_time.mktime(t))
179 container, sep, object = container.partition('/')
184 #if request with meta quering disable trash filtering
185 show_trashed = True if self.meta else False
186 l = self.client.list_objects(container, **args)
187 print_list(l, detail=self.detail)
191 syntax = '[<container>[/<object>]]'
192 description = 'get account/container/object metadata'
194 def add_options(self, parser):
195 parser.add_option('-r', action='store_true', dest='restricted',
196 default=False, help='show only user defined metadata')
197 parser.add_option('--until', action='store', dest='until',
198 default=None, help='show metadata until that date')
199 parser.add_option('--format', action='store', dest='format',
200 default='%d/%m/%Y', help='format to parse until date')
201 parser.add_option('--version', action='store', dest='version',
202 default=None, help='show specific version \
203 (applies only for objects)')
205 def execute(self, path=''):
206 container, sep, object = path.partition('/')
207 args = {'restricted': self.restricted}
208 if getattr(self, 'until'):
209 t = _time.strptime(self.until, self.format)
210 args['until'] = int(_time.mktime(t))
213 meta = self.client.retrieve_object_metadata(container, object,
217 meta = self.client.retrieve_container_metadata(container, **args)
219 meta = self.client.retrieve_account_metadata(**args)
221 print 'Entity does not exist'
223 print_dict(meta, header=None)
225 @cli_command('create')
226 class CreateContainer(Command):
227 syntax = '<container> [key=val] [...]'
228 description = 'create a container'
230 def execute(self, container, *args):
233 key, sep, val = arg.partition('=')
235 ret = self.client.create_container(container, **meta)
237 print 'Container already exists'
239 @cli_command('delete', 'rm')
240 class Delete(Command):
241 syntax = '<container>[/<object>]'
242 description = 'delete a container or an object'
244 def add_options(self, parser):
245 parser.add_option('--until', action='store', dest='until',
246 default=None, help='remove history until that date')
247 parser.add_option('--format', action='store', dest='format',
248 default='%d/%m/%Y', help='format to parse until date')
250 def execute(self, path):
251 container, sep, object = path.partition('/')
253 if getattr(self, 'until'):
254 t = _time.strptime(self.until, self.format)
255 until = int(_time.mktime(t))
258 self.client.delete_object(container, object, until)
260 self.client.delete_container(container, until)
263 class GetObject(Command):
264 syntax = '<container>/<object>'
265 description = 'get the data of an object'
267 def add_options(self, parser):
268 parser.add_option('-l', action='store_true', dest='detail',
269 default=False, help='show detailed output')
270 parser.add_option('--range', action='store', dest='range',
271 default=None, help='show range of data')
272 parser.add_option('--if-range', action='store', dest='if_range',
273 default=None, help='show range of data')
274 parser.add_option('--if-match', action='store', dest='if_match',
275 default=None, help='show output if ETags match')
276 parser.add_option('--if-none-match', action='store',
277 dest='if_none_match', default=None,
278 help='show output if ETags don\'t match')
279 parser.add_option('--if-modified-since', action='store', type='str',
280 dest='if_modified_since', default=None,
281 help='show output if modified since then')
282 parser.add_option('--if-unmodified-since', action='store', type='str',
283 dest='if_unmodified_since', default=None,
284 help='show output if not modified since then')
285 parser.add_option('-o', action='store', type='str',
286 dest='file', default=None,
287 help='save output in file')
288 parser.add_option('--version', action='store', type='str',
289 dest='version', default=None,
290 help='get the specific \
292 parser.add_option('--versionlist', action='store_true',
293 dest='versionlist', default=False,
294 help='get the full object version list')
296 def execute(self, path):
297 attrs = ['if_match', 'if_none_match', 'if_modified_since',
298 'if_unmodified_since']
299 args = self._build_args(attrs)
300 args['format'] = 'json' if self.detail else 'text'
302 args['range'] = 'bytes=%s' % self.range
303 if getattr(self, 'if_range'):
304 args['if-range'] = 'If-Range:%s' % getattr(self, 'if_range')
306 container, sep, object = path.partition('/')
309 if 'detail' in args.keys():
313 data = self.client.retrieve_object_versionlist(container, object, **args)
315 data = self.client.retrieve_object_version(container, object,
316 self.version, **args)
318 data = self.client.retrieve_object(container, object, **args)
320 f = self.file and open(self.file, 'w') or stdout
323 print_versions(data, f=f)
325 print_dict(data, f=f)
330 @cli_command('mkdir')
331 class PutMarker(Command):
332 syntax = '<container>/<directory marker>'
333 description = 'create a directory marker'
335 def execute(self, path):
336 container, sep, object = path.partition('/')
337 self.client.create_directory_marker(container, object)
340 class PutObject(Command):
341 syntax = '<container>/<object> [key=val] [...]'
342 description = 'create/override object'
344 def add_options(self, parser):
345 parser.add_option('--use_hashes', action='store_true', dest='use_hashes',
346 default=False, help='provide hashmap instead of data')
347 parser.add_option('--chunked', action='store_true', dest='chunked',
348 default=False, help='set chunked transfer mode')
349 parser.add_option('--etag', action='store', dest='etag',
350 default=None, help='check written data')
351 parser.add_option('--content-encoding', action='store',
352 dest='content_encoding', default=None,
353 help='provide the object MIME content type')
354 parser.add_option('--content-disposition', action='store', type='str',
355 dest='content_disposition', default=None,
356 help='provide the presentation style of the object')
357 #parser.add_option('-S', action='store',
358 # dest='segment_size', default=False,
359 # help='use for large file support')
360 parser.add_option('--manifest', action='store',
361 dest='x_object_manifest', default=None,
362 help='upload a manifestation file')
363 parser.add_option('--content-type', action='store',
364 dest='content_type', default=None,
365 help='create object with specific content type')
366 parser.add_option('--sharing', action='store',
367 dest='x_object_sharing', default=None,
368 help='define sharing object policy')
369 parser.add_option('-f', action='store',
370 dest='srcpath', default=None,
371 help='file descriptor to read from (pass - for standard input)')
372 parser.add_option('--public', action='store_true',
373 dest='x_object_public', default=False,
374 help='make object publicly accessible')
376 def execute(self, path, *args):
377 if path.find('=') != -1:
378 raise Fault('Missing path argument')
380 #prepare user defined meta
383 key, sep, val = arg.partition('=')
386 attrs = ['etag', 'content_encoding', 'content_disposition',
387 'content_type', 'x_object_sharing', 'x_object_public']
388 args = self._build_args(attrs)
390 container, sep, object = path.partition('/')
394 f = open(self.srcpath) if self.srcpath != '-' else stdin
396 if self.use_hashes and not f:
397 raise Fault('Illegal option combination')
400 self.client.create_object_using_chunks(container, object, f,
402 elif self.use_hashes:
405 hashmap = json.loads()
406 self.client.create_object_by_hashmap(container, object, hashmap,
409 print "Expected object"
410 elif self.x_object_manifest:
411 self.client.create_manifestation(container, object, self.x_object_manifest)
413 self.client.create_zero_length_object(container, object, meta=meta, **args)
415 self.client.create_object(container, object, f, meta=meta, **args)
419 @cli_command('copy', 'cp')
420 class CopyObject(Command):
421 syntax = '<src container>/<src object> [<dst container>/]<dst object> [key=val] [...]'
422 description = 'copy an object to a different location'
424 def add_options(self, parser):
425 parser.add_option('--version', action='store',
426 dest='version', default=False,
427 help='copy specific version')
428 parser.add_option('--public', action='store_true',
429 dest='public', default=False,
430 help='make object publicly accessible')
431 parser.add_option('--content-type', action='store',
432 dest='content_type', default=None,
433 help='change object\'s content type')
435 def execute(self, src, dst, *args):
436 src_container, sep, src_object = src.partition('/')
437 dst_container, sep, dst_object = dst.partition('/')
439 #prepare user defined meta
442 key, sep, val = arg.partition('=')
446 dst_container = src_container
449 args = {'content_type':self.content_type} if self.content_type else {}
450 self.client.copy_object(src_container, src_object, dst_container,
451 dst_object, meta, self.public, self.version,
455 class SetMeta(Command):
456 syntax = '[<container>[/<object>]] key=val [key=val] [...]'
457 description = 'set account/container/object metadata'
459 def execute(self, path, *args):
460 #in case of account fix the args
461 if path.find('=') != -1:
468 key, sep, val = arg.partition('=')
469 meta[key.strip()] = val.strip()
470 container, sep, object = path.partition('/')
472 self.client.update_object_metadata(container, object, **meta)
474 self.client.update_container_metadata(container, **meta)
476 self.client.update_account_metadata(**meta)
478 @cli_command('update')
479 class UpdateObject(Command):
480 syntax = '<container>/<object> path [key=val] [...]'
481 description = 'update object metadata/data (default mode: append)'
483 def add_options(self, parser):
484 parser.add_option('-a', action='store_true', dest='append',
485 default=True, help='append data')
486 parser.add_option('--offset', action='store',
488 default=None, help='starting offest to be updated')
489 parser.add_option('--range', action='store', dest='content-range',
490 default=None, help='range of data to be updated')
491 parser.add_option('--chunked', action='store_true', dest='chunked',
492 default=False, help='set chunked transfer mode')
493 parser.add_option('--content-encoding', action='store',
494 dest='content_encoding', default=None,
495 help='provide the object MIME content type')
496 parser.add_option('--content-disposition', action='store', type='str',
497 dest='content_disposition', default=None,
498 help='provide the presentation style of the object')
499 parser.add_option('--manifest', action='store', type='str',
500 dest='x_object_manifest', default=None,
501 help='use for large file support')
502 parser.add_option('--sharing', action='store',
503 dest='x_object_sharing', default=None,
504 help='define sharing object policy')
505 parser.add_option('--nosharing', action='store_true',
506 dest='no_sharing', default=None,
507 help='clear object sharing policy')
508 parser.add_option('-f', action='store',
509 dest='srcpath', default=None,
510 help='file descriptor to read from: pass - for standard input')
511 parser.add_option('--public', action='store_true',
512 dest='x_object_public', default=False,
513 help='make object publicly accessible')
514 parser.add_option('--replace', action='store_true',
515 dest='replace', default=False,
516 help='override metadata')
518 def execute(self, path, *args):
519 if path.find('=') != -1:
520 raise Fault('Missing path argument')
522 #prepare user defined meta
525 key, sep, val = arg.partition('=')
529 attrs = ['content_encoding', 'content_disposition', 'x_object_sharing',
530 'x_object_public', 'replace']
531 args = self._build_args(attrs)
534 args['x_object_sharing'] = ''
536 container, sep, object = path.partition('/')
540 f = open(self.srcpath) if self.srcpath != '-' else stdin
543 self.client.update_object_using_chunks(container, object, f,
546 self.client.update_object(container, object, f, meta=meta, **args)
550 @cli_command('move', 'mv')
551 class MoveObject(Command):
552 syntax = '<src container>/<src object> [<dst container>/]<dst object>'
553 description = 'move an object to a different location'
555 def add_options(self, parser):
556 parser.add_option('--public', action='store_true',
557 dest='public', default=False,
558 help='make object publicly accessible')
559 parser.add_option('--content-type', action='store',
560 dest='content_type', default=None,
561 help='change object\'s content type')
563 def execute(self, src, dst, *args):
564 src_container, sep, src_object = src.partition('/')
565 dst_container, sep, dst_object = dst.partition('/')
567 dst_container = src_container
570 #prepare user defined meta
573 key, sep, val = arg.partition('=')
576 args = {'content_type':self.content_type} if self.content_type else {}
577 self.client.move_object(src_container, src_object, dst_container,
578 dst_object, meta, self.public, **args)
580 @cli_command('unset')
581 class UnsetObject(Command):
582 syntax = '<container>/[<object>] key [key] [...]'
583 description = 'delete metadata info'
585 def execute(self, path, *args):
586 #in case of account fix the args
595 container, sep, object = path.partition('/')
597 self.client.delete_object_metadata(container, object, meta)
599 self.client.delete_container_metadata(container, meta)
601 self.client.delete_account_metadata(meta)
603 @cli_command('group')
604 class CreateGroup(Command):
605 syntax = 'key=val [key=val] [...]'
606 description = 'create account groups'
608 def execute(self, *args):
611 key, sep, val = arg.partition('=')
613 self.client.set_account_groups(**groups)
615 @cli_command('ungroup')
616 class DeleteGroup(Command):
617 syntax = 'key [key] [...]'
618 description = 'delete account groups'
620 def execute(self, *args):
624 self.client.unset_account_groups(groups)
626 @cli_command('policy')
627 class SetPolicy(Command):
628 syntax = 'container key=val [key=val] [...]'
629 description = 'set container policies'
631 def execute(self, path, *args):
632 if path.find('=') != -1:
633 raise Fault('Missing container argument')
635 container, sep, object = path.partition('/')
638 raise Fault('Only containers have policies')
642 key, sep, val = arg.partition('=')
645 self.client.set_container_policies(container, **policies)
647 @cli_command('publish')
648 class PublishObject(Command):
649 syntax = '<container>/<object>'
650 description = 'publish an object'
652 def execute(self, src):
653 src_container, sep, src_object = src.partition('/')
655 self.client.publish_object(src_container, src_object)
657 @cli_command('unpublish')
658 class UnpublishObject(Command):
659 syntax = '<container>/<object>'
660 description = 'unpublish an object'
662 def execute(self, src):
663 src_container, sep, src_object = src.partition('/')
665 self.client.unpublish_object(src_container, src_object)
667 @cli_command('sharing')
668 class SharingObject(Command):
669 syntax = 'list users sharing objects with the user'
670 description = 'list user accounts sharing objects with the user'
672 def add_options(self, parser):
673 parser.add_option('-l', action='store_true', dest='detail',
674 default=False, help='show detailed output')
675 parser.add_option('-n', action='store', type='int', dest='limit',
676 default=10000, help='show limited output')
677 parser.add_option('--marker', action='store', type='str',
678 dest='marker', default=None,
679 help='show output greater then marker')
683 attrs = ['limit', 'marker']
684 args = self._build_args(attrs)
685 args['format'] = 'json' if self.detail else 'text'
687 print_list(self.client.list_shared_by_others(**args))
691 syntax = '<file> <container>[/<prefix>]'
692 description = 'upload file to container (using prefix)'
694 def execute(self, file, path):
695 container, sep, prefix = path.partition('/')
696 upload(self.client, file, container, prefix)
698 @cli_command('receive')
699 class Receive(Command):
700 syntax = '<container>/<object> <file>'
701 description = 'download object to file'
703 def execute(self, path, file):
704 container, sep, object = path.partition('/')
705 download(self.client, container, object, file)
708 cmd = Command('', [])
710 parser.usage = '%prog <command> [options]'
714 for cls in set(_cli_commands.values()):
715 name = ', '.join(cls.commands)
716 description = getattr(cls, 'description', '')
717 commands.append(' %s %s' % (name.ljust(12), description))
718 print '\nCommands:\n' + '\n'.join(sorted(commands))
720 def print_dict(d, header='name', f=stdout, detail=True):
721 header = header if header in d else 'subdir'
722 if header and header in d:
723 f.write('%s\n' %d.pop(header).encode('utf8'))
725 patterns = ['^x_(account|container|object)_meta_(\w+)$']
726 patterns.append(patterns[0].replace('_', '-'))
727 for key, val in sorted(d.items()):
728 f.write('%s: %s\n' % (key.rjust(30), val))
730 def print_list(l, verbose=False, f=stdout, detail=True):
732 #if it's empty string continue
735 if type(elem) == types.DictionaryType:
736 print_dict(elem, f=f, detail=detail)
737 elif type(elem) == types.StringType:
739 elem = elem.split('Traceback')[0]
740 f.write('%s\n' % elem)
742 f.write('%s\n' % elem)
744 def print_versions(data, f=stdout):
745 if 'versions' not in data:
746 f.write('%s\n' %data)
748 f.write('versions:\n')
749 for id, t in data['versions']:
750 f.write('%s @ %s\n' % (str(id).rjust(30), datetime.fromtimestamp(float(t))))
755 cls = class_for_cli_command(name)
756 except (IndexError, KeyError):
760 cmd = cls(name, argv[2:])
763 cmd.execute(*cmd.args)
765 cmd.parser.print_help()
768 status = f.status and '%s ' % f.status or ''
769 print '%s%s' % (status, f.data)
771 if __name__ == '__main__':