3 # Copyright 2011 GRNET S.A. All rights reserved.
5 # Redistribution and use in source and binary forms, with or
6 # without modification, are permitted provided that the following
9 # 1. Redistributions of source code must retain the above
10 # copyright notice, this list of conditions and the following
13 # 2. Redistributions in binary form must reproduce the above
14 # copyright notice, this list of conditions and the following
15 # disclaimer in the documentation and/or other materials
16 # provided with the distribution.
18 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 # POSSIBILITY OF SUCH DAMAGE.
31 # The views and conclusions contained in the software and
32 # documentation are those of the authors and should not be
33 # interpreted as representing official policies, either expressed
34 # or implied, of GRNET S.A.
36 from getpass import getuser
37 from optparse import OptionParser
38 from os import environ
39 from sys import argv, exit, stdin, stdout
40 from datetime import datetime
41 from lib.client import Pithos_Client, Fault
42 from lib.util import get_user, get_auth, get_server, get_api
53 def cli_command(*args):
57 _cli_commands[name] = cls
61 def class_for_cli_command(name):
62 return _cli_commands[name]
64 class Command(object):
67 def __init__(self, name, argv):
68 parser = OptionParser('%%prog %s [options] %s' % (name, self.syntax))
69 parser.add_option('--host', dest='host', metavar='HOST',
70 default=get_server(), help='use server HOST')
71 parser.add_option('--user', dest='user', metavar='USERNAME',
73 help='use account USERNAME')
74 parser.add_option('--token', dest='token', metavar='AUTH',
76 help='use account AUTH')
77 parser.add_option('--api', dest='api', metavar='API',
78 default=get_api(), help='use api API')
79 parser.add_option('-v', action='store_true', dest='verbose',
80 default=False, help='use verbose output')
81 parser.add_option('-d', action='store_true', dest='debug',
82 default=False, help='use debug output')
83 self.add_options(parser)
84 options, args = parser.parse_args(argv)
87 for opt in parser.option_list:
90 val = getattr(options, key)
91 setattr(self, key, val)
93 self.client = Pithos_Client(self.host, self.token, self.user, self.api, self.verbose,
99 def _build_args(self, attrs):
101 for a in [a for a in attrs if getattr(self, a)]:
102 args[a] = getattr(self, a)
105 def add_options(self, parser):
108 def execute(self, *args):
111 @cli_command('list', 'ls')
113 syntax = '[<container>[/<object>]]'
114 description = 'list containers or objects'
116 def add_options(self, parser):
117 parser.add_option('-l', action='store_true', dest='detail',
118 default=False, help='show detailed output')
119 parser.add_option('-n', action='store', type='int', dest='limit',
120 default=10000, help='show limited output')
121 parser.add_option('--marker', action='store', type='str',
122 dest='marker', default=None,
123 help='show output greater then marker')
124 parser.add_option('--prefix', action='store', type='str',
125 dest='prefix', default=None,
126 help='show output starting with prefix')
127 parser.add_option('--delimiter', action='store', type='str',
128 dest='delimiter', default=None,
129 help='show output up to the delimiter')
130 parser.add_option('--path', action='store', type='str',
131 dest='path', default=None,
132 help='show output starting with prefix up to /')
133 parser.add_option('--meta', action='store', type='str',
134 dest='meta', default=None,
135 help='show output having the specified meta keys')
136 parser.add_option('--if-modified-since', action='store', type='str',
137 dest='if_modified_since', default=None,
138 help='show output if modified since then')
139 parser.add_option('--if-unmodified-since', action='store', type='str',
140 dest='if_unmodified_since', default=None,
141 help='show output if not modified since then')
142 parser.add_option('--until', action='store', dest='until',
143 default=None, help='show metadata until that date')
144 parser.add_option('--format', action='store', dest='format',
145 default='%d/%m/%Y', help='format to parse until date')
147 def execute(self, container=None):
149 self.list_objects(container)
151 self.list_containers()
153 def list_containers(self):
154 attrs = ['limit', 'marker', 'if_modified_since',
155 'if_unmodified_since']
156 args = self._build_args(attrs)
157 args['format'] = 'json' if self.detail else 'text'
159 if getattr(self, 'until'):
160 t = _time.strptime(self.until, self.format)
161 args['until'] = int(_time.mktime(t))
163 l = self.client.list_containers(**args)
166 def list_objects(self, container):
169 attrs = ['limit', 'marker', 'prefix', 'delimiter', 'path',
170 'meta', 'if_modified_since', 'if_unmodified_since']
171 args = self._build_args(attrs)
172 args['format'] = 'json' if self.detail else 'text'
175 t = _time.strptime(self.until, self.format)
176 args['until'] = int(_time.mktime(t))
178 container, sep, object = container.partition('/')
183 #if request with meta quering disable trash filtering
184 show_trashed = True if self.meta else False
185 l = self.client.list_objects(container, **args)
186 print_list(l, detail=self.detail)
190 syntax = '[<container>[/<object>]]'
191 description = 'get account/container/object metadata'
193 def add_options(self, parser):
194 parser.add_option('-r', action='store_true', dest='restricted',
195 default=False, help='show only user defined metadata')
196 parser.add_option('--until', action='store', dest='until',
197 default=None, help='show metadata until that date')
198 parser.add_option('--format', action='store', dest='format',
199 default='%d/%m/%Y', help='format to parse until date')
200 parser.add_option('--version', action='store', dest='version',
201 default=None, help='show specific version \
202 (applies only for objects)')
204 def execute(self, path=''):
205 container, sep, object = path.partition('/')
206 args = {'restricted':self.restricted}
207 if getattr(self, 'until'):
208 t = _time.strptime(self.until, self.format)
209 args['until'] = int(_time.mktime(t))
212 meta = self.client.retrieve_object_metadata(container, object,
216 meta = self.client.retrieve_container_metadata(container, **args)
218 meta = self.client.retrieve_account_metadata(**args)
220 print 'Entity does not exist'
222 print_dict(meta, header=None)
224 @cli_command('create')
225 class CreateContainer(Command):
226 syntax = '<container> [key=val] [...]'
227 description = 'create a container'
229 def execute(self, container, *args):
232 key, sep, val = arg.partition('=')
234 ret = self.client.create_container(container, **meta)
236 print 'Container already exists'
238 @cli_command('delete', 'rm')
239 class Delete(Command):
240 syntax = '<container>[/<object>]'
241 description = 'delete a container or an object'
243 def add_options(self, parser):
244 parser.add_option('--until', action='store', dest='until',
245 default=None, help='remove history until that date')
246 parser.add_option('--format', action='store', dest='format',
247 default='%d/%m/%Y', help='format to parse until date')
249 def execute(self, path):
250 container, sep, object = path.partition('/')
252 if getattr(self, 'until'):
253 t = _time.strptime(self.until, self.format)
254 until = int(_time.mktime(t))
257 self.client.delete_object(container, object, until)
259 self.client.delete_container(container, until)
262 class GetObject(Command):
263 syntax = '<container>/<object>'
264 description = 'get the data of an object'
266 def add_options(self, parser):
267 parser.add_option('-l', action='store_true', dest='detail',
268 default=False, help='show detailed output')
269 parser.add_option('--range', action='store', dest='range',
270 default=None, help='show range of data')
271 parser.add_option('--if-range', action='store', dest='if_range',
272 default=None, help='show range of data')
273 parser.add_option('--if-match', action='store', dest='if_match',
274 default=None, help='show output if ETags match')
275 parser.add_option('--if-none-match', action='store',
276 dest='if_none_match', default=None,
277 help='show output if ETags don\'t match')
278 parser.add_option('--if-modified-since', action='store', type='str',
279 dest='if_modified_since', default=None,
280 help='show output if modified since then')
281 parser.add_option('--if-unmodified-since', action='store', type='str',
282 dest='if_unmodified_since', default=None,
283 help='show output if not modified since then')
284 parser.add_option('-o', action='store', type='str',
285 dest='file', default=None,
286 help='save output in file')
287 parser.add_option('--version', action='store', type='str',
288 dest='version', default=None,
289 help='get the specific \
291 parser.add_option('--versionlist', action='store_true',
292 dest='versionlist', default=False,
293 help='get the full object version list')
295 def execute(self, path):
296 attrs = ['if_match', 'if_none_match', 'if_modified_since',
297 'if_unmodified_since']
298 args = self._build_args(attrs)
299 args['format'] = 'json' if self.detail else 'text'
301 args['range'] = 'bytes=%s' %self.range
302 if getattr(self, 'if_range'):
303 args['if-range'] = 'If-Range:%s' % getattr(self, 'if_range')
305 container, sep, object = path.partition('/')
308 if 'detail' in args.keys():
312 data = self.client.retrieve_object_versionlist(container, object, **args)
314 data = self.client.retrieve_object_version(container, object,
315 self.version, **args)
317 data = self.client.retrieve_object(container, object, **args)
319 f = self.file and open(self.file, 'w') or stdout
322 print_versions(data, f=f)
324 print_dict(data, f=f)
329 @cli_command('mkdir')
330 class PutMarker(Command):
331 syntax = '<container>/<directory marker>'
332 description = 'create a directory marker'
334 def execute(self, path):
335 container, sep, object = path.partition('/')
336 self.client.create_directory_marker(container, object)
339 class PutObject(Command):
340 syntax = '<container>/<object> [key=val] [...]'
341 description = 'create/override object'
343 def add_options(self, parser):
344 parser.add_option('--use_hashes', action='store_true', dest='use_hashes',
345 default=False, help='provide hashmap instead of data')
346 parser.add_option('--chunked', action='store_true', dest='chunked',
347 default=False, help='set chunked transfer mode')
348 parser.add_option('--etag', action='store', dest='etag',
349 default=None, help='check written data')
350 parser.add_option('--content-encoding', action='store',
351 dest='content_encoding', default=None,
352 help='provide the object MIME content type')
353 parser.add_option('--content-disposition', action='store', type='str',
354 dest='content_disposition', default=None,
355 help='provide the presentation style of the object')
356 #parser.add_option('-S', action='store',
357 # dest='segment_size', default=False,
358 # help='use for large file support')
359 parser.add_option('--manifest', action='store',
360 dest='x_object_manifest', default=None,
361 help='upload a manifestation file')
362 parser.add_option('--content-type', action='store',
363 dest='content_type', default=None,
364 help='create object with specific content type')
365 parser.add_option('--sharing', action='store',
366 dest='x_object_sharing', default=None,
367 help='define sharing object policy')
368 parser.add_option('-f', action='store',
369 dest='srcpath', default=None,
370 help='file descriptor to read from (pass - for standard input)')
371 parser.add_option('--public', action='store_true',
372 dest='x_object_public', default=False,
373 help='make object publicly accessible')
375 def execute(self, path, *args):
376 if path.find('=') != -1:
377 raise Fault('Missing path argument')
379 #prepare user defined meta
382 key, sep, val = arg.partition('=')
385 attrs = ['etag', 'content_encoding', 'content_disposition',
386 'content_type', 'x_object_sharing', 'x_object_public']
387 args = self._build_args(attrs)
389 container, sep, object = path.partition('/')
393 f = open(self.srcpath) if self.srcpath != '-' else stdin
395 if self.use_hashes and not f:
396 raise Fault('Illegal option combination')
399 self.client.create_object_using_chunks(container, object, f,
401 elif self.use_hashes:
402 format = 'json' if detail else 'text'
403 self.client.create_object_by_hashmap(container, object, f, format,
405 elif self.x_object_manifest:
406 self.client.create_manifestation(container, object, self.x_object_manifest)
408 self.client.create_zero_length_object(container, object, meta=meta, **args)
410 self.client.create_object(container, object, f, meta=meta, **args)
414 @cli_command('copy', 'cp')
415 class CopyObject(Command):
416 syntax = '<src container>/<src object> [<dst container>/]<dst object> [key=val] [...]'
417 description = 'copy an object to a different location'
419 def add_options(self, parser):
420 parser.add_option('--version', action='store',
421 dest='version', default=False,
422 help='copy specific version')
423 parser.add_option('--public', action='store_true',
424 dest='public', default=False,
425 help='make object publicly accessible')
427 def execute(self, src, dst, *args):
428 src_container, sep, src_object = src.partition('/')
429 dst_container, sep, dst_object = dst.partition('/')
431 #prepare user defined meta
434 key, sep, val = arg.partition('=')
438 dst_container = src_container
441 self.client.copy_object(src_container, src_object, dst_container,
442 dst_object, meta, self.public, self.version, **meta)
445 class SetMeta(Command):
446 syntax = '[<container>[/<object>]] key=val [key=val] [...]'
447 description = 'set account/container/object metadata'
449 def execute(self, path, *args):
450 #in case of account fix the args
451 if path.find('=') != -1:
458 key, sep, val = arg.partition('=')
459 meta[key.strip()] = val.strip()
460 container, sep, object = path.partition('/')
462 self.client.update_object_metadata(container, object, **meta)
464 self.client.update_container_metadata(container, **meta)
466 self.client.update_account_metadata(**meta)
468 @cli_command('update')
469 class UpdateObject(Command):
470 syntax = '<container>/<object> path [key=val] [...]'
471 description = 'update object metadata/data (default mode: append)'
473 def add_options(self, parser):
474 parser.add_option('-a', action='store_true', dest='append',
475 default=True, help='append data')
476 parser.add_option('--offset', action='store',
478 default=None, help='starting offest to be updated')
479 parser.add_option('--range', action='store', dest='content-range',
480 default=None, help='range of data to be updated')
481 parser.add_option('--chunked', action='store_true', dest='chunked',
482 default=False, help='set chunked transfer mode')
483 parser.add_option('--content-encoding', action='store',
484 dest='content_encoding', default=None,
485 help='provide the object MIME content type')
486 parser.add_option('--content-disposition', action='store', type='str',
487 dest='content_disposition', default=None,
488 help='provide the presentation style of the object')
489 parser.add_option('--manifest', action='store', type='str',
490 dest='x_object_manifest', default=None,
491 help='use for large file support')
492 parser.add_option('--sharing', action='store',
493 dest='x_object_sharing', default=None,
494 help='define sharing object policy')
495 parser.add_option('--nosharing', action='store_true',
496 dest='no_sharing', default=None,
497 help='clear object sharing policy')
498 parser.add_option('-f', action='store',
499 dest='srcpath', default=None,
500 help='file descriptor to read from: pass - for standard input')
501 parser.add_option('--public', action='store_true',
502 dest='x_object_public', default=False,
503 help='make object publicly accessible')
505 def execute(self, path, *args):
506 if path.find('=') != -1:
507 raise Fault('Missing path argument')
509 #prepare user defined meta
512 key, sep, val = arg.partition('=')
516 self.x_object_sharing = ''
518 attrs = ['content_encoding', 'content_disposition', 'x_object_sharing',
520 args = self._build_args(attrs)
522 container, sep, object = path.partition('/')
526 f = open(self.srcpath) if self.srcpath != '-' else stdin
529 self.client.update_object_using_chunks(container, object, f,
532 self.client.update_object(container, object, f, meta=meta, **args)
536 @cli_command('move', 'mv')
537 class MoveObject(Command):
538 syntax = '<src container>/<src object> [<dst container>/]<dst object>'
539 description = 'move an object to a different location'
541 def add_options(self, parser):
542 parser.add_option('--version', action='store',
543 dest='version', default=None,
544 help='move a specific object version')
545 parser.add_option('--public', action='store_true',
546 dest='public', default=False,
547 help='make object publicly accessible')
549 def execute(self, src, dst, *args):
550 src_container, sep, src_object = src.partition('/')
551 dst_container, sep, dst_object = dst.partition('/')
553 dst_container = src_container
556 #prepare user defined meta
559 key, sep, val = arg.partition('=')
562 self.client.move_object(src_container, src_object, dst_container,
563 dst_object, meta, self.public, self.version)
565 @cli_command('remove')
566 class TrashObject(Command):
567 syntax = '<container>/<object>'
568 description = 'trash an object'
570 def execute(self, src):
571 src_container, sep, src_object = src.partition('/')
573 self.client.trash_object(src_container, src_object)
575 @cli_command('restore')
576 class RestoreObject(Command):
577 syntax = '<container>/<object>'
578 description = 'restore a trashed object'
580 def execute(self, src):
581 src_container, sep, src_object = src.partition('/')
583 self.client.restore_object(src_container, src_object)
585 @cli_command('unset')
586 class UnsetObject(Command):
587 syntax = '<container>/[<object>] key [key] [...]'
588 description = 'delete metadata info'
590 def execute(self, path, *args):
591 #in case of account fix the args
600 container, sep, object = path.partition('/')
602 self.client.delete_object_metadata(container, object, meta)
604 self.client.delete_container_metadata(container, meta)
606 self.client.delete_account_metadata(meta)
608 @cli_command('group')
609 class CreateGroup(Command):
610 syntax = 'key=val [key=val] [...]'
611 description = 'create account groups'
613 def execute(self, *args):
616 key, sep, val = arg.partition('=')
618 self.client.set_account_groups(**groups)
620 @cli_command('ungroup')
621 class DeleteGroup(Command):
622 syntax = 'key [key] [...]'
623 description = 'delete account groups'
625 def execute(self, *args):
629 self.client.unset_account_groups(groups)
631 @cli_command('policy')
632 class SetPolicy(Command):
633 syntax = 'container key=val [key=val] [...]'
634 description = 'set container policies'
636 def execute(self, path, *args):
637 if path.find('=') != -1:
638 raise Fault('Missing container argument')
640 container, sep, object = path.partition('/')
643 raise Fault('Only containers have policies')
647 key, sep, val = arg.partition('=')
650 self.client.set_container_policies(container, **policies)
652 @cli_command('publish')
653 class PublishObject(Command):
654 syntax = '<container>/<object>'
655 description = 'publish an object'
657 def execute(self, src):
658 src_container, sep, src_object = src.partition('/')
660 self.client.publish_object(src_container, src_object)
662 @cli_command('unpublish')
663 class UnpublishObject(Command):
664 syntax = '<container>/<object>'
665 description = 'unpublish an object'
667 def execute(self, src):
668 src_container, sep, src_object = src.partition('/')
670 self.client.unpublish_object(src_container, src_object)
673 cmd = Command('', [])
675 parser.usage = '%prog <command> [options]'
679 for cls in set(_cli_commands.values()):
680 name = ', '.join(cls.commands)
681 description = getattr(cls, 'description', '')
682 commands.append(' %s %s' % (name.ljust(12), description))
683 print '\nCommands:\n' + '\n'.join(sorted(commands))
685 def print_dict(d, header='name', f=stdout, detail=True):
686 header = header if header in d else 'subdir'
687 if header and header in d:
688 f.write('%s\n' %d.pop(header).encode('utf8'))
690 patterns = ['^x_(account|container|object)_meta_(\w+)$']
691 patterns.append(patterns[0].replace('_', '-'))
692 for key, val in sorted(d.items()):
693 f.write('%s: %s\n' % (key.rjust(30), val))
695 def print_list(l, verbose=False, f=stdout, detail=True):
697 #if it's empty string continue
700 if type(elem) == types.DictionaryType:
701 print_dict(elem, f=f, detail=detail)
702 elif type(elem) == types.StringType:
704 elem = elem.split('Traceback')[0]
705 f.write('%s\n' % elem)
707 f.write('%s\n' % elem)
709 def print_versions(data, f=stdout):
710 if 'versions' not in data:
711 f.write('%s\n' %data)
713 f.write('versions:\n')
714 for id, t in data['versions']:
715 f.write('%s @ %s\n' % (str(id).rjust(30), datetime.fromtimestamp(t)))
720 cls = class_for_cli_command(name)
721 except (IndexError, KeyError):
725 cmd = cls(name, argv[2:])
728 cmd.execute(*cmd.args)
730 cmd.parser.print_help()
733 status = f.status and '%s ' % f.status or ''
734 print '%s%s' % (status, f.data)
736 if __name__ == '__main__':