3 # Copyright 2011 GRNET S.A. All rights reserved.
5 # Redistribution and use in source and binary forms, with or
6 # without modification, are permitted provided that the following
9 # 1. Redistributions of source code must retain the above
10 # copyright notice, this list of conditions and the following
13 # 2. Redistributions in binary form must reproduce the above
14 # copyright notice, this list of conditions and the following
15 # disclaimer in the documentation and/or other materials
16 # provided with the distribution.
18 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 # POSSIBILITY OF SUCH DAMAGE.
31 # The views and conclusions contained in the software and
32 # documentation are those of the authors and should not be
33 # interpreted as representing official policies, either expressed
34 # or implied, of GRNET S.A.
36 from getpass import getuser
37 from optparse import OptionParser
38 from os import environ
39 from sys import argv, exit, stdin, stdout
40 from pithos.lib.client import Pithos_Client, Fault
41 from datetime import datetime
50 DEFAULT_HOST = 'pithos.dev.grnet.gr'
51 #DEFAULT_HOST = '127.0.0.1:8000'
56 def cli_command(*args):
60 _cli_commands[name] = cls
64 def class_for_cli_command(name):
65 return _cli_commands[name]
67 class Command(object):
70 def __init__(self, name, argv):
71 parser = OptionParser('%%prog %s [options] %s' % (name, self.syntax))
72 parser.add_option('--host', dest='host', metavar='HOST',
73 default=_get_server(), help='use server HOST')
74 parser.add_option('--user', dest='user', metavar='USERNAME',
76 help='use account USERNAME')
77 parser.add_option('--token', dest='token', metavar='AUTH',
79 help='use account AUTH')
80 parser.add_option('--api', dest='api', metavar='API',
81 default=DEFAULT_API, help='use api API')
82 parser.add_option('-v', action='store_true', dest='verbose',
83 default=False, help='use verbose output')
84 parser.add_option('-d', action='store_true', dest='debug',
85 default=False, help='use debug output')
86 self.add_options(parser)
87 options, args = parser.parse_args(argv)
90 for opt in parser.option_list:
93 val = getattr(options, key)
94 setattr(self, key, val)
96 self.client = Pithos_Client(self.host, self.token, self.user, self.api, self.verbose,
102 def _build_args(self, attrs):
104 for a in [a for a in attrs if getattr(self, a)]:
105 args[a] = getattr(self, a)
108 def add_options(self, parser):
111 def execute(self, *args):
114 @cli_command('list', 'ls')
116 syntax = '[<container>[/<object>]]'
117 description = 'list containers or objects'
119 def add_options(self, parser):
120 parser.add_option('-l', action='store_true', dest='detail',
121 default=False, help='show detailed output')
122 parser.add_option('-n', action='store', type='int', dest='limit',
123 default=10000, help='show limited output')
124 parser.add_option('--marker', action='store', type='str',
125 dest='marker', default=None,
126 help='show output greater then marker')
127 parser.add_option('--prefix', action='store', type='str',
128 dest='prefix', default=None,
129 help='show output starting with prefix')
130 parser.add_option('--delimiter', action='store', type='str',
131 dest='delimiter', default=None,
132 help='show output up to the delimiter')
133 parser.add_option('--path', action='store', type='str',
134 dest='path', default=None,
135 help='show output starting with prefix up to /')
136 parser.add_option('--meta', action='store', type='str',
137 dest='meta', default=None,
138 help='show output having the specified meta keys')
139 parser.add_option('--if-modified-since', action='store', type='str',
140 dest='if_modified_since', default=None,
141 help='show output if modified since then')
142 parser.add_option('--if-unmodified-since', action='store', type='str',
143 dest='if_unmodified_since', default=None,
144 help='show output if not modified since then')
145 parser.add_option('--until', action='store', dest='until',
146 default=None, help='show metadata until that date')
147 parser.add_option('--format', action='store', dest='format',
148 default='%d/%m/%Y', help='format to parse until date')
150 def execute(self, container=None):
152 self.list_objects(container)
154 self.list_containers()
156 def list_containers(self):
157 attrs = ['limit', 'marker', 'if_modified_since',
158 'if_unmodified_since']
159 args = self._build_args(attrs)
160 args['format'] = 'json' if self.detail else 'text'
162 if getattr(self, 'until'):
163 t = _time.strptime(self.until, self.format)
164 args['until'] = int(_time.mktime(t))
166 l = self.client.list_containers(**args)
169 def list_objects(self, container):
172 attrs = ['limit', 'marker', 'prefix', 'delimiter', 'path',
173 'meta', 'if_modified_since', 'if_unmodified_since']
174 args = self._build_args(attrs)
175 args['format'] = 'json' if self.detail else 'text'
178 t = _time.strptime(self.until, self.format)
179 args['until'] = int(_time.mktime(t))
181 container, sep, object = container.partition('/')
186 #if request with meta quering disable trash filtering
187 show_trashed = True if self.meta else False
188 l = self.client.list_objects(container, **args)
189 print_list(l, detail=self.detail)
193 syntax = '[<container>[/<object>]]'
194 description = 'get account/container/object metadata'
196 def add_options(self, parser):
197 parser.add_option('-r', action='store_true', dest='restricted',
198 default=False, help='show only user defined metadata')
199 parser.add_option('--until', action='store', dest='until',
200 default=None, help='show metadata until that date')
201 parser.add_option('--format', action='store', dest='format',
202 default='%d/%m/%Y', help='format to parse until date')
203 parser.add_option('--version', action='store', dest='version',
204 default=None, help='show specific version \
205 (applies only for objects)')
207 def execute(self, path=''):
208 container, sep, object = path.partition('/')
209 args = {'restricted':self.restricted}
210 if getattr(self, 'until'):
211 t = _time.strptime(self.until, self.format)
212 args['until'] = int(_time.mktime(t))
215 meta = self.client.retrieve_object_metadata(container, object,
219 meta = self.client.retrieve_container_metadata(container, **args)
221 meta = self.client.retrieve_account_metadata(**args)
223 print 'Entity does not exist'
225 print_dict(meta, header=None)
227 @cli_command('create')
228 class CreateContainer(Command):
229 syntax = '<container> [key=val] [...]'
230 description = 'create a container'
232 def execute(self, container, *args):
235 key, sep, val = arg.partition('=')
237 ret = self.client.create_container(container, **meta)
239 print 'Container already exists'
241 @cli_command('delete', 'rm')
242 class Delete(Command):
243 syntax = '<container>[/<object>]'
244 description = 'delete a container or an object'
246 def add_options(self, parser):
247 parser.add_option('--until', action='store', dest='until',
248 default=None, help='remove history until that date')
249 parser.add_option('--format', action='store', dest='format',
250 default='%d/%m/%Y', help='format to parse until date')
252 def execute(self, path):
253 container, sep, object = path.partition('/')
255 if getattr(self, 'until'):
256 t = _time.strptime(self.until, self.format)
257 until = int(_time.mktime(t))
260 self.client.delete_object(container, object, until)
262 self.client.delete_container(container, until)
265 class GetObject(Command):
266 syntax = '<container>/<object>'
267 description = 'get the data of an object'
269 def add_options(self, parser):
270 parser.add_option('-l', action='store_true', dest='detail',
271 default=False, help='show detailed output')
272 parser.add_option('--range', action='store', dest='range',
273 default=None, help='show range of data')
274 parser.add_option('--if-range', action='store', dest='if_range',
275 default=None, help='show range of data')
276 parser.add_option('--if-match', action='store', dest='if_match',
277 default=None, help='show output if ETags match')
278 parser.add_option('--if-none-match', action='store',
279 dest='if_none_match', default=None,
280 help='show output if ETags don\'t match')
281 parser.add_option('--if-modified-since', action='store', type='str',
282 dest='if_modified_since', default=None,
283 help='show output if modified since then')
284 parser.add_option('--if-unmodified-since', action='store', type='str',
285 dest='if_unmodified_since', default=None,
286 help='show output if not modified since then')
287 parser.add_option('-o', action='store', type='str',
288 dest='file', default=None,
289 help='save output in file')
290 parser.add_option('--version', action='store', type='str',
291 dest='version', default=None,
292 help='get the specific \
294 parser.add_option('--versionlist', action='store_true',
295 dest='versionlist', default=False,
296 help='get the full object version list')
298 def execute(self, path):
299 attrs = ['if_match', 'if_none_match', 'if_modified_since',
300 'if_unmodified_since']
301 args = self._build_args(attrs)
302 args['format'] = 'json' if self.detail else 'text'
304 args['range'] = 'bytes=%s' %self.range
305 if getattr(self, 'if_range'):
306 args['if-range'] = 'If-Range:%s' % getattr(self, 'if_range')
308 container, sep, object = path.partition('/')
311 if 'detail' in args.keys():
313 data = self.client.retrieve_object_versionlist(container, object, **args)
315 data = self.client.retrieve_object_version(container, object,
316 self.version, **args)
318 data = self.client.retrieve_object(container, object, **args)
320 f = self.file and open(self.file, 'w') or stdout
322 data = json.loads(data)
324 print_versions(data, f=f)
326 print_dict(data, f=f)
331 @cli_command('mkdir')
332 class PutMarker(Command):
333 syntax = '<container>/<directory marker>'
334 description = 'create a directory marker'
336 def execute(self, path):
337 container, sep, object = path.partition('/')
338 self.client.create_directory_marker(container, object)
341 class PutObject(Command):
342 syntax = '<container>/<object> [key=val] [...]'
343 description = 'create/override object'
345 def add_options(self, parser):
346 parser.add_option('--use_hashes', action='store_true', dest='use_hashes',
347 default=False, help='provide hashmap instead of data')
348 parser.add_option('--chunked', action='store_true', dest='chunked',
349 default=False, help='set chunked transfer mode')
350 parser.add_option('--etag', action='store', dest='etag',
351 default=None, help='check written data')
352 parser.add_option('--content-encoding', action='store',
353 dest='content_encoding', default=None,
354 help='provide the object MIME content type')
355 parser.add_option('--content-disposition', action='store', type='str',
356 dest='content_disposition', default=None,
357 help='provide the presentation style of the object')
358 #parser.add_option('-S', action='store',
359 # dest='segment_size', default=False,
360 # help='use for large file support')
361 parser.add_option('--manifest', action='store',
362 dest='x_object_manifest', default=None,
363 help='upload a manifestation file')
364 parser.add_option('--content-type', action='store',
365 dest='content_type', default=None,
366 help='create object with specific content type')
367 parser.add_option('--sharing', action='store',
368 dest='x_object_sharing', default=None,
369 help='define sharing object policy')
370 parser.add_option('-f', action='store',
371 dest='srcpath', default=None,
372 help='file descriptor to read from (pass - for standard input)')
373 parser.add_option('--public', action='store_true',
374 dest='x_object_public', default=False,
375 help='make object publicly accessible')
377 def execute(self, path, *args):
378 if path.find('=') != -1:
379 raise Fault('Missing path argument')
381 #prepare user defined meta
384 key, sep, val = arg.partition('=')
387 attrs = ['etag', 'content_encoding', 'content_disposition',
388 'content_type', 'x_object_sharing', 'x_object_public']
389 args = self._build_args(attrs)
391 container, sep, object = path.partition('/')
395 f = open(self.srcpath) if self.srcpath != '-' else stdin
397 if self.use_hashes and not f:
398 raise Fault('Illegal option combination')
401 self.client.create_object_using_chunks(container, object, f,
403 elif self.use_hashes:
404 format = 'json' if detail else 'text'
405 self.client.create_object_by_hashmap(container, object, f, format,
407 elif self.x_object_manifest:
408 self.client.create_manifestation(container, object, self.x_object_manifest)
410 self.client.create_zero_length_object(container, object, meta=meta, **args)
412 self.client.create_object(container, object, f, meta=meta, **args)
416 @cli_command('copy', 'cp')
417 class CopyObject(Command):
418 syntax = '<src container>/<src object> [<dst container>/]<dst object> [key=val] [...]'
419 description = 'copy an object to a different location'
421 def add_options(self, parser):
422 parser.add_option('--version', action='store',
423 dest='version', default=False,
424 help='copy specific version')
425 parser.add_option('--public', action='store_true',
426 dest='public', default=False,
427 help='make object publicly accessible')
429 def execute(self, src, dst, *args):
430 src_container, sep, src_object = src.partition('/')
431 dst_container, sep, dst_object = dst.partition('/')
433 #prepare user defined meta
436 key, sep, val = arg.partition('=')
440 dst_container = src_container
443 self.client.copy_object(src_container, src_object, dst_container,
444 dst_object, meta, self.public, self.version, **meta)
447 class SetMeta(Command):
448 syntax = '[<container>[/<object>]] key=val [key=val] [...]'
449 description = 'set account/container/object metadata'
451 def execute(self, path, *args):
452 #in case of account fix the args
453 if path.find('=') != -1:
460 key, sep, val = arg.partition('=')
461 meta[key.strip()] = val.strip()
462 container, sep, object = path.partition('/')
464 self.client.update_object_metadata(container, object, **meta)
466 self.client.update_container_metadata(container, **meta)
468 self.client.update_account_metadata(**meta)
470 @cli_command('update')
471 class UpdateObject(Command):
472 syntax = '<container>/<object> path [key=val] [...]'
473 description = 'update object metadata/data (default mode: append)'
475 def add_options(self, parser):
476 parser.add_option('-a', action='store_true', dest='append',
477 default=True, help='append data')
478 parser.add_option('--offset', action='store',
480 default=None, help='starting offest to be updated')
481 parser.add_option('--range', action='store', dest='content-range',
482 default=None, help='range of data to be updated')
483 parser.add_option('--chunked', action='store_true', dest='chunked',
484 default=False, help='set chunked transfer mode')
485 parser.add_option('--content-encoding', action='store',
486 dest='content_encoding', default=None,
487 help='provide the object MIME content type')
488 parser.add_option('--content-disposition', action='store', type='str',
489 dest='content_disposition', default=None,
490 help='provide the presentation style of the object')
491 parser.add_option('--manifest', action='store', type='str',
492 dest='x_object_manifest', default=None,
493 help='use for large file support')
494 parser.add_option('--sharing', action='store',
495 dest='x_object_sharing', default=None,
496 help='define sharing object policy')
497 parser.add_option('--nosharing', action='store_true',
498 dest='no_sharing', default=None,
499 help='clear object sharing policy')
500 parser.add_option('-f', action='store',
501 dest='srcpath', default=None,
502 help='file descriptor to read from: pass - for standard input')
503 parser.add_option('--public', action='store_true',
504 dest='x_object_public', default=False,
505 help='make object publicly accessible')
507 def execute(self, path, *args):
508 if path.find('=') != -1:
509 raise Fault('Missing path argument')
511 #prepare user defined meta
514 key, sep, val = arg.partition('=')
518 self.x_object_sharing = ''
520 attrs = ['content_encoding', 'content_disposition', 'x_object_sharing',
522 args = self._build_args(attrs)
524 container, sep, object = path.partition('/')
528 f = open(self.srcpath) if self.srcpath != '-' else stdin
531 self.client.update_object_using_chunks(container, object, f,
534 self.client.update_object(container, object, f, meta=meta, **args)
538 @cli_command('move', 'mv')
539 class MoveObject(Command):
540 syntax = '<src container>/<src object> [<dst container>/]<dst object>'
541 description = 'move an object to a different location'
543 def add_options(self, parser):
544 parser.add_option('--version', action='store',
545 dest='version', default=None,
546 help='move a specific object version')
547 parser.add_option('--public', action='store_true',
548 dest='public', default=False,
549 help='make object publicly accessible')
551 def execute(self, src, dst, *args):
552 src_container, sep, src_object = src.partition('/')
553 dst_container, sep, dst_object = dst.partition('/')
555 dst_container = src_container
558 #prepare user defined meta
561 key, sep, val = arg.partition('=')
564 self.client.move_object(src_container, src_object, dst_container,
565 dst_object, meta, self.public, self.version)
567 @cli_command('remove')
568 class TrashObject(Command):
569 syntax = '<container>/<object>'
570 description = 'trash an object'
572 def execute(self, src):
573 src_container, sep, src_object = src.partition('/')
575 self.client.trash_object(src_container, src_object)
577 @cli_command('restore')
578 class RestoreObject(Command):
579 syntax = '<container>/<object>'
580 description = 'restore a trashed object'
582 def execute(self, src):
583 src_container, sep, src_object = src.partition('/')
585 self.client.restore_object(src_container, src_object)
587 @cli_command('unset')
588 class UnsetObject(Command):
589 syntax = '<container>/[<object>] key [key] [...]'
590 description = 'delete metadata info'
592 def execute(self, path, *args):
593 #in case of account fix the args
602 container, sep, object = path.partition('/')
604 self.client.delete_object_metadata(container, object, meta)
606 self.client.delete_container_metadata(container, meta)
608 self.client.delete_account_metadata(meta)
610 @cli_command('group')
611 class CreateGroup(Command):
612 syntax = 'key=val [key=val] [...]'
613 description = 'create account groups'
615 def execute(self, *args):
618 key, sep, val = arg.partition('=')
620 self.client.set_account_groups(**groups)
622 @cli_command('ungroup')
623 class DeleteGroup(Command):
624 syntax = 'key [key] [...]'
625 description = 'delete account groups'
627 def execute(self, *args):
631 self.client.unset_account_groups(groups)
633 @cli_command('policy')
634 class SetPolicy(Command):
635 syntax = 'container key=val [key=val] [...]'
636 description = 'set container policies'
638 def execute(self, path, *args):
639 if path.find('=') != -1:
640 raise Fault('Missing container argument')
642 container, sep, object = path.partition('/')
645 raise Fault('Only containers have policies')
649 key, sep, val = arg.partition('=')
652 self.client.set_container_policies(container, **policies)
654 @cli_command('publish')
655 class PublishObject(Command):
656 syntax = '<container>/<object>'
657 description = 'publish an object'
659 def execute(self, src):
660 src_container, sep, src_object = src.partition('/')
662 self.client.publish_object(src_container, src_object)
664 @cli_command('unpublish')
665 class UnpublishObject(Command):
666 syntax = '<container>/<object>'
667 description = 'unpublish an object'
669 def execute(self, src):
670 src_container, sep, src_object = src.partition('/')
672 self.client.unpublish_object(src_container, src_object)
675 cmd = Command('', [])
677 parser.usage = '%prog <command> [options]'
681 for cls in set(_cli_commands.values()):
682 name = ', '.join(cls.commands)
683 description = getattr(cls, 'description', '')
684 commands.append(' %s %s' % (name.ljust(12), description))
685 print '\nCommands:\n' + '\n'.join(sorted(commands))
687 def print_dict(d, header='name', f=stdout, detail=True):
688 header = header if header in d else 'subdir'
689 if header and header in d:
690 f.write('%s\n' %d.pop(header).encode('utf8'))
692 patterns = ['^x_(account|container|object)_meta_(\w+)$']
693 patterns.append(patterns[0].replace('_', '-'))
694 for key, val in sorted(d.items()):
695 f.write('%s: %s\n' % (key.rjust(30), val))
697 def print_list(l, verbose=False, f=stdout, detail=True):
699 #if it's empty string continue
702 if type(elem) == types.DictionaryType:
703 print_dict(elem, f=f, detail=detail)
704 elif type(elem) == types.StringType:
706 elem = elem.split('Traceback')[0]
707 f.write('%s\n' % elem)
709 f.write('%s\n' % elem)
711 def print_versions(data, f=stdout):
712 if 'versions' not in data:
713 f.write('%s\n' %data)
715 f.write('versions:\n')
716 for id, t in data['versions']:
717 f.write('%s @ %s\n' % (str(id).rjust(30), datetime.fromtimestamp(t)))
721 return os.environ['PITHOS_USER']
727 return os.environ['PITHOS_AUTH']
733 return os.environ['PITHOS_SERVER']
740 cls = class_for_cli_command(name)
741 except (IndexError, KeyError):
745 cmd = cls(name, argv[2:])
748 cmd.execute(*cmd.args)
750 cmd.parser.print_help()
753 status = f.status and '%s ' % f.status or ''
754 print '%s%s' % (status, f.data)
756 if __name__ == '__main__':