3 # Copyright 2011 GRNET S.A. All rights reserved.
5 # Redistribution and use in source and binary forms, with or
6 # without modification, are permitted provided that the following
9 # 1. Redistributions of source code must retain the above
10 # copyright notice, this list of conditions and the following
13 # 2. Redistributions in binary form must reproduce the above
14 # copyright notice, this list of conditions and the following
15 # disclaimer in the documentation and/or other materials
16 # provided with the distribution.
18 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 # POSSIBILITY OF SUCH DAMAGE.
31 # The views and conclusions contained in the software and
32 # documentation are those of the authors and should not be
33 # interpreted as representing official policies, either expressed
34 # or implied, of GRNET S.A.
36 from getpass import getuser
37 from optparse import OptionParser
38 from os import environ
39 from sys import argv, exit, stdin, stdout
40 from pithos.lib.client import Client, Fault
41 from datetime import datetime
50 #DEFAULT_HOST = 'pithos.dev.grnet.gr'
51 DEFAULT_HOST = '127.0.0.1:8000'
56 def cli_command(*args):
60 _cli_commands[name] = cls
64 def class_for_cli_command(name):
65 return _cli_commands[name]
67 class Command(object):
70 def __init__(self, name, argv):
71 parser = OptionParser('%%prog %s [options] %s' % (name, self.syntax))
72 parser.add_option('--host', dest='host', metavar='HOST',
73 default=DEFAULT_HOST, help='use server HOST')
74 parser.add_option('--user', dest='user', metavar='USERNAME',
76 help='use account USERNAME')
77 parser.add_option('--token', dest='token', metavar='AUTH',
79 help='use account AUTH')
80 parser.add_option('--api', dest='api', metavar='API',
81 default=DEFAULT_API, help='use api API')
82 parser.add_option('-v', action='store_true', dest='verbose',
83 default=False, help='use verbose output')
84 parser.add_option('-d', action='store_true', dest='debug',
85 default=False, help='use debug output')
86 self.add_options(parser)
87 options, args = parser.parse_args(argv)
90 for opt in parser.option_list:
93 val = getattr(options, key)
94 setattr(self, key, val)
96 self.client = Client(self.host, self.token, self.user, self.api, self.verbose,
102 def add_options(self, parser):
105 def execute(self, *args):
108 @cli_command('list', 'ls')
110 syntax = '[<container>[/<object>]]'
111 description = 'list containers or objects'
113 def add_options(self, parser):
114 parser.add_option('-l', action='store_true', dest='detail',
115 default=False, help='show detailed output')
116 parser.add_option('-n', action='store', type='int', dest='limit',
117 default=1000, help='show limited output')
118 parser.add_option('--marker', action='store', type='str',
119 dest='marker', default=None,
120 help='show output greater then marker')
121 parser.add_option('--prefix', action='store', type='str',
122 dest='prefix', default=None,
123 help='show output starting with prefix')
124 parser.add_option('--delimiter', action='store', type='str',
125 dest='delimiter', default=None,
126 help='show output up to the delimiter')
127 parser.add_option('--path', action='store', type='str',
128 dest='path', default=None,
129 help='show output starting with prefix up to /')
130 parser.add_option('--meta', action='store', type='str',
131 dest='meta', default=None,
132 help='show output having the specified meta keys')
133 parser.add_option('--if-modified-since', action='store', type='str',
134 dest='if_modified_since', default=None,
135 help='show output if modified since then')
136 parser.add_option('--if-unmodified-since', action='store', type='str',
137 dest='if_unmodified_since', default=None,
138 help='show output if not modified since then')
139 parser.add_option('--until', action='store', dest='until',
140 default=False, help='show metadata until that date')
141 parser.add_option('--format', action='store', dest='format',
142 default='%d/%m/%Y', help='format to parse until date')
144 def execute(self, container=None):
146 self.list_objects(container)
148 self.list_containers()
150 def list_containers(self):
151 params = {'limit':self.limit, 'marker':self.marker}
152 headers = {'IF_MODIFIED_SINCE':self.if_modified_since,
153 'IF_UNMODIFIED_SINCE':self.if_unmodified_since}
156 t = _time.strptime(self.until, self.format)
157 params['until'] = int(_time.mktime(t))
159 l = self.client.list_containers(self.detail, params, headers)
162 def list_objects(self, container):
165 attrs = ['limit', 'marker', 'prefix', 'delimiter', 'path', 'meta']
166 for a in [a for a in attrs if getattr(self, a)]:
167 params[a] = getattr(self, a)
170 t = _time.strptime(self.until, self.format)
171 params['until'] = int(_time.mktime(t))
173 headers = {'IF_MODIFIED_SINCE':self.if_modified_since,
174 'IF_UNMODIFIED_SINCE':self.if_unmodified_since}
175 container, sep, object = container.partition('/')
180 #if request with meta quering disable trash filtering
181 show_trashed = True if self.meta else False
182 l = self.client.list_objects(container, detail, headers,
183 include_trashed = show_trashed, **params)
184 print_list(l, detail=self.detail)
188 syntax = '[<container>[/<object>]]'
189 description = 'get the metadata of an account, a container or an object'
191 def add_options(self, parser):
192 parser.add_option('-r', action='store_true', dest='restricted',
193 default=False, help='show only user defined metadata')
194 parser.add_option('--until', action='store', dest='until',
195 default=False, help='show metadata until that date')
196 parser.add_option('--format', action='store', dest='format',
197 default='%d/%m/%Y', help='format to parse until date')
198 parser.add_option('--version', action='store', dest='version',
199 default=None, help='show specific version \
200 (applies only for objects)')
202 def execute(self, path=''):
203 container, sep, object = path.partition('/')
205 t = _time.strptime(self.until, self.format)
206 self.until = int(_time.mktime(t))
208 meta = self.client.retrieve_object_metadata(container, object,
212 meta = self.client.retrieve_container_metadata(container,
216 meta = self.client.account_metadata(self.restricted, self.until)
218 print 'Entity does not exist'
220 print_dict(meta, header=None)
222 @cli_command('create')
223 class CreateContainer(Command):
224 syntax = '<container> [key=val] [...]'
225 description = 'create a container'
227 def execute(self, container, *args):
231 key, sep, val = arg.partition('=')
233 ret = self.client.create_container(container, headers, **meta)
235 print 'Container already exists'
237 @cli_command('delete', 'rm')
238 class Delete(Command):
239 syntax = '<container>[/<object>]'
240 description = 'delete a container or an object'
242 def execute(self, path):
243 container, sep, object = path.partition('/')
245 self.client.delete_object(container, object)
247 self.client.delete_container(container)
250 class GetObject(Command):
251 syntax = '<container>/<object>'
252 description = 'get the data of an object'
254 def add_options(self, parser):
255 parser.add_option('-l', action='store_true', dest='detail',
256 default=False, help='show detailed output')
257 parser.add_option('--range', action='store', dest='range',
258 default=None, help='show range of data')
259 parser.add_option('--if-range', action='store', dest='if-range',
260 default=None, help='show range of data')
261 parser.add_option('--if-match', action='store', dest='if-match',
262 default=None, help='show output if ETags match')
263 parser.add_option('--if-none-match', action='store',
264 dest='if-none-match', default=None,
265 help='show output if ETags don\'t match')
266 parser.add_option('--if-modified-since', action='store', type='str',
267 dest='if-modified-since', default=None,
268 help='show output if modified since then')
269 parser.add_option('--if-unmodified-since', action='store', type='str',
270 dest='if-unmodified-since', default=None,
271 help='show output if not modified since then')
272 parser.add_option('-o', action='store', type='str',
273 dest='file', default=None,
274 help='save output in file')
275 parser.add_option('--version', action='store', type='str',
276 dest='version', default=None,
277 help='get the specific \
279 parser.add_option('--versionlist', action='store_true',
280 dest='versionlist', default=False,
281 help='get the full object version list')
283 def execute(self, path):
286 headers['RANGE'] = 'bytes=%s' %self.range
287 if getattr(self, 'if-range'):
288 headers['IF_RANGE'] = 'If-Range:%s' % getattr(self, 'if-range')
289 attrs = ['if-match', 'if-none-match', 'if-modified-since',
290 'if-unmodified-since']
291 attrs = [a for a in attrs if getattr(self, a)]
293 headers[a.replace('-', '_').upper()] = getattr(self, a)
294 container, sep, object = path.partition('/')
296 self.version = 'list'
298 data = self.client.retrieve_object(container, object, self.detail,
299 headers, self.version)
300 f = self.file and open(self.file, 'w') or stdout
302 data = json.loads(data)
304 print_versions(data, f=f)
306 print_dict(data, f=f)
311 @cli_command('mkdir')
312 class PutMarker(Command):
313 syntax = '<container>/<directory marker>'
314 description = 'create a directory marker'
316 def execute(self, path):
317 container, sep, object = path.partition('/')
318 self.client.create_directory_marker(container, object)
321 class PutObject(Command):
322 syntax = '<container>/<object> <path> [key=val] [...]'
323 description = 'create/override object'
325 def add_options(self, parser):
326 parser.add_option('--use_hashes', action='store_true', dest='use_hashes',
327 default=False, help='provide hashmap instead of data')
328 parser.add_option('--chunked', action='store_true', dest='chunked',
329 default=False, help='set chunked transfer mode')
330 parser.add_option('--etag', action='store', dest='etag',
331 default=None, help='check written data')
332 parser.add_option('--content-encoding', action='store',
333 dest='content-encoding', default=None,
334 help='provide the object MIME content type')
335 parser.add_option('--content-disposition', action='store', type='str',
336 dest='content-disposition', default=None,
337 help='provide the presentation style of the object')
338 parser.add_option('-S', action='store',
339 dest='segment-size', default=False,
340 help='use for large file support')
341 parser.add_option('--manifest', action='store_true',
342 dest='manifest', default=None,
343 help='upload a manifestation file')
344 parser.add_option('--type', action='store',
345 dest='content-type', default=False,
346 help='create object with specific content type')
347 parser.add_option('--sharing', action='store',
348 dest='sharing', default=None,
349 help='define sharing object policy')
350 parser.add_option('-f', action='store',
351 dest='srcpath', default=None,
352 help='file descriptor to read from (pass - for standard input)')
353 parser.add_option('--public', action='store',
354 dest='public', default=None,
355 help='make object publicly accessible (\'True\'/\'False\')')
357 def execute(self, path, *args):
358 if path.find('=') != -1:
359 raise Fault('Missing path argument')
361 #prepare user defined meta
364 key, sep, val = arg.partition('=')
368 manifest = getattr(self, 'manifest')
370 # if it's manifestation file
371 # send zero-byte data with X-Object-Manifest header
373 headers['X_OBJECT_MANIFEST'] = manifest
375 headers['X_OBJECT_SHARING'] = self.sharing
377 attrs = ['etag', 'content-encoding', 'content-disposition',
379 attrs = [a for a in attrs if getattr(self, a)]
381 headers[a.replace('-', '_').upper()] = getattr(self, a)
383 container, sep, object = path.partition('/')
387 f = open(self.srcpath) if self.srcpath != '-' else stdin
389 if self.use_hashes and not f:
390 raise Fault('Illegal option combination')
391 if self.public not in ['True', 'False', None]:
392 raise Fault('Not acceptable value for public')
393 public = eval(self.public) if self.public else None
394 self.client.create_object(container, object, f, chunked=self.chunked,
395 headers=headers, use_hashes=self.use_hashes,
396 public=public, **meta)
400 @cli_command('copy', 'cp')
401 class CopyObject(Command):
402 syntax = '<src container>/<src object> [<dst container>/]<dst object>'
403 description = 'copy an object to a different location'
405 def add_options(self, parser):
406 parser.add_option('--version', action='store',
407 dest='version', default=False,
408 help='copy specific version')
409 parser.add_option('--public', action='store',
410 dest='public', default=None,
411 help='publish/unpublish object (\'True\'/\'False\')')
413 def execute(self, src, dst):
414 src_container, sep, src_object = src.partition('/')
415 dst_container, sep, dst_object = dst.partition('/')
417 dst_container = src_container
419 version = getattr(self, 'version')
422 headers['X_SOURCE_VERSION'] = version
423 if self.public and self.nopublic:
424 raise Fault('Conflicting options')
425 if self.public not in ['True', 'False', None]:
426 raise Fault('Not acceptable value for public')
427 public = eval(self.public) if self.public else None
428 self.client.copy_object(src_container, src_object, dst_container,
429 dst_object, public, headers)
432 class SetMeta(Command):
433 syntax = '[<container>[/<object>]] key=val [key=val] [...]'
434 description = 'set metadata'
436 def execute(self, path, *args):
437 #in case of account fix the args
438 if path.find('=') != -1:
445 key, sep, val = arg.partition('=')
446 meta[key.strip()] = val.strip()
447 container, sep, object = path.partition('/')
449 self.client.update_object_metadata(container, object, **meta)
451 self.client.update_container_metadata(container, **meta)
453 self.client.update_account_metadata(**meta)
455 @cli_command('update')
456 class UpdateObject(Command):
457 syntax = '<container>/<object> path [key=val] [...]'
458 description = 'update object metadata/data (default mode: append)'
460 def add_options(self, parser):
461 parser.add_option('-a', action='store_true', dest='append',
462 default=True, help='append data')
463 parser.add_option('--offset', action='store',
465 default=None, help='starting offest to be updated')
466 parser.add_option('--range', action='store', dest='content-range',
467 default=None, help='range of data to be updated')
468 parser.add_option('--chunked', action='store_true', dest='chunked',
469 default=False, help='set chunked transfer mode')
470 parser.add_option('--content-encoding', action='store',
471 dest='content-encoding', default=None,
472 help='provide the object MIME content type')
473 parser.add_option('--content-disposition', action='store', type='str',
474 dest='content-disposition', default=None,
475 help='provide the presentation style of the object')
476 parser.add_option('--manifest', action='store', type='str',
477 dest='manifest', default=None,
478 help='use for large file support')
479 parser.add_option('--sharing', action='store',
480 dest='sharing', default=None,
481 help='define sharing object policy')
482 parser.add_option('--nosharing', action='store_true',
483 dest='no_sharing', default=None,
484 help='clear object sharing policy')
485 parser.add_option('-f', action='store',
486 dest='srcpath', default=None,
487 help='file descriptor to read from: pass - for standard input')
488 parser.add_option('--public', action='store',
489 dest='public', default=None,
490 help='publish/unpublish object (\'True\'/\'False\')')
492 def execute(self, path, *args):
493 if path.find('=') != -1:
494 raise Fault('Missing path argument')
498 headers['X_OBJECT_MANIFEST'] = self.manifest
500 headers['X_OBJECT_SHARING'] = self.sharing
503 headers['X_OBJECT_SHARING'] = ''
505 attrs = ['content-encoding', 'content-disposition']
506 attrs = [a for a in attrs if getattr(self, a)]
508 headers[a.replace('-', '_').upper()] = getattr(self, a)
510 #prepare user defined meta
513 key, sep, val = arg.partition('=')
516 container, sep, object = path.partition('/')
521 f = self.srcpath != '-' and open(self.srcpath) or stdin
523 chunked = True if (self.chunked or f == stdin) else False
524 if self.public not in ['True', 'False', None]:
525 raise Fault('Not acceptable value for public')
526 public = eval(self.public) if self.public else None
527 self.client.update_object(container, object, f, chunked=chunked,
528 headers=headers, offset=self.offset,
529 public=public, **meta)
533 @cli_command('move', 'mv')
534 class MoveObject(Command):
535 syntax = '<src container>/<src object> [<dst container>/]<dst object>'
536 description = 'move an object to a different location'
538 def add_options(self, parser):
539 parser.add_option('--public', action='store',
540 dest='public', default=None,
541 help='publish/unpublish object (\'True\'/\'False\')')
543 def execute(self, src, dst):
544 src_container, sep, src_object = src.partition('/')
545 dst_container, sep, dst_object = dst.partition('/')
547 dst_container = src_container
549 if self.public not in ['True', 'False', None]:
550 raise Fault('Not acceptable value for public')
551 public = eval(self.public) if self.public else None
552 self.client.move_object(src_container, src_object, dst_container,
553 dst_object, public, headers)
555 @cli_command('remove')
556 class TrashObject(Command):
557 syntax = '<container>/<object>'
558 description = 'trash an object'
560 def execute(self, src):
561 src_container, sep, src_object = src.partition('/')
563 self.client.trash_object(src_container, src_object)
565 @cli_command('restore')
566 class RestoreObject(Command):
567 syntax = '<container>/<object>'
568 description = 'restore a trashed object'
570 def execute(self, src):
571 src_container, sep, src_object = src.partition('/')
573 self.client.restore_object(src_container, src_object)
575 @cli_command('unset')
576 class UnsetObject(Command):
577 syntax = '<container>/[<object>] key [key] [...]'
578 description = 'delete metadata info'
580 def execute(self, path, *args):
581 #in case of account fix the args
590 container, sep, object = path.partition('/')
592 self.client.delete_object_metadata(container, object, meta)
594 self.client.delete_container_metadata(container, meta)
596 self.client.delete_account_metadata(meta)
598 @cli_command('group')
599 class SetGroup(Command):
600 syntax = 'key=val [key=val] [...]'
601 description = 'set group account info'
603 def execute(self, *args):
606 key, sep, val = arg.partition('=')
608 self.client.set_account_groups(**groups)
610 @cli_command('policy')
611 class SetPolicy(Command):
612 syntax = 'container key=val [key=val] [...]'
613 description = 'set container policies'
615 def execute(self, path, *args):
616 if path.find('=') != -1:
617 raise Fault('Missing container argument')
619 container, sep, object = path.partition('/')
622 raise Fault('Only containers have policies')
626 key, sep, val = arg.partition('=')
629 self.client.set_container_policies(container, **policies)
631 @cli_command('publish')
632 class PublishObject(Command):
633 syntax = '<container>/<object>'
634 description = 'publish an object'
636 def execute(self, src):
637 src_container, sep, src_object = src.partition('/')
639 self.client.publish_object(src_container, src_object)
641 @cli_command('unpublish')
642 class UnpublishObject(Command):
643 syntax = '<container>/<object>'
644 description = 'unpublish an object'
646 def execute(self, src):
647 src_container, sep, src_object = src.partition('/')
649 self.client.unpublish_object(src_container, src_object)
652 cmd = Command('', [])
654 parser.usage = '%prog <command> [options]'
658 for cls in set(_cli_commands.values()):
659 name = ', '.join(cls.commands)
660 description = getattr(cls, 'description', '')
661 commands.append(' %s %s' % (name.ljust(12), description))
662 print '\nCommands:\n' + '\n'.join(sorted(commands))
664 def print_dict(d, header='name', f=stdout, detail=True):
665 header = header in d and header or 'subdir'
666 if header and header in d:
667 f.write('%s\n' %d.pop(header))
669 patterns = ['^x_(account|container|object)_meta_(\w+)$']
670 patterns.append(patterns[0].replace('_', '-'))
671 for key, val in sorted(d.items()):
677 f.write('%s: %s\n' % (key.rjust(30), val))
679 def print_list(l, verbose=False, f=stdout, detail=True):
681 #if it's empty string continue
684 if type(elem) == types.DictionaryType:
685 print_dict(elem, f=f, detail=detail)
686 elif type(elem) == types.StringType:
688 elem = elem.split('Traceback')[0]
689 f.write('%s\n' % elem)
691 f.write('%s\n' % elem)
693 def print_versions(data, f=stdout):
694 if 'versions' not in data:
695 f.write('%s\n' %data)
697 f.write('versions:\n')
698 for id, t in data['versions']:
699 f.write('%s @ %s\n' % (str(id).rjust(30), datetime.fromtimestamp(t)))
703 return os.environ['PITHOS_USER']
709 return os.environ['PITHOS_AUTH']
717 cls = class_for_cli_command(name)
718 except (IndexError, KeyError):
722 cmd = cls(name, argv[2:])
725 cmd.execute(*cmd.args)
727 cmd.parser.print_help()
730 status = f.status and '%s ' % f.status or ''
731 print '%s%s' % (status, f.data)
733 if __name__ == '__main__':