3 # Copyright 2011 GRNET S.A. All rights reserved.
5 # Redistribution and use in source and binary forms, with or
6 # without modification, are permitted provided that the following
9 # 1. Redistributions of source code must retain the above
10 # copyright notice, this list of conditions and the following
13 # 2. Redistributions in binary form must reproduce the above
14 # copyright notice, this list of conditions and the following
15 # disclaimer in the documentation and/or other materials
16 # provided with the distribution.
18 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 # POSSIBILITY OF SUCH DAMAGE.
31 # The views and conclusions contained in the software and
32 # documentation are those of the authors and should not be
33 # interpreted as representing official policies, either expressed
34 # or implied, of GRNET S.A.
36 from getpass import getuser
37 from optparse import OptionParser
38 from os import environ
39 from sys import argv, exit, stdin, stdout
40 from pithos.lib.client import Client, Fault
41 from datetime import datetime
50 DEFAULT_HOST = 'pithos.dev.grnet.gr'
51 #DEFAULT_HOST = '127.0.0.1:8000'
56 def cli_command(*args):
60 _cli_commands[name] = cls
64 def class_for_cli_command(name):
65 return _cli_commands[name]
67 class Command(object):
70 def __init__(self, name, argv):
71 parser = OptionParser('%%prog %s [options] %s' % (name, self.syntax))
72 parser.add_option('--host', dest='host', metavar='HOST',
73 default=DEFAULT_HOST, help='use server HOST')
74 parser.add_option('--user', dest='user', metavar='USERNAME',
76 help='use account USERNAME')
77 parser.add_option('--token', dest='token', metavar='AUTH',
79 help='use account AUTH')
80 parser.add_option('--api', dest='api', metavar='API',
81 default=DEFAULT_API, help='use api API')
82 parser.add_option('-v', action='store_true', dest='verbose',
83 default=False, help='use verbose output')
84 parser.add_option('-d', action='store_true', dest='debug',
85 default=False, help='use debug output')
86 self.add_options(parser)
87 options, args = parser.parse_args(argv)
90 for opt in parser.option_list:
93 val = getattr(options, key)
94 setattr(self, key, val)
96 self.client = Client(self.host, self.token, self.user, self.api, self.verbose,
102 def add_options(self, parser):
105 def execute(self, *args):
108 @cli_command('list', 'ls')
110 syntax = '[<container>[/<object>]]'
111 description = 'list containers or objects'
113 def add_options(self, parser):
114 parser.add_option('-l', action='store_true', dest='detail',
115 default=False, help='show detailed output')
116 parser.add_option('-n', action='store', type='int', dest='limit',
117 default=1000, help='show limited output')
118 parser.add_option('--marker', action='store', type='str',
119 dest='marker', default=None,
120 help='show output greater then marker')
121 parser.add_option('--prefix', action='store', type='str',
122 dest='prefix', default=None,
123 help='show output starting with prefix')
124 parser.add_option('--delimiter', action='store', type='str',
125 dest='delimiter', default=None,
126 help='show output up to the delimiter')
127 parser.add_option('--path', action='store', type='str',
128 dest='path', default=None,
129 help='show output starting with prefix up to /')
130 parser.add_option('--meta', action='store', type='str',
131 dest='meta', default=None,
132 help='show output having the specified meta keys')
133 parser.add_option('--if-modified-since', action='store', type='str',
134 dest='if_modified_since', default=None,
135 help='show output if modified since then')
136 parser.add_option('--if-unmodified-since', action='store', type='str',
137 dest='if_unmodified_since', default=None,
138 help='show output if not modified since then')
139 parser.add_option('--until', action='store', dest='until',
140 default=False, help='show metadata until that date')
141 parser.add_option('--format', action='store', dest='format',
142 default='%d/%m/%Y', help='format to parse until date')
144 def execute(self, container=None):
146 self.list_objects(container)
148 self.list_containers()
150 def list_containers(self):
151 params = {'limit':self.limit, 'marker':self.marker}
152 headers = {'IF_MODIFIED_SINCE':self.if_modified_since,
153 'IF_UNMODIFIED_SINCE':self.if_unmodified_since}
156 t = _time.strptime(self.until, self.format)
157 params['until'] = int(_time.mktime(t))
159 l = self.client.list_containers(self.detail, params, headers)
162 def list_objects(self, container):
165 attrs = ['limit', 'marker', 'prefix', 'delimiter', 'path', 'meta']
166 for a in [a for a in attrs if getattr(self, a)]:
167 params[a] = getattr(self, a)
170 t = _time.strptime(self.until, self.format)
171 params['until'] = int(_time.mktime(t))
173 headers = {'IF_MODIFIED_SINCE':self.if_modified_since,
174 'IF_UNMODIFIED_SINCE':self.if_unmodified_since}
175 container, sep, object = container.partition('/')
180 #if request with meta quering disable trash filtering
181 show_trashed = True if self.meta else False
182 l = self.client.list_objects(container, detail, headers,
183 include_trashed = show_trashed, **params)
184 print_list(l, detail=self.detail)
188 syntax = '[<container>[/<object>]]'
189 description = 'get account/container/object metadata'
191 def add_options(self, parser):
192 parser.add_option('-r', action='store_true', dest='restricted',
193 default=False, help='show only user defined metadata')
194 parser.add_option('--until', action='store', dest='until',
195 default=False, help='show metadata until that date')
196 parser.add_option('--format', action='store', dest='format',
197 default='%d/%m/%Y', help='format to parse until date')
198 parser.add_option('--version', action='store', dest='version',
199 default=None, help='show specific version \
200 (applies only for objects)')
202 def execute(self, path=''):
203 container, sep, object = path.partition('/')
205 t = _time.strptime(self.until, self.format)
206 self.until = int(_time.mktime(t))
208 meta = self.client.retrieve_object_metadata(container, object,
212 meta = self.client.retrieve_container_metadata(container,
216 meta = self.client.account_metadata(self.restricted, self.until)
218 print 'Entity does not exist'
220 print_dict(meta, header=None)
222 @cli_command('create')
223 class CreateContainer(Command):
224 syntax = '<container> [key=val] [...]'
225 description = 'create a container'
227 def execute(self, container, *args):
231 key, sep, val = arg.partition('=')
233 ret = self.client.create_container(container, headers, **meta)
235 print 'Container already exists'
237 @cli_command('delete', 'rm')
238 class Delete(Command):
239 syntax = '<container>[/<object>]'
240 description = 'delete a container or an object'
242 def execute(self, path):
243 container, sep, object = path.partition('/')
245 self.client.delete_object(container, object)
247 self.client.delete_container(container)
250 class GetObject(Command):
251 syntax = '<container>/<object>'
252 description = 'get the data of an object'
254 def add_options(self, parser):
255 parser.add_option('-l', action='store_true', dest='detail',
256 default=False, help='show detailed output')
257 parser.add_option('--range', action='store', dest='range',
258 default=None, help='show range of data')
259 parser.add_option('--if-range', action='store', dest='if-range',
260 default=None, help='show range of data')
261 parser.add_option('--if-match', action='store', dest='if-match',
262 default=None, help='show output if ETags match')
263 parser.add_option('--if-none-match', action='store',
264 dest='if-none-match', default=None,
265 help='show output if ETags don\'t match')
266 parser.add_option('--if-modified-since', action='store', type='str',
267 dest='if-modified-since', default=None,
268 help='show output if modified since then')
269 parser.add_option('--if-unmodified-since', action='store', type='str',
270 dest='if-unmodified-since', default=None,
271 help='show output if not modified since then')
272 parser.add_option('-o', action='store', type='str',
273 dest='file', default=None,
274 help='save output in file')
275 parser.add_option('--version', action='store', type='str',
276 dest='version', default=None,
277 help='get the specific \
279 parser.add_option('--versionlist', action='store_true',
280 dest='versionlist', default=False,
281 help='get the full object version list')
283 def execute(self, path):
286 headers['RANGE'] = 'bytes=%s' %self.range
287 if getattr(self, 'if-range'):
288 headers['IF_RANGE'] = 'If-Range:%s' % getattr(self, 'if-range')
289 attrs = ['if-match', 'if-none-match', 'if-modified-since',
290 'if-unmodified-since']
291 attrs = [a for a in attrs if getattr(self, a)]
293 headers[a.replace('-', '_').upper()] = getattr(self, a)
294 container, sep, object = path.partition('/')
296 self.version = 'list'
298 data = self.client.retrieve_object(container, object, self.detail,
299 headers, self.version)
300 f = self.file and open(self.file, 'w') or stdout
302 data = json.loads(data)
304 print_versions(data, f=f)
306 print_dict(data, f=f)
311 @cli_command('mkdir')
312 class PutMarker(Command):
313 syntax = '<container>/<directory marker>'
314 description = 'create a directory marker'
316 def execute(self, path):
317 container, sep, object = path.partition('/')
318 self.client.create_directory_marker(container, object)
321 class PutObject(Command):
322 syntax = '<container>/<object> [key=val] [...]'
323 description = 'create/override object'
325 def add_options(self, parser):
326 parser.add_option('--use_hashes', action='store_true', dest='use_hashes',
327 default=False, help='provide hashmap instead of data')
328 parser.add_option('--chunked', action='store_true', dest='chunked',
329 default=False, help='set chunked transfer mode')
330 parser.add_option('--etag', action='store', dest='etag',
331 default=None, help='check written data')
332 parser.add_option('--content-encoding', action='store',
333 dest='content-encoding', default=None,
334 help='provide the object MIME content type')
335 parser.add_option('--content-disposition', action='store', type='str',
336 dest='content-disposition', default=None,
337 help='provide the presentation style of the object')
338 parser.add_option('-S', action='store',
339 dest='segment-size', default=False,
340 help='use for large file support')
341 parser.add_option('--manifest', action='store_true',
342 dest='manifest', default=None,
343 help='upload a manifestation file')
344 parser.add_option('--type', action='store',
345 dest='content-type', default=False,
346 help='create object with specific content type')
347 parser.add_option('--sharing', action='store',
348 dest='sharing', default=None,
349 help='define sharing object policy')
350 parser.add_option('-f', action='store',
351 dest='srcpath', default=None,
352 help='file descriptor to read from (pass - for standard input)')
353 parser.add_option('--public', action='store',
354 dest='public', default=None,
355 help='make object publicly accessible (\'True\'/\'False\')')
357 def execute(self, path, *args):
358 if path.find('=') != -1:
359 raise Fault('Missing path argument')
361 #prepare user defined meta
364 key, sep, val = arg.partition('=')
368 manifest = getattr(self, 'manifest')
370 # if it's manifestation file
371 # send zero-byte data with X-Object-Manifest header
373 headers['X_OBJECT_MANIFEST'] = manifest
375 headers['X_OBJECT_SHARING'] = self.sharing
377 attrs = ['etag', 'content-encoding', 'content-disposition',
379 attrs = [a for a in attrs if getattr(self, a)]
381 headers[a.replace('-', '_').upper()] = getattr(self, a)
383 container, sep, object = path.partition('/')
387 f = open(self.srcpath) if self.srcpath != '-' else stdin
389 if self.use_hashes and not f:
390 raise Fault('Illegal option combination')
391 if self.public not in ['True', 'False', None]:
392 raise Fault('Not acceptable value for public')
393 public = eval(self.public) if self.public else None
394 self.client.create_object(container, object, f, chunked=self.chunked,
395 headers=headers, use_hashes=self.use_hashes,
396 public=public, **meta)
400 @cli_command('copy', 'cp')
401 class CopyObject(Command):
402 syntax = '<src container>/<src object> [<dst container>/]<dst object>'
403 description = 'copy an object to a different location'
405 def add_options(self, parser):
406 parser.add_option('--version', action='store',
407 dest='version', default=False,
408 help='copy specific version')
409 parser.add_option('--public', action='store',
410 dest='public', default=None,
411 help='publish/unpublish object (\'True\'/\'False\')')
413 def execute(self, src, dst):
414 src_container, sep, src_object = src.partition('/')
415 dst_container, sep, dst_object = dst.partition('/')
417 dst_container = src_container
419 version = getattr(self, 'version')
423 headers['X_SOURCE_VERSION'] = version
424 if self.public and self.nopublic:
425 raise Fault('Conflicting options')
426 if self.public not in ['True', 'False', None]:
427 raise Fault('Not acceptable value for public')
428 public = eval(self.public) if self.public else None
429 self.client.copy_object(src_container, src_object, dst_container,
430 dst_object, public, headers)
433 class SetMeta(Command):
434 syntax = '[<container>[/<object>]] key=val [key=val] [...]'
435 description = 'set account/container/object metadata'
437 def execute(self, path, *args):
438 #in case of account fix the args
439 if path.find('=') != -1:
446 key, sep, val = arg.partition('=')
447 meta[key.strip()] = val.strip()
448 container, sep, object = path.partition('/')
450 self.client.update_object_metadata(container, object, **meta)
452 self.client.update_container_metadata(container, **meta)
454 self.client.update_account_metadata(**meta)
456 @cli_command('update')
457 class UpdateObject(Command):
458 syntax = '<container>/<object> path [key=val] [...]'
459 description = 'update object metadata/data (default mode: append)'
461 def add_options(self, parser):
462 parser.add_option('-a', action='store_true', dest='append',
463 default=True, help='append data')
464 parser.add_option('--offset', action='store',
466 default=None, help='starting offest to be updated')
467 parser.add_option('--range', action='store', dest='content-range',
468 default=None, help='range of data to be updated')
469 parser.add_option('--chunked', action='store_true', dest='chunked',
470 default=False, help='set chunked transfer mode')
471 parser.add_option('--content-encoding', action='store',
472 dest='content-encoding', default=None,
473 help='provide the object MIME content type')
474 parser.add_option('--content-disposition', action='store', type='str',
475 dest='content-disposition', default=None,
476 help='provide the presentation style of the object')
477 parser.add_option('--manifest', action='store', type='str',
478 dest='manifest', default=None,
479 help='use for large file support')
480 parser.add_option('--sharing', action='store',
481 dest='sharing', default=None,
482 help='define sharing object policy')
483 parser.add_option('--nosharing', action='store_true',
484 dest='no_sharing', default=None,
485 help='clear object sharing policy')
486 parser.add_option('-f', action='store',
487 dest='srcpath', default=None,
488 help='file descriptor to read from: pass - for standard input')
489 parser.add_option('--public', action='store',
490 dest='public', default=None,
491 help='publish/unpublish object (\'True\'/\'False\')')
493 def execute(self, path, *args):
494 if path.find('=') != -1:
495 raise Fault('Missing path argument')
499 headers['X_OBJECT_MANIFEST'] = self.manifest
501 headers['X_OBJECT_SHARING'] = self.sharing
503 headers['X_OBJECT_SHARING'] = ''
505 attrs = ['content-encoding', 'content-disposition']
506 attrs = [a for a in attrs if getattr(self, a)]
508 headers[a.replace('-', '_').upper()] = getattr(self, a)
510 #prepare user defined meta
513 key, sep, val = arg.partition('=')
516 container, sep, object = path.partition('/')
521 f = self.srcpath != '-' and open(self.srcpath) or stdin
523 chunked = True if (self.chunked or f == stdin) else False
524 if self.public not in ['True', 'False', None]:
525 raise Fault('Not acceptable value for public')
526 public = eval(self.public) if self.public else None
527 self.client.update_object(container, object, f, chunked=chunked,
528 headers=headers, offset=self.offset,
529 public=public, **meta)
533 @cli_command('move', 'mv')
534 class MoveObject(Command):
535 syntax = '<src container>/<src object> [<dst container>/]<dst object>'
536 description = 'move an object to a different location'
538 def add_options(self, parser):
539 parser.add_option('--public', action='store',
540 dest='public', default=None,
541 help='publish/unpublish object (\'True\'/\'False\')')
543 def execute(self, src, dst):
544 src_container, sep, src_object = src.partition('/')
545 dst_container, sep, dst_object = dst.partition('/')
547 dst_container = src_container
549 if self.public not in ['True', 'False', None]:
550 raise Fault('Not acceptable value for public')
551 public = eval(self.public) if self.public else None
552 self.client.move_object(src_container, src_object, dst_container,
553 dst_object, public, headers)
555 @cli_command('remove')
556 class TrashObject(Command):
557 syntax = '<container>/<object>'
558 description = 'trash an object'
560 def execute(self, src):
561 src_container, sep, src_object = src.partition('/')
563 self.client.trash_object(src_container, src_object)
565 @cli_command('restore')
566 class RestoreObject(Command):
567 syntax = '<container>/<object>'
568 description = 'restore a trashed object'
570 def execute(self, src):
571 src_container, sep, src_object = src.partition('/')
573 self.client.restore_object(src_container, src_object)
575 @cli_command('unset')
576 class UnsetObject(Command):
577 syntax = '<container>/[<object>] key [key] [...]'
578 description = 'delete metadata info'
580 def execute(self, path, *args):
581 #in case of account fix the args
590 container, sep, object = path.partition('/')
592 self.client.delete_object_metadata(container, object, meta)
594 self.client.delete_container_metadata(container, meta)
596 self.client.delete_account_metadata(meta)
598 @cli_command('group')
599 class CreateGroup(Command):
600 syntax = 'key=val [key=val] [...]'
601 description = 'create account groups'
603 def execute(self, *args):
606 key, sep, val = arg.partition('=')
608 self.client.set_account_groups(**groups)
610 @cli_command('ungroup')
611 class DeleteGroup(Command):
612 syntax = 'key [key] [...]'
613 description = 'delete account groups'
615 def execute(self, *args):
619 self.client.unset_account_groups(groups)
621 @cli_command('policy')
622 class SetPolicy(Command):
623 syntax = 'container key=val [key=val] [...]'
624 description = 'set container policies'
626 def execute(self, path, *args):
627 if path.find('=') != -1:
628 raise Fault('Missing container argument')
630 container, sep, object = path.partition('/')
633 raise Fault('Only containers have policies')
637 key, sep, val = arg.partition('=')
640 self.client.set_container_policies(container, **policies)
642 @cli_command('publish')
643 class PublishObject(Command):
644 syntax = '<container>/<object>'
645 description = 'publish an object'
647 def execute(self, src):
648 src_container, sep, src_object = src.partition('/')
650 self.client.publish_object(src_container, src_object)
652 @cli_command('unpublish')
653 class UnpublishObject(Command):
654 syntax = '<container>/<object>'
655 description = 'unpublish an object'
657 def execute(self, src):
658 src_container, sep, src_object = src.partition('/')
660 self.client.unpublish_object(src_container, src_object)
663 cmd = Command('', [])
665 parser.usage = '%prog <command> [options]'
669 for cls in set(_cli_commands.values()):
670 name = ', '.join(cls.commands)
671 description = getattr(cls, 'description', '')
672 commands.append(' %s %s' % (name.ljust(12), description))
673 print '\nCommands:\n' + '\n'.join(sorted(commands))
675 def print_dict(d, header='name', f=stdout, detail=True):
676 header = header in d and header or 'subdir'
677 if header and header in d:
678 f.write('%s\n' %d.pop(header))
680 patterns = ['^x_(account|container|object)_meta_(\w+)$']
681 patterns.append(patterns[0].replace('_', '-'))
682 for key, val in sorted(d.items()):
688 f.write('%s: %s\n' % (key.rjust(30), val))
690 def print_list(l, verbose=False, f=stdout, detail=True):
692 #if it's empty string continue
695 if type(elem) == types.DictionaryType:
696 print_dict(elem, f=f, detail=detail)
697 elif type(elem) == types.StringType:
699 elem = elem.split('Traceback')[0]
700 f.write('%s\n' % elem)
702 f.write('%s\n' % elem)
704 def print_versions(data, f=stdout):
705 if 'versions' not in data:
706 f.write('%s\n' %data)
708 f.write('versions:\n')
709 for id, t in data['versions']:
710 f.write('%s @ %s\n' % (str(id).rjust(30), datetime.fromtimestamp(t)))
714 return os.environ['PITHOS_USER']
720 return os.environ['PITHOS_AUTH']
728 cls = class_for_cli_command(name)
729 except (IndexError, KeyError):
733 cmd = cls(name, argv[2:])
736 cmd.execute(*cmd.args)
738 cmd.parser.print_help()
741 status = f.status and '%s ' % f.status or ''
742 print '%s%s' % (status, f.data)
744 if __name__ == '__main__':