3 # Copyright 2011 GRNET S.A. All rights reserved.
5 # Redistribution and use in source and binary forms, with or
6 # without modification, are permitted provided that the following
9 # 1. Redistributions of source code must retain the above
10 # copyright notice, this list of conditions and the following
13 # 2. Redistributions in binary form must reproduce the above
14 # copyright notice, this list of conditions and the following
15 # disclaimer in the documentation and/or other materials
16 # provided with the distribution.
18 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 # POSSIBILITY OF SUCH DAMAGE.
31 # The views and conclusions contained in the software and
32 # documentation are those of the authors and should not be
33 # interpreted as representing official policies, either expressed
34 # or implied, of GRNET S.A.
36 from getpass import getuser
37 from optparse import OptionParser
38 from os import environ
39 from sys import argv, exit, stdin, stdout
40 from pithos.lib.client import Client, Fault
41 from datetime import datetime
50 DEFAULT_HOST = 'pithos.dev.grnet.gr'
55 def cli_command(*args):
59 _cli_commands[name] = cls
63 def class_for_cli_command(name):
64 return _cli_commands[name]
66 class Command(object):
69 def __init__(self, name, argv):
70 parser = OptionParser('%%prog %s [options] %s' % (name, self.syntax))
71 parser.add_option('--host', dest='host', metavar='HOST',
72 default=DEFAULT_HOST, help='use server HOST')
73 parser.add_option('--user', dest='user', metavar='USERNAME',
75 help='use account USERNAME')
76 parser.add_option('--token', dest='token', metavar='AUTH',
78 help='use account AUTH')
79 parser.add_option('--api', dest='api', metavar='API',
80 default=DEFAULT_API, help='use api API')
81 parser.add_option('-v', action='store_true', dest='verbose',
82 default=False, help='use verbose output')
83 parser.add_option('-d', action='store_true', dest='debug',
84 default=False, help='use debug output')
85 self.add_options(parser)
86 options, args = parser.parse_args(argv)
89 for opt in parser.option_list:
92 val = getattr(options, key)
93 setattr(self, key, val)
95 self.client = Client(self.host, self.token, self.user, self.api, self.verbose,
101 def add_options(self, parser):
104 def execute(self, *args):
107 @cli_command('list', 'ls')
109 syntax = '[<container>[/<object>]]'
110 description = 'list containers or objects'
112 def add_options(self, parser):
113 parser.add_option('-l', action='store_true', dest='detail',
114 default=False, help='show detailed output')
115 parser.add_option('-n', action='store', type='int', dest='limit',
116 default=1000, help='show limited output')
117 parser.add_option('--marker', action='store', type='str',
118 dest='marker', default=None,
119 help='show output greater then marker')
120 parser.add_option('--prefix', action='store', type='str',
121 dest='prefix', default=None,
122 help='show output starting with prefix')
123 parser.add_option('--delimiter', action='store', type='str',
124 dest='delimiter', default=None,
125 help='show output up to the delimiter')
126 parser.add_option('--path', action='store', type='str',
127 dest='path', default=None,
128 help='show output starting with prefix up to /')
129 parser.add_option('--meta', action='store', type='str',
130 dest='meta', default=None,
131 help='show output having the specified meta keys')
132 parser.add_option('--if-modified-since', action='store', type='str',
133 dest='if_modified_since', default=None,
134 help='show output if modified since then')
135 parser.add_option('--if-unmodified-since', action='store', type='str',
136 dest='if_unmodified_since', default=None,
137 help='show output if not modified since then')
138 parser.add_option('--until', action='store', dest='until',
139 default=False, help='show metadata until that date')
140 parser.add_option('--format', action='store', dest='format',
141 default='%d/%m/%Y', help='format to parse until date')
143 def execute(self, container=None):
145 self.list_objects(container)
147 self.list_containers()
149 def list_containers(self):
150 params = {'limit':self.limit, 'marker':self.marker}
151 headers = {'IF_MODIFIED_SINCE':self.if_modified_since,
152 'IF_UNMODIFIED_SINCE':self.if_unmodified_since}
155 t = _time.strptime(self.until, self.format)
156 params['until'] = int(_time.mktime(t))
158 l = self.client.list_containers(self.detail, params, headers)
161 def list_objects(self, container):
164 attrs = ['limit', 'marker', 'prefix', 'delimiter', 'path', 'meta']
165 for a in [a for a in attrs if getattr(self, a)]:
166 params[a] = getattr(self, a)
169 t = _time.strptime(self.until, self.format)
170 params['until'] = int(_time.mktime(t))
172 headers = {'IF_MODIFIED_SINCE':self.if_modified_since,
173 'IF_UNMODIFIED_SINCE':self.if_unmodified_since}
174 container, sep, object = container.partition('/')
179 #if request with meta quering disable trash filtering
180 show_trashed = True if self.meta else False
181 l = self.client.list_objects(container, detail, headers,
182 include_trashed = show_trashed, **params)
183 print_list(l, detail=self.detail)
187 syntax = '[<container>[/<object>]]'
188 description = 'get the metadata of an account, a container or an object'
190 def add_options(self, parser):
191 parser.add_option('-r', action='store_true', dest='restricted',
192 default=False, help='show only user defined metadata')
193 parser.add_option('--until', action='store', dest='until',
194 default=False, help='show metadata until that date')
195 parser.add_option('--format', action='store', dest='format',
196 default='%d/%m/%Y', help='format to parse until date')
197 parser.add_option('--version', action='store', dest='version',
198 default=None, help='show specific version \
199 (applies only for objects)')
201 def execute(self, path=''):
202 container, sep, object = path.partition('/')
204 t = _time.strptime(self.until, self.format)
205 self.until = int(_time.mktime(t))
207 meta = self.client.retrieve_object_metadata(container, object,
211 meta = self.client.retrieve_container_metadata(container,
215 meta = self.client.account_metadata(self.restricted, self.until)
217 print 'Entity does not exist'
219 print_dict(meta, header=None)
221 @cli_command('create')
222 class CreateContainer(Command):
223 syntax = '<container> [key=val] [...]'
224 description = 'create a container'
226 def execute(self, container, *args):
230 key, sep, val = arg.partition('=')
232 ret = self.client.create_container(container, headers, **meta)
234 print 'Container already exists'
236 @cli_command('delete', 'rm')
237 class Delete(Command):
238 syntax = '<container>[/<object>]'
239 description = 'delete a container or an object'
241 def execute(self, path):
242 container, sep, object = path.partition('/')
244 self.client.delete_object(container, object)
246 self.client.delete_container(container)
249 class GetObject(Command):
250 syntax = '<container>/<object>'
251 description = 'get the data of an object'
253 def add_options(self, parser):
254 parser.add_option('-l', action='store_true', dest='detail',
255 default=False, help='show detailed output')
256 parser.add_option('--range', action='store', dest='range',
257 default=None, help='show range of data')
258 parser.add_option('--if-match', action='store', dest='if-match',
259 default=None, help='show output if ETags match')
260 parser.add_option('--if-none-match', action='store',
261 dest='if-none-match', default=None,
262 help='show output if ETags don\'t match')
263 parser.add_option('--if-modified-since', action='store', type='str',
264 dest='if-modified-since', default=None,
265 help='show output if modified since then')
266 parser.add_option('--if-unmodified-since', action='store', type='str',
267 dest='if-unmodified-since', default=None,
268 help='show output if not modified since then')
269 parser.add_option('-o', action='store', type='str',
270 dest='file', default=None,
271 help='save output in file')
272 parser.add_option('--version', action='store', type='str',
273 dest='version', default=None,
274 help='get the specific \
276 parser.add_option('--versionlist', action='store_true',
277 dest='versionlist', default=False,
278 help='get the full object version list')
280 def execute(self, path):
283 headers['RANGE'] = 'bytes=%s' %self.range
284 attrs = ['if-match', 'if-none-match', 'if-modified-since',
285 'if-unmodified-since']
286 attrs = [a for a in attrs if getattr(self, a)]
288 headers[a.replace('-', '_').upper()] = getattr(self, a)
289 container, sep, object = path.partition('/')
291 self.version = 'list'
293 data = self.client.retrieve_object(container, object, self.detail,
294 headers, self.version)
295 f = self.file and open(self.file, 'w') or stdout
297 data = json.loads(data)
299 print_versions(data, f=f)
301 print_dict(data, f=f)
306 @cli_command('mkdir')
307 class PutMarker(Command):
308 syntax = '<container>/<directory marker>'
309 description = 'create a directory marker'
311 def execute(self, path):
312 container, sep, object = path.partition('/')
313 self.client.create_directory_marker(container, object)
316 class PutObject(Command):
317 syntax = '<container>/<object> <path> [key=val] [...]'
318 description = 'create/override object'
320 def add_options(self, parser):
321 parser.add_option('--use_hashes', action='store_true', dest='use_hashes',
322 default=False, help='provide hashmap instead of data')
323 parser.add_option('--chunked', action='store_true', dest='chunked',
324 default=False, help='set chunked transfer mode')
325 parser.add_option('--etag', action='store', dest='etag',
326 default=None, help='check written data')
327 parser.add_option('--content-encoding', action='store',
328 dest='content-encoding', default=None,
329 help='provide the object MIME content type')
330 parser.add_option('--content-disposition', action='store', type='str',
331 dest='content-disposition', default=None,
332 help='provide the presentation style of the object')
333 parser.add_option('-S', action='store',
334 dest='segment-size', default=False,
335 help='use for large file support')
336 parser.add_option('--manifest', action='store_true',
337 dest='manifest', default=None,
338 help='upload a manifestation file')
339 parser.add_option('--type', action='store',
340 dest='content-type', default=False,
341 help='create object with specific content type')
342 parser.add_option('--sharing', action='store',
343 dest='sharing', default=None,
344 help='define sharing object policy')
345 parser.add_option('-f', action='store',
346 dest='srcpath', default=None,
347 help='file descriptor to read from (pass - for standard input)')
348 parser.add_option('--public', action='store',
349 dest='public', default=None,
350 help='make object publicly accessible (\'True\'/\'False\')')
352 def execute(self, path, *args):
353 if path.find('=') != -1:
354 raise Fault('Missing path argument')
356 #prepare user defined meta
359 key, sep, val = arg.partition('=')
363 manifest = getattr(self, 'manifest')
365 # if it's manifestation file
366 # send zero-byte data with X-Object-Manifest header
368 headers['X_OBJECT_MANIFEST'] = manifest
370 headers['X_OBJECT_SHARING'] = self.sharing
372 attrs = ['etag', 'content-encoding', 'content-disposition',
374 attrs = [a for a in attrs if getattr(self, a)]
376 headers[a.replace('-', '_').upper()] = getattr(self, a)
378 container, sep, object = path.partition('/')
382 f = open(self.srcpath) if self.srcpath != '-' else stdin
384 if self.use_hashes and not f:
385 raise Fault('Illegal option combination')
386 if self.public not in ['True', 'False', None]:
387 raise Fault('Not acceptable value for public')
388 public = eval(self.public) if self.public else None
389 self.client.create_object(container, object, f, chunked=self.chunked,
390 headers=headers, use_hashes=self.use_hashes,
391 public=public, **meta)
395 @cli_command('copy', 'cp')
396 class CopyObject(Command):
397 syntax = '<src container>/<src object> [<dst container>/]<dst object>'
398 description = 'copy an object to a different location'
400 def add_options(self, parser):
401 parser.add_option('--version', action='store',
402 dest='version', default=False,
403 help='copy specific version')
404 parser.add_option('--public', action='store',
405 dest='public', default=None,
406 help='publish/unpublish object (\'True\'/\'False\')')
408 def execute(self, src, dst):
409 src_container, sep, src_object = src.partition('/')
410 dst_container, sep, dst_object = dst.partition('/')
412 dst_container = src_container
414 version = getattr(self, 'version')
417 headers['X_SOURCE_VERSION'] = version
418 if self.public and self.nopublic:
419 raise Fault('Conflicting options')
420 if self.public not in ['True', 'False', None]:
421 raise Fault('Not acceptable value for public')
422 public = eval(self.public) if self.public else None
423 self.client.copy_object(src_container, src_object, dst_container,
424 dst_object, public, headers)
427 class SetMeta(Command):
428 syntax = '[<container>[/<object>]] key=val [key=val] [...]'
429 description = 'set metadata'
431 def execute(self, path, *args):
432 #in case of account fix the args
433 if path.find('=') != -1:
440 key, sep, val = arg.partition('=')
441 meta[key.strip()] = val.strip()
442 container, sep, object = path.partition('/')
444 self.client.update_object_metadata(container, object, **meta)
446 self.client.update_container_metadata(container, **meta)
448 self.client.update_account_metadata(**meta)
450 @cli_command('update')
451 class UpdateObject(Command):
452 syntax = '<container>/<object> path [key=val] [...]'
453 description = 'update object metadata/data (default mode: append)'
455 def add_options(self, parser):
456 parser.add_option('-a', action='store_true', dest='append',
457 default=True, help='append data')
458 parser.add_option('--offset', action='store',
460 default=None, help='starting offest to be updated')
461 parser.add_option('--range', action='store', dest='content-range',
462 default=None, help='range of data to be updated')
463 parser.add_option('--chunked', action='store_true', dest='chunked',
464 default=False, help='set chunked transfer mode')
465 parser.add_option('--content-encoding', action='store',
466 dest='content-encoding', default=None,
467 help='provide the object MIME content type')
468 parser.add_option('--content-disposition', action='store', type='str',
469 dest='content-disposition', default=None,
470 help='provide the presentation style of the object')
471 parser.add_option('--manifest', action='store', type='str',
472 dest='manifest', default=None,
473 help='use for large file support')
474 parser.add_option('--sharing', action='store',
475 dest='sharing', default=None,
476 help='define sharing object policy')
477 parser.add_option('--nosharing', action='store_true',
478 dest='no_sharing', default=None,
479 help='clear object sharing policy')
480 parser.add_option('-f', action='store',
481 dest='srcpath', default=None,
482 help='file descriptor to read from: pass - for standard input')
483 parser.add_option('--public', action='store',
484 dest='public', default=None,
485 help='publish/unpublish object (\'True\'/\'False\')')
487 def execute(self, path, *args):
488 if path.find('=') != -1:
489 raise Fault('Missing path argument')
493 headers['X_OBJECT_MANIFEST'] = self.manifest
495 headers['X_OBJECT_SHARING'] = self.sharing
498 headers['X_OBJECT_SHARING'] = ''
500 attrs = ['content-encoding', 'content-disposition']
501 attrs = [a for a in attrs if getattr(self, a)]
503 headers[a.replace('-', '_').upper()] = getattr(self, a)
505 #prepare user defined meta
508 key, sep, val = arg.partition('=')
511 container, sep, object = path.partition('/')
516 f = self.srcpath != '-' and open(self.srcpath) or stdin
518 chunked = True if (self.chunked or f == stdin) else False
519 if self.public not in ['True', 'False', None]:
520 raise Fault('Not acceptable value for public')
521 public = eval(self.public) if self.public else None
522 self.client.update_object(container, object, f, chunked=chunked,
523 headers=headers, offset=self.offset,
524 public=public, **meta)
528 @cli_command('move', 'mv')
529 class MoveObject(Command):
530 syntax = '<src container>/<src object> [<dst container>/]<dst object>'
531 description = 'move an object to a different location'
533 def add_options(self, parser):
534 parser.add_option('--public', action='store',
535 dest='public', default=None,
536 help='publish/unpublish object (\'True\'/\'False\')')
538 def execute(self, src, dst):
539 src_container, sep, src_object = src.partition('/')
540 dst_container, sep, dst_object = dst.partition('/')
542 dst_container = src_container
544 if self.public not in ['True', 'False', None]:
545 raise Fault('Not acceptable value for public')
546 public = eval(self.public) if self.public else None
547 self.client.move_object(src_container, src_object, dst_container,
548 dst_object, public, headers)
550 @cli_command('remove')
551 class TrashObject(Command):
552 syntax = '<container>/<object>'
553 description = 'trash an object'
555 def execute(self, src):
556 src_container, sep, src_object = src.partition('/')
558 self.client.trash_object(src_container, src_object)
560 @cli_command('restore')
561 class RestoreObject(Command):
562 syntax = '<container>/<object>'
563 description = 'restore a trashed object'
565 def execute(self, src):
566 src_container, sep, src_object = src.partition('/')
568 self.client.restore_object(src_container, src_object)
570 @cli_command('unset')
571 class UnsetObject(Command):
572 syntax = '<container>/[<object>] key [key] [...]'
573 description = 'delete metadata info'
575 def execute(self, path, *args):
576 #in case of account fix the args
585 container, sep, object = path.partition('/')
587 self.client.delete_object_metadata(container, object, meta)
589 self.client.delete_container_metadata(container, meta)
591 self.client.delete_account_metadata(meta)
593 @cli_command('group')
594 class SetGroup(Command):
595 syntax = 'key=val [key=val] [...]'
596 description = 'set group account info'
598 def execute(self, *args):
601 key, sep, val = arg.partition('=')
603 self.client.set_account_groups(**groups)
605 @cli_command('policy')
606 class SetPolicy(Command):
607 syntax = 'container key=val [key=val] [...]'
608 description = 'set container policies'
610 def execute(self, path, *args):
611 if path.find('=') != -1:
612 raise Fault('Missing container argument')
614 container, sep, object = path.partition('/')
617 raise Fault('Only containers have policies')
621 key, sep, val = arg.partition('=')
624 self.client.set_container_policies(container, **policies)
626 @cli_command('publish')
627 class PublishObject(Command):
628 syntax = '<container>/<object>'
629 description = 'publish an object'
631 def execute(self, src):
632 src_container, sep, src_object = src.partition('/')
634 self.client.publish_object(src_container, src_object)
636 @cli_command('unpublish')
637 class UnpublishObject(Command):
638 syntax = '<container>/<object>'
639 description = 'unpublish an object'
641 def execute(self, src):
642 src_container, sep, src_object = src.partition('/')
644 self.client.unpublish_object(src_container, src_object)
647 cmd = Command('', [])
649 parser.usage = '%prog <command> [options]'
653 for cls in set(_cli_commands.values()):
654 name = ', '.join(cls.commands)
655 description = getattr(cls, 'description', '')
656 commands.append(' %s %s' % (name.ljust(12), description))
657 print '\nCommands:\n' + '\n'.join(sorted(commands))
659 def print_dict(d, header='name', f=stdout, detail=True):
660 header = header in d and header or 'subdir'
661 if header and header in d:
662 f.write('%s\n' %d.pop(header))
664 patterns = ['^x_(account|container|object)_meta_(\w+)$']
665 patterns.append(patterns[0].replace('_', '-'))
666 for key, val in sorted(d.items()):
672 f.write('%s: %s\n' % (key.rjust(30), val))
674 def print_list(l, verbose=False, f=stdout, detail=True):
676 #if it's empty string continue
679 if type(elem) == types.DictionaryType:
680 print_dict(elem, f=f, detail=detail)
681 elif type(elem) == types.StringType:
683 elem = elem.split('Traceback')[0]
684 f.write('%s\n' % elem)
686 f.write('%s\n' % elem)
688 def print_versions(data, f=stdout):
689 if 'versions' not in data:
690 f.write('%s\n' %data)
692 f.write('versions:\n')
693 for id, t in data['versions']:
694 f.write('%s @ %s\n' % (str(id).rjust(30), datetime.fromtimestamp(t)))
698 return os.environ['PITHOS_USER']
704 return os.environ['PITHOS_AUTH']
712 cls = class_for_cli_command(name)
713 except (IndexError, KeyError):
717 cmd = cls(name, argv[2:])
720 cmd.execute(*cmd.args)
722 cmd.parser.print_help()
725 status = f.status and '%s ' % f.status or ''
726 print '%s%s' % (status, f.data)
728 if __name__ == '__main__':