3 # Copyright 2011 GRNET S.A. All rights reserved.
5 # Redistribution and use in source and binary forms, with or
6 # without modification, are permitted provided that the following
9 # 1. Redistributions of source code must retain the above
10 # copyright notice, this list of conditions and the following
13 # 2. Redistributions in binary form must reproduce the above
14 # copyright notice, this list of conditions and the following
15 # disclaimer in the documentation and/or other materials
16 # provided with the distribution.
18 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 # POSSIBILITY OF SUCH DAMAGE.
31 # The views and conclusions contained in the software and
32 # documentation are those of the authors and should not be
33 # interpreted as representing official policies, either expressed
34 # or implied, of GRNET S.A.
36 from getpass import getuser
37 from optparse import OptionParser
38 from os import environ
39 from sys import argv, exit, stdin, stdout
40 from pithos.lib.client import Client, Fault
41 from datetime import datetime
50 DEFAULT_HOST = 'pithos.dev.grnet.gr'
55 def cli_command(*args):
59 _cli_commands[name] = cls
63 def class_for_cli_command(name):
64 return _cli_commands[name]
66 class Command(object):
69 def __init__(self, name, argv):
70 parser = OptionParser('%%prog %s [options] %s' % (name, self.syntax))
71 parser.add_option('--host', dest='host', metavar='HOST',
72 default=DEFAULT_HOST, help='use server HOST')
73 parser.add_option('--user', dest='user', metavar='USERNAME',
75 help='use account USERNAME')
76 parser.add_option('--token', dest='token', metavar='AUTH',
78 help='use account AUTH')
79 parser.add_option('--api', dest='api', metavar='API',
80 default=DEFAULT_API, help='use api API')
81 parser.add_option('-v', action='store_true', dest='verbose',
82 default=False, help='use verbose output')
83 parser.add_option('-d', action='store_true', dest='debug',
84 default=False, help='use debug output')
85 self.add_options(parser)
86 options, args = parser.parse_args(argv)
89 for opt in parser.option_list:
92 val = getattr(options, key)
93 setattr(self, key, val)
95 self.client = Client(self.host, self.token, self.user, self.api, self.verbose,
101 def add_options(self, parser):
104 def execute(self, *args):
107 @cli_command('list', 'ls')
109 syntax = '[<container>[/<object>]]'
110 description = 'list containers or objects'
112 def add_options(self, parser):
113 parser.add_option('-l', action='store_true', dest='detail',
114 default=False, help='show detailed output')
115 parser.add_option('-n', action='store', type='int', dest='limit',
116 default=1000, help='show limited output')
117 parser.add_option('--marker', action='store', type='str',
118 dest='marker', default=None,
119 help='show output greater then marker')
120 parser.add_option('--prefix', action='store', type='str',
121 dest='prefix', default=None,
122 help='show output starting with prefix')
123 parser.add_option('--delimiter', action='store', type='str',
124 dest='delimiter', default=None,
125 help='show output up to the delimiter')
126 parser.add_option('--path', action='store', type='str',
127 dest='path', default=None,
128 help='show output starting with prefix up to /')
129 parser.add_option('--meta', action='store', type='str',
130 dest='meta', default=None,
131 help='show output having the specified meta keys')
132 parser.add_option('--if-modified-since', action='store', type='str',
133 dest='if_modified_since', default=None,
134 help='show output if modified since then')
135 parser.add_option('--if-unmodified-since', action='store', type='str',
136 dest='if_unmodified_since', default=None,
137 help='show output if not modified since then')
138 parser.add_option('--until', action='store', dest='until',
139 default=False, help='show metadata until that date')
140 parser.add_option('--format', action='store', dest='format',
141 default='%d/%m/%Y', help='format to parse until date')
143 def execute(self, container=None):
145 self.list_objects(container)
147 self.list_containers()
149 def list_containers(self):
150 params = {'limit':self.limit, 'marker':self.marker}
151 headers = {'IF_MODIFIED_SINCE':self.if_modified_since,
152 'IF_UNMODIFIED_SINCE':self.if_unmodified_since}
155 t = _time.strptime(self.until, self.format)
156 params['until'] = int(_time.mktime(t))
158 l = self.client.list_containers(self.detail, params, headers)
161 def list_objects(self, container):
164 attrs = ['limit', 'marker', 'prefix', 'delimiter', 'path', 'meta']
165 for a in [a for a in attrs if getattr(self, a)]:
166 params[a] = getattr(self, a)
169 t = _time.strptime(self.until, self.format)
170 params['until'] = int(_time.mktime(t))
172 headers = {'IF_MODIFIED_SINCE':self.if_modified_since,
173 'IF_UNMODIFIED_SINCE':self.if_unmodified_since}
174 container, sep, object = container.partition('/')
179 #if request with meta quering disable trash filtering
180 show_trashed = True if self.meta else False
181 l = self.client.list_objects(container, detail, headers,
182 include_trashed = show_trashed, **params)
183 print_list(l, detail=self.detail)
187 syntax = '[<container>[/<object>]]'
188 description = 'get the metadata of an account, a container or an object'
190 def add_options(self, parser):
191 parser.add_option('-r', action='store_true', dest='restricted',
192 default=False, help='show only user defined metadata')
193 parser.add_option('--until', action='store', dest='until',
194 default=False, help='show metadata until that date')
195 parser.add_option('--format', action='store', dest='format',
196 default='%d/%m/%Y', help='format to parse until date')
197 parser.add_option('--version', action='store', dest='version',
198 default=None, help='show specific version \
199 (applies only for objects)')
201 def execute(self, path=''):
202 container, sep, object = path.partition('/')
204 t = _time.strptime(self.until, self.format)
205 self.until = int(_time.mktime(t))
207 meta = self.client.retrieve_object_metadata(container, object,
211 meta = self.client.retrieve_container_metadata(container,
215 meta = self.client.account_metadata(self.restricted, self.until)
217 print 'Entity does not exist'
219 print_dict(meta, header=None)
221 @cli_command('create')
222 class CreateContainer(Command):
223 syntax = '<container> [key=val] [...]'
224 description = 'create a container'
226 def execute(self, container, *args):
230 key, sep, val = arg.partition('=')
232 ret = self.client.create_container(container, headers, **meta)
234 print 'Container already exists'
236 @cli_command('delete', 'rm')
237 class Delete(Command):
238 syntax = '<container>[/<object>]'
239 description = 'delete a container or an object'
241 def execute(self, path):
242 container, sep, object = path.partition('/')
244 self.client.delete_object(container, object)
246 self.client.delete_container(container)
249 class GetObject(Command):
250 syntax = '<container>/<object>'
251 description = 'get the data of an object'
253 def add_options(self, parser):
254 parser.add_option('-l', action='store_true', dest='detail',
255 default=False, help='show detailed output')
256 parser.add_option('--range', action='store', dest='range',
257 default=None, help='show range of data')
258 parser.add_option('--if-range', action='store', dest='if-range',
259 default=None, help='show range of data')
260 parser.add_option('--if-match', action='store', dest='if-match',
261 default=None, help='show output if ETags match')
262 parser.add_option('--if-none-match', action='store',
263 dest='if-none-match', default=None,
264 help='show output if ETags don\'t match')
265 parser.add_option('--if-modified-since', action='store', type='str',
266 dest='if-modified-since', default=None,
267 help='show output if modified since then')
268 parser.add_option('--if-unmodified-since', action='store', type='str',
269 dest='if-unmodified-since', default=None,
270 help='show output if not modified since then')
271 parser.add_option('-o', action='store', type='str',
272 dest='file', default=None,
273 help='save output in file')
274 parser.add_option('--version', action='store', type='str',
275 dest='version', default=None,
276 help='get the specific \
278 parser.add_option('--versionlist', action='store_true',
279 dest='versionlist', default=False,
280 help='get the full object version list')
282 def execute(self, path):
285 headers['RANGE'] = 'bytes=%s' %self.range
286 if getattr(self, 'if-range'):
287 headers['IF_RANGE'] = 'If-Range:%s' % getattr(self, 'if-range')
288 attrs = ['if-match', 'if-none-match', 'if-modified-since',
289 'if-unmodified-since']
290 attrs = [a for a in attrs if getattr(self, a)]
292 headers[a.replace('-', '_').upper()] = getattr(self, a)
293 container, sep, object = path.partition('/')
295 self.version = 'list'
297 data = self.client.retrieve_object(container, object, self.detail,
298 headers, self.version)
299 f = self.file and open(self.file, 'w') or stdout
301 data = json.loads(data)
303 print_versions(data, f=f)
305 print_dict(data, f=f)
310 @cli_command('mkdir')
311 class PutMarker(Command):
312 syntax = '<container>/<directory marker>'
313 description = 'create a directory marker'
315 def execute(self, path):
316 container, sep, object = path.partition('/')
317 self.client.create_directory_marker(container, object)
320 class PutObject(Command):
321 syntax = '<container>/<object> <path> [key=val] [...]'
322 description = 'create/override object'
324 def add_options(self, parser):
325 parser.add_option('--use_hashes', action='store_true', dest='use_hashes',
326 default=False, help='provide hashmap instead of data')
327 parser.add_option('--chunked', action='store_true', dest='chunked',
328 default=False, help='set chunked transfer mode')
329 parser.add_option('--etag', action='store', dest='etag',
330 default=None, help='check written data')
331 parser.add_option('--content-encoding', action='store',
332 dest='content-encoding', default=None,
333 help='provide the object MIME content type')
334 parser.add_option('--content-disposition', action='store', type='str',
335 dest='content-disposition', default=None,
336 help='provide the presentation style of the object')
337 parser.add_option('-S', action='store',
338 dest='segment-size', default=False,
339 help='use for large file support')
340 parser.add_option('--manifest', action='store_true',
341 dest='manifest', default=None,
342 help='upload a manifestation file')
343 parser.add_option('--type', action='store',
344 dest='content-type', default=False,
345 help='create object with specific content type')
346 parser.add_option('--sharing', action='store',
347 dest='sharing', default=None,
348 help='define sharing object policy')
349 parser.add_option('-f', action='store',
350 dest='srcpath', default=None,
351 help='file descriptor to read from (pass - for standard input)')
352 parser.add_option('--public', action='store',
353 dest='public', default=None,
354 help='make object publicly accessible (\'True\'/\'False\')')
356 def execute(self, path, *args):
357 if path.find('=') != -1:
358 raise Fault('Missing path argument')
360 #prepare user defined meta
363 key, sep, val = arg.partition('=')
367 manifest = getattr(self, 'manifest')
369 # if it's manifestation file
370 # send zero-byte data with X-Object-Manifest header
372 headers['X_OBJECT_MANIFEST'] = manifest
374 headers['X_OBJECT_SHARING'] = self.sharing
376 attrs = ['etag', 'content-encoding', 'content-disposition',
378 attrs = [a for a in attrs if getattr(self, a)]
380 headers[a.replace('-', '_').upper()] = getattr(self, a)
382 container, sep, object = path.partition('/')
386 f = open(self.srcpath) if self.srcpath != '-' else stdin
388 if self.use_hashes and not f:
389 raise Fault('Illegal option combination')
390 if self.public not in ['True', 'False', None]:
391 raise Fault('Not acceptable value for public')
392 public = eval(self.public) if self.public else None
393 self.client.create_object(container, object, f, chunked=self.chunked,
394 headers=headers, use_hashes=self.use_hashes,
395 public=public, **meta)
399 @cli_command('copy', 'cp')
400 class CopyObject(Command):
401 syntax = '<src container>/<src object> [<dst container>/]<dst object>'
402 description = 'copy an object to a different location'
404 def add_options(self, parser):
405 parser.add_option('--version', action='store',
406 dest='version', default=False,
407 help='copy specific version')
408 parser.add_option('--public', action='store',
409 dest='public', default=None,
410 help='publish/unpublish object (\'True\'/\'False\')')
412 def execute(self, src, dst):
413 src_container, sep, src_object = src.partition('/')
414 dst_container, sep, dst_object = dst.partition('/')
416 dst_container = src_container
418 version = getattr(self, 'version')
421 headers['X_SOURCE_VERSION'] = version
422 if self.public and self.nopublic:
423 raise Fault('Conflicting options')
424 if self.public not in ['True', 'False', None]:
425 raise Fault('Not acceptable value for public')
426 public = eval(self.public) if self.public else None
427 self.client.copy_object(src_container, src_object, dst_container,
428 dst_object, public, headers)
431 class SetMeta(Command):
432 syntax = '[<container>[/<object>]] key=val [key=val] [...]'
433 description = 'set metadata'
435 def execute(self, path, *args):
436 #in case of account fix the args
437 if path.find('=') != -1:
444 key, sep, val = arg.partition('=')
445 meta[key.strip()] = val.strip()
446 container, sep, object = path.partition('/')
448 self.client.update_object_metadata(container, object, **meta)
450 self.client.update_container_metadata(container, **meta)
452 self.client.update_account_metadata(**meta)
454 @cli_command('update')
455 class UpdateObject(Command):
456 syntax = '<container>/<object> path [key=val] [...]'
457 description = 'update object metadata/data (default mode: append)'
459 def add_options(self, parser):
460 parser.add_option('-a', action='store_true', dest='append',
461 default=True, help='append data')
462 parser.add_option('--offset', action='store',
464 default=None, help='starting offest to be updated')
465 parser.add_option('--range', action='store', dest='content-range',
466 default=None, help='range of data to be updated')
467 parser.add_option('--chunked', action='store_true', dest='chunked',
468 default=False, help='set chunked transfer mode')
469 parser.add_option('--content-encoding', action='store',
470 dest='content-encoding', default=None,
471 help='provide the object MIME content type')
472 parser.add_option('--content-disposition', action='store', type='str',
473 dest='content-disposition', default=None,
474 help='provide the presentation style of the object')
475 parser.add_option('--manifest', action='store', type='str',
476 dest='manifest', default=None,
477 help='use for large file support')
478 parser.add_option('--sharing', action='store',
479 dest='sharing', default=None,
480 help='define sharing object policy')
481 parser.add_option('--nosharing', action='store_true',
482 dest='no_sharing', default=None,
483 help='clear object sharing policy')
484 parser.add_option('-f', action='store',
485 dest='srcpath', default=None,
486 help='file descriptor to read from: pass - for standard input')
487 parser.add_option('--public', action='store',
488 dest='public', default=None,
489 help='publish/unpublish object (\'True\'/\'False\')')
491 def execute(self, path, *args):
492 if path.find('=') != -1:
493 raise Fault('Missing path argument')
497 headers['X_OBJECT_MANIFEST'] = self.manifest
499 headers['X_OBJECT_SHARING'] = self.sharing
502 headers['X_OBJECT_SHARING'] = ''
504 attrs = ['content-encoding', 'content-disposition']
505 attrs = [a for a in attrs if getattr(self, a)]
507 headers[a.replace('-', '_').upper()] = getattr(self, a)
509 #prepare user defined meta
512 key, sep, val = arg.partition('=')
515 container, sep, object = path.partition('/')
520 f = self.srcpath != '-' and open(self.srcpath) or stdin
522 chunked = True if (self.chunked or f == stdin) else False
523 if self.public not in ['True', 'False', None]:
524 raise Fault('Not acceptable value for public')
525 public = eval(self.public) if self.public else None
526 self.client.update_object(container, object, f, chunked=chunked,
527 headers=headers, offset=self.offset,
528 public=public, **meta)
532 @cli_command('move', 'mv')
533 class MoveObject(Command):
534 syntax = '<src container>/<src object> [<dst container>/]<dst object>'
535 description = 'move an object to a different location'
537 def add_options(self, parser):
538 parser.add_option('--public', action='store',
539 dest='public', default=None,
540 help='publish/unpublish object (\'True\'/\'False\')')
542 def execute(self, src, dst):
543 src_container, sep, src_object = src.partition('/')
544 dst_container, sep, dst_object = dst.partition('/')
546 dst_container = src_container
548 if self.public not in ['True', 'False', None]:
549 raise Fault('Not acceptable value for public')
550 public = eval(self.public) if self.public else None
551 self.client.move_object(src_container, src_object, dst_container,
552 dst_object, public, headers)
554 @cli_command('remove')
555 class TrashObject(Command):
556 syntax = '<container>/<object>'
557 description = 'trash an object'
559 def execute(self, src):
560 src_container, sep, src_object = src.partition('/')
562 self.client.trash_object(src_container, src_object)
564 @cli_command('restore')
565 class RestoreObject(Command):
566 syntax = '<container>/<object>'
567 description = 'restore a trashed object'
569 def execute(self, src):
570 src_container, sep, src_object = src.partition('/')
572 self.client.restore_object(src_container, src_object)
574 @cli_command('unset')
575 class UnsetObject(Command):
576 syntax = '<container>/[<object>] key [key] [...]'
577 description = 'delete metadata info'
579 def execute(self, path, *args):
580 #in case of account fix the args
589 container, sep, object = path.partition('/')
591 self.client.delete_object_metadata(container, object, meta)
593 self.client.delete_container_metadata(container, meta)
595 self.client.delete_account_metadata(meta)
597 @cli_command('group')
598 class SetGroup(Command):
599 syntax = 'key=val [key=val] [...]'
600 description = 'set group account info'
602 def execute(self, *args):
605 key, sep, val = arg.partition('=')
607 self.client.set_account_groups(**groups)
609 @cli_command('policy')
610 class SetPolicy(Command):
611 syntax = 'container key=val [key=val] [...]'
612 description = 'set container policies'
614 def execute(self, path, *args):
615 if path.find('=') != -1:
616 raise Fault('Missing container argument')
618 container, sep, object = path.partition('/')
621 raise Fault('Only containers have policies')
625 key, sep, val = arg.partition('=')
628 self.client.set_container_policies(container, **policies)
630 @cli_command('publish')
631 class PublishObject(Command):
632 syntax = '<container>/<object>'
633 description = 'publish an object'
635 def execute(self, src):
636 src_container, sep, src_object = src.partition('/')
638 self.client.publish_object(src_container, src_object)
640 @cli_command('unpublish')
641 class UnpublishObject(Command):
642 syntax = '<container>/<object>'
643 description = 'unpublish an object'
645 def execute(self, src):
646 src_container, sep, src_object = src.partition('/')
648 self.client.unpublish_object(src_container, src_object)
651 cmd = Command('', [])
653 parser.usage = '%prog <command> [options]'
657 for cls in set(_cli_commands.values()):
658 name = ', '.join(cls.commands)
659 description = getattr(cls, 'description', '')
660 commands.append(' %s %s' % (name.ljust(12), description))
661 print '\nCommands:\n' + '\n'.join(sorted(commands))
663 def print_dict(d, header='name', f=stdout, detail=True):
664 header = header in d and header or 'subdir'
665 if header and header in d:
666 f.write('%s\n' %d.pop(header))
668 patterns = ['^x_(account|container|object)_meta_(\w+)$']
669 patterns.append(patterns[0].replace('_', '-'))
670 for key, val in sorted(d.items()):
676 f.write('%s: %s\n' % (key.rjust(30), val))
678 def print_list(l, verbose=False, f=stdout, detail=True):
680 #if it's empty string continue
683 if type(elem) == types.DictionaryType:
684 print_dict(elem, f=f, detail=detail)
685 elif type(elem) == types.StringType:
687 elem = elem.split('Traceback')[0]
688 f.write('%s\n' % elem)
690 f.write('%s\n' % elem)
692 def print_versions(data, f=stdout):
693 if 'versions' not in data:
694 f.write('%s\n' %data)
696 f.write('versions:\n')
697 for id, t in data['versions']:
698 f.write('%s @ %s\n' % (str(id).rjust(30), datetime.fromtimestamp(t)))
702 return os.environ['PITHOS_USER']
708 return os.environ['PITHOS_AUTH']
716 cls = class_for_cli_command(name)
717 except (IndexError, KeyError):
721 cmd = cls(name, argv[2:])
724 cmd.execute(*cmd.args)
726 cmd.parser.print_help()
729 status = f.status and '%s ' % f.status or ''
730 print '%s%s' % (status, f.data)
732 if __name__ == '__main__':