code.grnet.gr Git - kamaki/blob - kamaki/cli/commands/pithos.py

   1 # Copyright 2011-2013 GRNET S.A. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or
   4 # without modification, are permitted provided that the following
   5 # conditions are met:
   6 #
   7 #   1. Redistributions of source code must retain the above
   8 #      copyright notice, this list of conditions and the following
   9 #      disclaimer.
  10 #
  11 #   2. Redistributions in binary form must reproduce the above
  12 #      copyright notice, this list of conditions and the following
  13 #      disclaimer in the documentation and/or other materials
  14 #      provided with the distribution.
  15 #
  16 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  17 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  23 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  24 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  26 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27 # POSSIBILITY OF SUCH DAMAGE.
  28 #
  29 # The views and conclusions contained in the software and
  30 # documentation are those of the authors and should not be
  31 # interpreted as representing official policies, either expressed
  32 # or implied, of GRNET S.A.command
  33
  34 from time import localtime, strftime
  35 from io import StringIO
  36 from pydoc import pager
  37 from os import path, walk, makedirs
  38
  39 from kamaki.clients.pithos import PithosClient, ClientError
  40
  41 from kamaki.cli import command
  42 from kamaki.cli.command_tree import CommandTree
  43 from kamaki.cli.commands import (
  44     _command_init, errors, addLogSettings, DontRaiseKeyError, _optional_json,
  45     _name_filter, _optional_output_cmd)
  46 from kamaki.cli.errors import (
  47     CLIBaseUrlError, CLIError, CLIInvalidArgument, raiseCLIError,
  48     CLISyntaxError)
  49 from kamaki.cli.argument import (
  50     FlagArgument, IntArgument, ValueArgument, DateArgument, KeyValueArgument,
  51     ProgressBarArgument, RepeatableArgument, DataSizeArgument)
  52 from kamaki.cli.utils import (
  53     format_size, bold, get_path_size, guess_mime_type)
  54
  55 file_cmds = CommandTree('file', 'Pithos+/Storage object level API commands')
  56 container_cmds = CommandTree(
  57     'container', 'Pithos+/Storage container level API commands')
  58 sharer_cmds = CommandTree('sharer', 'Pithos+/Storage sharers')
  59 group_cmds = CommandTree('group', 'Pithos+/Storage user groups')
  60 _commands = [file_cmds, container_cmds, sharer_cmds, group_cmds]
  61
  62
  63 class _pithos_init(_command_init):
  64     """Initilize a pithos+ client
  65     There is always a default account (current user uuid)
  66     There is always a default container (pithos)
  67     """
  68
  69     @DontRaiseKeyError
  70     def _custom_container(self):
  71         return self.config.get_cloud(self.cloud, 'pithos_container')
  72
  73     @DontRaiseKeyError
  74     def _custom_uuid(self):
  75         return self.config.get_cloud(self.cloud, 'pithos_uuid')
  76
  77     def _set_account(self):
  78         self.account = self._custom_uuid()
  79         if self.account:
  80             return
  81         astakos = getattr(self, 'auth_base', None)
  82         if astakos:
  83             self.account = astakos.user_term('id', self.token)
  84         else:
  85             raise CLIBaseUrlError(service='astakos')
  86
  87     @errors.generic.all
  88     @addLogSettings
  89     def _run(self):
  90         cloud = getattr(self, 'cloud', None)
  91         if cloud:
  92             self.base_url = self._custom_url('pithos')
  93         else:
  94             self.cloud = 'default'
  95         self.token = self._custom_token('pithos')
  96         self.container = self._custom_container() or 'pithos'
  97
  98         astakos = getattr(self, 'auth_base', None)
  99         if astakos:
 100             self.token = self.token or astakos.token
 101             if not self.base_url:
 102                 pithos_endpoints = astakos.get_service_endpoints(
 103                     self._custom_type('pithos') or 'object-store',
 104                     self._custom_version('pithos') or '')
 105                 self.base_url = pithos_endpoints['publicURL']
 106         else:
 107             raise CLIBaseUrlError(service='astakos')
 108
 109         self._set_account()
 110         self.client = PithosClient(
 111             self.base_url, self.token, self.account, self.container)
 112
 113     def main(self):
 114         self._run()
 115
 116
 117 class _pithos_account(_pithos_init):
 118     """Setup account"""
 119
 120     def __init__(self, arguments={}, auth_base=None, cloud=None):
 121         super(_pithos_account, self).__init__(arguments, auth_base, cloud)
 122         self['account'] = ValueArgument(
 123             'Use (a different) user uuid', ('-A', '--account'))
 124
 125     def print_objects(self, object_list):
 126         for index, obj in enumerate(object_list):
 127             pretty_obj = obj.copy()
 128             index += 1
 129             empty_space = ' ' * (len(str(len(object_list))) - len(str(index)))
 130             if 'subdir' in obj:
 131                 continue
 132             if self._is_dir(obj):
 133                 size = 'D'
 134             else:
 135                 size = format_size(obj['bytes'])
 136                 pretty_obj['bytes'] = '%s (%s)' % (obj['bytes'], size)
 137             oname = obj['name'] if self['more'] else bold(obj['name'])
 138             prfx = ('%s%s. ' % (empty_space, index)) if self['enum'] else ''
 139             if self['detail']:
 140                 self.writeln('%s%s' % (prfx, oname))
 141                 self.print_dict(pretty_obj, exclude=('name'))
 142                 self.writeln()
 143             else:
 144                 oname = '%s%9s %s' % (prfx, size, oname)
 145                 oname += '/' if self._is_dir(obj) else u''
 146                 self.writeln(oname)
 147
 148     @staticmethod
 149     def _is_dir(remote_dict):
 150         return 'application/directory' == remote_dict.get(
 151             'content_type', remote_dict.get('content-type', ''))
 152
 153     def _run(self):
 154         super(_pithos_account, self)._run()
 155         self.client.account = self['account'] or getattr(
 156             self, 'account', getattr(self.client, 'account', None))
 157
 158
 159 class _pithos_container(_pithos_account):
 160     """Setup container"""
 161
 162     def __init__(self, arguments={}, auth_base=None, cloud=None):
 163         super(_pithos_container, self).__init__(arguments, auth_base, cloud)
 164         self['container'] = ValueArgument(
 165             'Use this container (default: pithos)', ('-C', '--container'))
 166
 167     @staticmethod
 168     def _resolve_pithos_url(url):
 169         """Match urls of one of the following formats:
 170         pithos://ACCOUNT/CONTAINER/OBJECT_PATH
 171         /CONTAINER/OBJECT_PATH
 172         return account, container, path
 173         """
 174         account, container, obj_path, prefix = '', '', url, 'pithos://'
 175         if url.startswith(prefix):
 176             account, sep, url = url[len(prefix):].partition('/')
 177             url = '/%s' % url
 178         if url.startswith('/'):
 179             container, sep, obj_path = url[1:].partition('/')
 180         return account, container, obj_path
 181
 182     def _run(self, url=None):
 183         acc, con, self.path = self._resolve_pithos_url(url or '')
 184         #  self.account = acc or getattr(self, 'account', '')
 185         super(_pithos_container, self)._run()
 186         self.container = con or self['container'] or getattr(
 187             self, 'container', None) or getattr(self.client, 'container', '')
 188         self.client.account = acc or self.client.account
 189         self.client.container = self.container
 190
 191
 192 @command(file_cmds)
 193 class file_info(_pithos_container, _optional_json):
 194     """Get information/details about a file"""
 195
 196     arguments = dict(
 197         object_version=ValueArgument(
 198             'download a file of a specific version', '--object-version'),
 199         hashmap=FlagArgument(
 200             'Get file hashmap instead of details', '--hashmap'),
 201         matching_etag=ValueArgument(
 202             'show output if ETags match', '--if-match'),
 203         non_matching_etag=ValueArgument(
 204             'show output if ETags DO NOT match', '--if-none-match'),
 205         modified_since_date=DateArgument(
 206             'show output modified since then', '--if-modified-since'),
 207         unmodified_since_date=DateArgument(
 208             'show output unmodified since then', '--if-unmodified-since'),
 209         sharing=FlagArgument(
 210             'show object permissions and sharing information', '--sharing'),
 211         metadata=FlagArgument('show only object metadata', '--metadata'),
 212         versions=FlagArgument(
 213             'show the list of versions for the file', '--object-versions')
 214     )
 215
 216     def version_print(self, versions):
 217         return {'/%s/%s' % (self.container, self.path): [
 218             dict(version_id=vitem[0], created=strftime(
 219                 '%d-%m-%Y %H:%M:%S',
 220                 localtime(float(vitem[1])))) for vitem in versions]}
 221
 222     @errors.generic.all
 223     @errors.pithos.connection
 224     @errors.pithos.container
 225     @errors.pithos.object_path
 226     def _run(self):
 227         if self['hashmap']:
 228             r = self.client.get_object_hashmap(
 229                 self.path,
 230                 version=self['object_version'],
 231                 if_match=self['matching_etag'],
 232                 if_none_match=self['non_matching_etag'],
 233                 if_modified_since=self['modified_since_date'],
 234                 if_unmodified_since=self['unmodified_since_date'])
 235         elif self['sharing']:
 236             r = self.client.get_object_sharing(self.path)
 237             r['public url'] = self.client.get_object_info(
 238                 self.path, version=self['object_version']).get(
 239                     'x-object-public', None)
 240         elif self['metadata']:
 241             r, preflen = dict(), len('x-object-meta-')
 242             for k, v in self.client.get_object_meta(self.path).items():
 243                 r[k[preflen:]] = v
 244         elif self['versions']:
 245             r = self.version_print(
 246                 self.client.get_object_versionlist(self.path))
 247         else:
 248             r = self.client.get_object_info(
 249                 self.path, version=self['object_version'])
 250         self._print(r, self.print_dict)
 251
 252     def main(self, path_or_url):
 253         super(self.__class__, self)._run(path_or_url)
 254         self._run()
 255
 256
 257 @command(file_cmds)
 258 class file_list(_pithos_container, _optional_json, _name_filter):
 259     """List all objects in a container or a directory object"""
 260
 261     arguments = dict(
 262         detail=FlagArgument('detailed output', ('-l', '--list')),
 263         limit=IntArgument('limit number of listed items', ('-n', '--number')),
 264         marker=ValueArgument('output greater that marker', '--marker'),
 265         delimiter=ValueArgument('show output up to delimiter', '--delimiter'),
 266         meta=ValueArgument(
 267             'show output with specified meta keys', '--meta',
 268             default=[]),
 269         if_modified_since=ValueArgument(
 270             'show output modified since then', '--if-modified-since'),
 271         if_unmodified_since=ValueArgument(
 272             'show output not modified since then', '--if-unmodified-since'),
 273         until=DateArgument('show metadata until then', '--until'),
 274         format=ValueArgument(
 275             'format to parse until data (default: d/m/Y H:M:S )', '--format'),
 276         shared_by_me=FlagArgument(
 277             'show only files shared to other users', '--shared-by-me'),
 278         public=FlagArgument('show only published objects', '--public'),
 279         more=FlagArgument('read long results', '--more'),
 280         enum=FlagArgument('Enumerate results', '--enumerate'),
 281         recursive=FlagArgument(
 282             'Recursively list containers and their contents',
 283             ('-R', '--recursive'))
 284     )
 285
 286     @errors.generic.all
 287     @errors.pithos.connection
 288     @errors.pithos.container
 289     @errors.pithos.object_path
 290     def _run(self):
 291         r = self.client.container_get(
 292             limit=False if self['more'] else self['limit'],
 293             marker=self['marker'],
 294             prefix=self['name_pref'],
 295             delimiter=self['delimiter'],
 296             path=self.path or '',
 297             show_only_shared=self['shared_by_me'],
 298             public=self['public'],
 299             if_modified_since=self['if_modified_since'],
 300             if_unmodified_since=self['if_unmodified_since'],
 301             until=self['until'],
 302             meta=self['meta'])
 303
 304         #  REMOVE THIS if version >> 0.12
 305         if not r.json:
 306             self.error('  NOTE: Since v0.12, use / for containers e.g.,')
 307             self.error('    [kamaki] file list /pithos')
 308
 309         files = self._filter_by_name(r.json)
 310         if self['more']:
 311             outbu, self._out = self._out, StringIO()
 312         try:
 313             if self['json_output'] or self['output_format']:
 314                 self._print(files)
 315             else:
 316                 self.print_objects(files)
 317         finally:
 318             if self['more']:
 319                 pager(self._out.getvalue())
 320                 self._out = outbu
 321
 322     def main(self, path_or_url=''):
 323         super(self.__class__, self)._run(path_or_url)
 324         self._run()
 325
 326
 327 @command(file_cmds)
 328 class file_modify(_pithos_container):
 329     """Modify the attributes of a file or directory object"""
 330
 331     arguments = dict(
 332         publish=FlagArgument(
 333             'Make an object public (returns the public URL)', '--publish'),
 334         unpublish=FlagArgument(
 335             'Make an object unpublic', '--unpublish'),
 336         uuid_for_read_permission=RepeatableArgument(
 337             'Give read access to user/group (can be repeated, accumulative). '
 338             'Format for users: UUID . Format for groups: UUID:GROUP . '
 339             'Use * for all users/groups', '--read-permission'),
 340         uuid_for_write_permission=RepeatableArgument(
 341             'Give write access to user/group (can be repeated, accumulative). '
 342             'Format for users: UUID . Format for groups: UUID:GROUP . '
 343             'Use * for all users/groups', '--write-permission'),
 344         no_permissions=FlagArgument('Remove permissions', '--no-permissions'),
 345         metadata_to_set=KeyValueArgument(
 346             'Add metadata (KEY=VALUE) to an object (can be repeated)',
 347             '--metadata-add'),
 348         metadata_key_to_delete=RepeatableArgument(
 349             'Delete object metadata (can be repeated)', '--metadata-del'),
 350     )
 351     required = [
 352         'publish', 'unpublish', 'uuid_for_read_permission', 'metadata_to_set',
 353         'uuid_for_write_permission', 'no_permissions',
 354         'metadata_key_to_delete']
 355
 356     @errors.generic.all
 357     @errors.pithos.connection
 358     @errors.pithos.container
 359     @errors.pithos.object_path
 360     def _run(self):
 361         if self['publish']:
 362             self.writeln(self.client.publish_object(self.path))
 363         if self['unpublish']:
 364             self.client.unpublish_object(self.path)
 365         if self['uuid_for_read_permission'] or self[
 366                 'uuid_for_write_permission']:
 367             perms = self.client.get_object_sharing(self.path)
 368             read, write = perms.get('read', ''), perms.get('write', '')
 369             read = read.split(',') if read else []
 370             write = write.split(',') if write else []
 371             read += (self['uuid_for_read_permission'] or [])
 372             write += (self['uuid_for_write_permission'] or [])
 373             self.client.set_object_sharing(
 374                 self.path, read_permission=read, write_permission=write)
 375             self.print_dict(self.client.get_object_sharing(self.path))
 376         if self['no_permissions']:
 377             self.client.del_object_sharing(self.path)
 378         metadata = self['metadata_to_set'] or dict()
 379         for k in (self['metadata_key_to_delete'] or []):
 380             metadata[k] = ''
 381         if metadata:
 382             self.client.set_object_meta(self.path, metadata)
 383             self.print_dict(self.client.get_object_meta(self.path))
 384
 385     def main(self, path_or_url):
 386         super(self.__class__, self)._run(path_or_url)
 387         if self['publish'] and self['unpublish']:
 388             raise CLIInvalidArgument(
 389                 'Arguments %s and %s cannot be used together' % (
 390                     self.arguments['publish'].lvalue,
 391                     self.arguments['publish'].lvalue))
 392         if self['no_permissions'] and (
 393                 self['uuid_for_read_permission'] or self[
 394                     'uuid_for_write_permission']):
 395             raise CLIInvalidArgument(
 396                 '%s cannot be used with other permission arguments' % (
 397                     self.arguments['no_permissions'].lvalue))
 398         self._run()
 399
 400
 401 @command(file_cmds)
 402 class file_create(_pithos_container, _optional_output_cmd):
 403     """Create an empty file"""
 404
 405     arguments = dict(
 406         content_type=ValueArgument(
 407             'Set content type (default: application/octet-stream)',
 408             '--content-type',
 409             default='application/octet-stream')
 410     )
 411
 412     @errors.generic.all
 413     @errors.pithos.connection
 414     @errors.pithos.container
 415     def _run(self):
 416         self._optional_output(
 417             self.client.create_object(self.path, self['content_type']))
 418
 419     def main(self, path_or_url):
 420         super(self.__class__, self)._run(path_or_url)
 421         self._run()
 422
 423
 424 @command(file_cmds)
 425 class file_mkdir(_pithos_container, _optional_output_cmd):
 426     """Create a directory: /file create --content-type='applcation/directory'
 427     """
 428
 429     @errors.generic.all
 430     @errors.pithos.connection
 431     @errors.pithos.container
 432     def _run(self):
 433         self._optional_output(self.client.create_directory(self.path))
 434
 435     def main(self, path_or_url):
 436         super(self.__class__, self)._run(path_or_url)
 437         self._run()
 438
 439
 440 @command(file_cmds)
 441 class file_delete(_pithos_container):
 442     """Delete a file or directory object"""
 443
 444     arguments = dict(
 445         until_date=DateArgument('remove history until then', '--until'),
 446         yes=FlagArgument('Do not prompt for permission', '--yes'),
 447         recursive=FlagArgument(
 448             'If a directory, empty first', ('-r', '--recursive')),
 449         delimiter=ValueArgument(
 450             'delete objects prefixed with <object><delimiter>', '--delimiter')
 451     )
 452
 453     @errors.generic.all
 454     @errors.pithos.connection
 455     @errors.pithos.container
 456     @errors.pithos.object_path
 457     def _run(self):
 458         if self.path:
 459             if self['yes'] or self.ask_user(
 460                     'Delete /%s/%s ?' % (self.container, self.path)):
 461                 self.client.del_object(
 462                     self.path,
 463                     until=self['until_date'],
 464                     delimiter='/' if self['recursive'] else self['delimiter'])
 465             else:
 466                 self.error('Aborted')
 467         else:
 468             if self['yes'] or self.ask_user(
 469                     'Empty container /%s ?' % self.container):
 470                 self.client.container_delete(self.container, delimiter='/')
 471             else:
 472                 self.error('Aborted')
 473
 474     def main(self, path_or_url):
 475         super(self.__class__, self)._run(path_or_url)
 476         self._run()
 477
 478
 479 class _source_destination(_pithos_container, _optional_output_cmd):
 480
 481     sd_arguments = dict(
 482         destination_user_uuid=ValueArgument(
 483             'default: current user uuid', '--to-account'),
 484         destination_container=ValueArgument(
 485             'default: pithos', '--to-container'),
 486         source_prefix=FlagArgument(
 487             'Transfer all files that are prefixed with SOURCE PATH If the '
 488             'destination path is specified, replace SOURCE_PATH with '
 489             'DESTINATION_PATH',
 490             ('-r', '--recursive')),
 491         force=FlagArgument(
 492             'Overwrite destination objects, if needed', ('-f', '--force')),
 493         source_version=ValueArgument(
 494             'The version of the source object', '--source-version')
 495     )
 496
 497     def __init__(self, arguments={}, auth_base=None, cloud=None):
 498         self.arguments.update(arguments)
 499         self.arguments.update(self.sd_arguments)
 500         super(_source_destination, self).__init__(
 501             self.arguments, auth_base, cloud)
 502
 503     def _report_transfer(self, src, dst, transfer_name):
 504         if not dst:
 505             if transfer_name in ('move', ):
 506                 self.error('  delete source directory %s' % src)
 507             return
 508         dst_prf = '' if self.account == self.dst_client.account else (
 509                 'pithos://%s' % self.dst_client.account)
 510         if src:
 511             src_prf = '' if self.account == self.dst_client.account else (
 512                     'pithos://%s' % self.account)
 513             self.error('  %s %s/%s/%s\n  -->  %s/%s/%s' % (
 514                 transfer_name,
 515                 src_prf, self.container, src,
 516                 dst_prf, self.dst_client.container, dst))
 517         else:
 518             self.error('  mkdir %s/%s/%s' % (
 519                 dst_prf, self.dst_client.container, dst))
 520
 521     @errors.generic.all
 522     @errors.pithos.account
 523     def _src_dst(self, version=None):
 524         """Preconditions:
 525         self.account, self.container, self.path
 526         self.dst_acc, self.dst_con, self.dst_path
 527         They should all be configured properly
 528         :returns: [(src_path, dst_path), ...], if src_path is None, create
 529             destination directory
 530         """
 531         src_objects, dst_objects, pairs = dict(), dict(), []
 532         try:
 533             for obj in self.dst_client.list_objects(
 534                     prefix=self.dst_path or self.path or '/'):
 535                 dst_objects[obj['name']] = obj
 536         except ClientError as ce:
 537             if ce.status in (404, ):
 538                 raise CLIError(
 539                     'Destination container pithos://%s/%s not found' % (
 540                         self.dst_client.account, self.dst_client.container))
 541             raise ce
 542         if self['source_prefix']:
 543             #  Copy and replace prefixes
 544             for src_obj in self.client.list_objects(prefix=self.path):
 545                 src_objects[src_obj['name']] = src_obj
 546             for src_path, src_obj in src_objects.items():
 547                 dst_path = '%s%s' % (
 548                     self.dst_path or self.path, src_path[len(self.path):])
 549                 dst_obj = dst_objects.get(dst_path, None)
 550                 if self['force'] or not dst_obj:
 551                     #  Just do it
 552                     pairs.append((
 553                         None if self._is_dir(src_obj) else src_path, dst_path))
 554                     if self._is_dir(src_obj):
 555                         pairs.append((self.path or dst_path, None))
 556                 elif not (self._is_dir(dst_obj) and self._is_dir(src_obj)):
 557                     raise CLIError(
 558                         'Destination object exists', importance=2, details=[
 559                             'Failed while transfering:',
 560                             '    pithos://%s/%s/%s' % (
 561                                     self.account,
 562                                     self.container,
 563                                     src_path),
 564                             '--> pithos://%s/%s/%s' % (
 565                                     self.dst_client.account,
 566                                     self.dst_client.container,
 567                                     dst_path),
 568                             'Use %s to transfer overwrite' % (
 569                                     self.arguments['force'].lvalue)])
 570         else:
 571             #  One object transfer
 572             try:
 573                 src_version_arg = self.arguments.get('source_version', None)
 574                 src_obj = self.client.get_object_info(
 575                     self.path,
 576                     version=src_version_arg.value if src_version_arg else None)
 577             except ClientError as ce:
 578                 if ce.status in (204, ):
 579                     raise CLIError(
 580                         'Missing specific path container %s' % self.container,
 581                         importance=2, details=[
 582                             'To transfer container contents %s' % (
 583                                 self.arguments['source_prefix'].lvalue)])
 584                 raise
 585             dst_path = self.dst_path or self.path
 586             dst_obj = dst_objects.get(dst_path or self.path, None)
 587             if self['force'] or not dst_obj:
 588                 pairs.append(
 589                     (None if self._is_dir(src_obj) else self.path, dst_path))
 590                 if self._is_dir(src_obj):
 591                     pairs.append((self.path or dst_path, None))
 592             elif self._is_dir(src_obj):
 593                 raise CLIError(
 594                     'Cannot transfer an application/directory object',
 595                     importance=2, details=[
 596                         'The object pithos://%s/%s/%s is a directory' % (
 597                             self.account,
 598                             self.container,
 599                             self.path),
 600                         'To recursively copy a directory, use',
 601                         '  %s' % self.arguments['source_prefix'].lvalue,
 602                         'To create a file, use',
 603                         '  /file create  (general purpose)',
 604                         '  /file mkdir   (a directory object)'])
 605             else:
 606                 raise CLIError(
 607                     'Destination object exists',
 608                     importance=2, details=[
 609                         'Failed while transfering:',
 610                         '    pithos://%s/%s/%s' % (
 611                                 self.account,
 612                                 self.container,
 613                                 self.path),
 614                         '--> pithos://%s/%s/%s' % (
 615                                 self.dst_client.account,
 616                                 self.dst_client.container,
 617                                 dst_path),
 618                         'Use %s to transfer overwrite' % (
 619                                 self.arguments['force'].lvalue)])
 620         return pairs
 621
 622     def _run(self, source_path_or_url, destination_path_or_url=''):
 623         super(_source_destination, self)._run(source_path_or_url)
 624         dst_acc, dst_con, dst_path = self._resolve_pithos_url(
 625             destination_path_or_url)
 626         self.dst_client = PithosClient(
 627             base_url=self.client.base_url, token=self.client.token,
 628             container=self[
 629                 'destination_container'] or dst_con or self.client.container,
 630             account=self[
 631                 'destination_user_uuid'] or dst_acc or self.account)
 632         self.dst_path = dst_path or self.path
 633
 634
 635 @command(file_cmds)
 636 class file_copy(_source_destination):
 637     """Copy objects, even between different accounts or containers"""
 638
 639     arguments = dict(
 640         public=ValueArgument('publish new object', '--public'),
 641         content_type=ValueArgument(
 642             'change object\'s content type', '--content-type'),
 643         source_version=ValueArgument(
 644             'The version of the source object', '--object-version')
 645     )
 646
 647     @errors.generic.all
 648     @errors.pithos.connection
 649     @errors.pithos.container
 650     @errors.pithos.account
 651     def _run(self):
 652         for src, dst in self._src_dst(self['source_version']):
 653             self._report_transfer(src, dst, 'copy')
 654             if src and dst:
 655                 self.dst_client.copy_object(
 656                     src_container=self.client.container,
 657                     src_object=src,
 658                     dst_container=self.dst_client.container,
 659                     dst_object=dst,
 660                     source_account=self.client.account,
 661                     source_version=self['source_version'],
 662                     public=self['public'],
 663                     content_type=self['content_type'])
 664             elif dst:
 665                 self.dst_client.create_directory(dst)
 666
 667     def main(self, source_path_or_url, destination_path_or_url=None):
 668         super(file_copy, self)._run(
 669             source_path_or_url, destination_path_or_url or '')
 670         self._run()
 671
 672
 673 @command(file_cmds)
 674 class file_move(_source_destination):
 675     """Move objects, even between different accounts or containers"""
 676
 677     arguments = dict(
 678         public=ValueArgument('publish new object', '--public'),
 679         content_type=ValueArgument(
 680             'change object\'s content type', '--content-type')
 681     )
 682
 683     @errors.generic.all
 684     @errors.pithos.connection
 685     @errors.pithos.container
 686     @errors.pithos.account
 687     def _run(self):
 688         for src, dst in self._src_dst():
 689             self._report_transfer(src, dst, 'move')
 690             if src and dst:
 691                 self.dst_client.move_object(
 692                     src_container=self.client.container,
 693                     src_object=src,
 694                     dst_container=self.dst_client.container,
 695                     dst_object=dst,
 696                     source_account=self.account,
 697                     public=self['public'],
 698                     content_type=self['content_type'])
 699             elif dst:
 700                 self.dst_client.create_directory(dst)
 701             else:
 702                 self.client.del_object(src)
 703
 704     def main(self, source_path_or_url, destination_path_or_url=None):
 705         super(file_move, self)._run(
 706             source_path_or_url, destination_path_or_url or '')
 707         self._run()
 708
 709
 710 @command(file_cmds)
 711 class file_append(_pithos_container, _optional_output_cmd):
 712     """Append local file to (existing) remote object
 713     The remote object should exist.
 714     If the remote object is a directory, it is transformed into a file.
 715     In the later case, objects under the directory remain intact.
 716     """
 717
 718     arguments = dict(
 719         progress_bar=ProgressBarArgument(
 720             'do not show progress bar', ('-N', '--no-progress-bar'),
 721             default=False),
 722         max_threads=IntArgument('default: 1', '--threads'),
 723     )
 724
 725     @errors.generic.all
 726     @errors.pithos.connection
 727     @errors.pithos.container
 728     @errors.pithos.object_path
 729     def _run(self, local_path):
 730         if self['max_threads'] > 0:
 731             self.client.MAX_THREADS = int(self['max_threads'])
 732         (progress_bar, upload_cb) = self._safe_progress_bar('Appending')
 733         try:
 734             with open(local_path, 'rb') as f:
 735                 self._optional_output(
 736                     self.client.append_object(self.path, f, upload_cb))
 737         finally:
 738             self._safe_progress_bar_finish(progress_bar)
 739
 740     def main(self, local_path, remote_path_or_url):
 741         super(self.__class__, self)._run(remote_path_or_url)
 742         self._run(local_path)
 743
 744
 745 @command(file_cmds)
 746 class file_truncate(_pithos_container, _optional_output_cmd):
 747     """Truncate remote file up to size"""
 748
 749     arguments = dict(
 750         size_in_bytes=IntArgument('Length of file after truncation', '--size')
 751     )
 752     required = ('size_in_bytes', )
 753
 754     @errors.generic.all
 755     @errors.pithos.connection
 756     @errors.pithos.container
 757     @errors.pithos.object_path
 758     @errors.pithos.object_size
 759     def _run(self, size):
 760         self._optional_output(self.client.truncate_object(self.path, size))
 761
 762     def main(self, path_or_url):
 763         super(self.__class__, self)._run(path_or_url)
 764         self._run(size=self['size_in_bytes'])
 765
 766
 767 @command(file_cmds)
 768 class file_overwrite(_pithos_container, _optional_output_cmd):
 769     """Overwrite part of a remote file"""
 770
 771     arguments = dict(
 772         progress_bar=ProgressBarArgument(
 773             'do not show progress bar', ('-N', '--no-progress-bar'),
 774             default=False),
 775         start_position=IntArgument('File position in bytes', '--from'),
 776         end_position=IntArgument('File position in bytes', '--to')
 777     )
 778     required = ('start_position', 'end_position')
 779
 780     @errors.generic.all
 781     @errors.pithos.connection
 782     @errors.pithos.container
 783     @errors.pithos.object_path
 784     @errors.pithos.object_size
 785     def _run(self, local_path, start, end):
 786         start, end = int(start), int(end)
 787         (progress_bar, upload_cb) = self._safe_progress_bar(
 788             'Overwrite %s bytes' % (end - start))
 789         try:
 790             with open(path.abspath(local_path), 'rb') as f:
 791                 self._optional_output(self.client.overwrite_object(
 792                     obj=self.path,
 793                     start=start,
 794                     end=end,
 795                     source_file=f,
 796                     upload_cb=upload_cb))
 797         finally:
 798             self._safe_progress_bar_finish(progress_bar)
 799
 800     def main(self, local_path, path_or_url):
 801         super(self.__class__, self)._run(path_or_url)
 802         self.path = self.path or path.basename(local_path)
 803         self._run(
 804             local_path=local_path,
 805             start=self['start_position'],
 806             end=self['end_position'])
 807
 808
 809 @command(file_cmds)
 810 class file_upload(_pithos_container, _optional_output_cmd):
 811     """Upload a file"""
 812
 813     arguments = dict(
 814         max_threads=IntArgument('default: 5', '--threads'),
 815         content_encoding=ValueArgument(
 816             'set MIME content type', '--content-encoding'),
 817         content_disposition=ValueArgument(
 818             'specify objects presentation style', '--content-disposition'),
 819         content_type=ValueArgument('specify content type', '--content-type'),
 820         uuid_for_read_permission=RepeatableArgument(
 821             'Give read access to a user or group (can be repeated) '
 822             'Use * for all users',
 823             '--read-permission'),
 824         uuid_for_write_permission=RepeatableArgument(
 825             'Give write access to a user or group (can be repeated) '
 826             'Use * for all users',
 827             '--write-permission'),
 828         public=FlagArgument('make object publicly accessible', '--public'),
 829         progress_bar=ProgressBarArgument(
 830             'do not show progress bar',
 831             ('-N', '--no-progress-bar'),
 832             default=False),
 833         overwrite=FlagArgument('Force (over)write', ('-f', '--force')),
 834         recursive=FlagArgument(
 835             'Recursively upload directory *contents* + subdirectories',
 836             ('-r', '--recursive')),
 837         unchunked=FlagArgument(
 838             'Upload file as one block (not recommended)', '--unchunked'),
 839         md5_checksum=ValueArgument(
 840             'Confirm upload with a custom checksum (MD5)', '--etag'),
 841         use_hashes=FlagArgument(
 842             'Source file contains hashmap not data', '--source-is-hashmap'),
 843     )
 844
 845     def _sharing(self):
 846         sharing = dict()
 847         readlist = self['uuid_for_read_permission']
 848         if readlist:
 849             sharing['read'] = self['uuid_for_read_permission']
 850         writelist = self['uuid_for_write_permission']
 851         if writelist:
 852             sharing['write'] = self['uuid_for_write_permission']
 853         return sharing or None
 854
 855     def _check_container_limit(self, path):
 856         cl_dict = self.client.get_container_limit()
 857         container_limit = int(cl_dict['x-container-policy-quota'])
 858         r = self.client.container_get()
 859         used_bytes = sum(int(o['bytes']) for o in r.json)
 860         path_size = get_path_size(path)
 861         if container_limit and path_size > (container_limit - used_bytes):
 862             raise CLIError(
 863                 'Container %s (limit(%s) - used(%s)) < (size(%s) of %s)' % (
 864                     self.client.container,
 865                     format_size(container_limit),
 866                     format_size(used_bytes),
 867                     format_size(path_size),
 868                     path),
 869                 details=[
 870                     'Check accound limit: /file quota',
 871                     'Check container limit:',
 872                     '\t/file containerlimit get %s' % self.client.container,
 873                     'Increase container limit:',
 874                     '\t/file containerlimit set <new limit> %s' % (
 875                         self.client.container)])
 876
 877     def _src_dst(self, local_path, remote_path, objlist=None):
 878         lpath = path.abspath(local_path)
 879         short_path = path.basename(path.abspath(local_path))
 880         rpath = remote_path or short_path
 881         if path.isdir(lpath):
 882             if not self['recursive']:
 883                 raise CLIError('%s is a directory' % lpath, details=[
 884                     'Use %s to upload directories & contents' % (
 885                         self.arguments['recursive'].lvalue)])
 886             robj = self.client.container_get(path=rpath)
 887             if not self['overwrite']:
 888                 if robj.json:
 889                     raise CLIError(
 890                         'Objects/files prefixed as %s already exist' % rpath,
 891                         details=['Existing objects:'] + ['\t/%s/\t%s' % (
 892                             o['name'],
 893                             o['content_type'][12:]) for o in robj.json] + [
 894                             'Use -f to add, overwrite or resume'])
 895                 else:
 896                     try:
 897                         topobj = self.client.get_object_info(rpath)
 898                         if not self._is_dir(topobj):
 899                             raise CLIError(
 900                                 'Object /%s/%s exists but not a directory' % (
 901                                     self.container, rpath),
 902                                 details=['Use -f to overwrite'])
 903                     except ClientError as ce:
 904                         if ce.status not in (404, ):
 905                             raise
 906             self._check_container_limit(lpath)
 907             prev = ''
 908             for top, subdirs, files in walk(lpath):
 909                 if top != prev:
 910                     prev = top
 911                     try:
 912                         rel_path = rpath + top.split(lpath)[1]
 913                     except IndexError:
 914                         rel_path = rpath
 915                     self.error('mkdir /%s/%s' % (
 916                         self.client.container, rel_path))
 917                     self.client.create_directory(rel_path)
 918                 for f in files:
 919                     fpath = path.join(top, f)
 920                     if path.isfile(fpath):
 921                         rel_path = rel_path.replace(path.sep, '/')
 922                         pathfix = f.replace(path.sep, '/')
 923                         yield open(fpath, 'rb'), '%s/%s' % (rel_path, pathfix)
 924                     else:
 925                         self.error('%s is not a regular file' % fpath)
 926         else:
 927             if not path.isfile(lpath):
 928                 raise CLIError(('%s is not a regular file' % lpath) if (
 929                     path.exists(lpath)) else '%s does not exist' % lpath)
 930             try:
 931                 robj = self.client.get_object_info(rpath)
 932                 if remote_path and self._is_dir(robj):
 933                     rpath += '/%s' % (short_path.replace(path.sep, '/'))
 934                     self.client.get_object_info(rpath)
 935                 if not self['overwrite']:
 936                     raise CLIError(
 937                         'Object /%s/%s already exists' % (
 938                             self.container, rpath),
 939                         details=['use -f to overwrite / resume'])
 940             except ClientError as ce:
 941                 if ce.status not in (404, ):
 942                     raise
 943             self._check_container_limit(lpath)
 944             yield open(lpath, 'rb'), rpath
 945
 946     def _run(self, local_path, remote_path):
 947         self.client.MAX_THREADS = int(self['max_threads'] or 5)
 948         params = dict(
 949             content_encoding=self['content_encoding'],
 950             content_type=self['content_type'],
 951             content_disposition=self['content_disposition'],
 952             sharing=self._sharing(),
 953             public=self['public'])
 954         uploaded, container_info_cache = list, dict()
 955         rpref = 'pithos://%s' if self['account'] else ''
 956         for f, rpath in self._src_dst(local_path, remote_path):
 957             self.error('%s --> %s/%s/%s' % (
 958                 f.name, rpref, self.client.container, rpath))
 959             if not (self['content_type'] and self['content_encoding']):
 960                 ctype, cenc = guess_mime_type(f.name)
 961                 params['content_type'] = self['content_type'] or ctype
 962                 params['content_encoding'] = self['content_encoding'] or cenc
 963             if self['unchunked']:
 964                 r = self.client.upload_object_unchunked(
 965                     rpath, f,
 966                     etag=self['md5_checksum'], withHashFile=self['use_hashes'],
 967                     **params)
 968                 if self['with_output'] or self['json_output']:
 969                     r['name'] = '/%s/%s' % (self.client.container, rpath)
 970                     uploaded.append(r)
 971             else:
 972                 try:
 973                     (progress_bar, upload_cb) = self._safe_progress_bar(
 974                         'Uploading %s' % f.name.split(path.sep)[-1])
 975                     if progress_bar:
 976                         hash_bar = progress_bar.clone()
 977                         hash_cb = hash_bar.get_generator(
 978                             'Calculating block hashes')
 979                     else:
 980                         hash_cb = None
 981                     r = self.client.upload_object(
 982                         rpath, f,
 983                         hash_cb=hash_cb,
 984                         upload_cb=upload_cb,
 985                         container_info_cache=container_info_cache,
 986                         **params)
 987                     if self['with_output'] or self['json_output']:
 988                         r['name'] = '/%s/%s' % (self.client.container, rpath)
 989                         uploaded.append(r)
 990                 except Exception:
 991                     self._safe_progress_bar_finish(progress_bar)
 992                     raise
 993                 finally:
 994                     self._safe_progress_bar_finish(progress_bar)
 995         self._optional_output(uploaded)
 996         self.error('Upload completed')
 997
 998     def main(self, local_path, remote_path_or_url):
 999         super(self.__class__, self)._run(remote_path_or_url)
1000         remote_path = self.path or path.basename(path.abspath(local_path))
1001         self._run(local_path=local_path, remote_path=remote_path)
1002
1003
1004 class RangeArgument(ValueArgument):
1005     """
1006     :value type: string of the form <start>-<end> where <start> and <end> are
1007         integers
1008     :value returns: the input string, after type checking <start> and <end>
1009     """
1010
1011     @property
1012     def value(self):
1013         return getattr(self, '_value', self.default)
1014
1015     @value.setter
1016     def value(self, newvalues):
1017         if newvalues:
1018             self._value = getattr(self, '_value', self.default)
1019             for newvalue in newvalues.split(','):
1020                 self._value = ('%s,' % self._value) if self._value else ''
1021                 start, sep, end = newvalue.partition('-')
1022                 if sep:
1023                     if start:
1024                         start, end = (int(start), int(end))
1025                         if start > end:
1026                             raise CLIInvalidArgument(
1027                                 'Invalid range %s' % newvalue, details=[
1028                                 'Valid range formats',
1029                                 '  START-END', '  UP_TO', '  -FROM',
1030                                 'where all values are integers',
1031                                 'OR a compination (csv), e.g.,',
1032                                 '  %s=5,10-20,-5' % self.lvalue])
1033                         self._value += '%s-%s' % (start, end)
1034                     else:
1035                         self._value += '-%s' % int(end)
1036                 else:
1037                     self._value += '%s' % int(start)
1038
1039
1040 @command(file_cmds)
1041 class file_cat(_pithos_container):
1042     """Fetch remote file contents"""
1043
1044     arguments = dict(
1045         range=RangeArgument('show range of data e.g., 5,10-20,-5', '--range'),
1046         if_match=ValueArgument('show output if ETags match', '--if-match'),
1047         if_none_match=ValueArgument(
1048             'show output if ETags match', '--if-none-match'),
1049         if_modified_since=DateArgument(
1050             'show output modified since then', '--if-modified-since'),
1051         if_unmodified_since=DateArgument(
1052             'show output unmodified since then', '--if-unmodified-since'),
1053         object_version=ValueArgument(
1054             'Get contents of the chosen version', '--object-version')
1055     )
1056
1057     @errors.generic.all
1058     @errors.pithos.connection
1059     @errors.pithos.container
1060     @errors.pithos.object_path
1061     def _run(self):
1062         r = self.client.download_object(
1063             self.path, self._out,
1064             range_str=self['range'],
1065             version=self['object_version'],
1066             if_match=self['if_match'],
1067             if_none_match=self['if_none_match'],
1068             if_modified_since=self['if_modified_since'],
1069             if_unmodified_since=self['if_unmodified_since'])
1070         print r
1071
1072     def main(self, path_or_url):
1073         super(self.__class__, self)._run(path_or_url)
1074         self._run()
1075
1076
1077 @command(file_cmds)
1078 class file_download(_pithos_container):
1079     """Download a remove file or directory object to local file system"""
1080
1081     arguments = dict(
1082         resume=FlagArgument(
1083             'Resume/Overwrite (attempt resume, else overwrite)',
1084             ('-f', '--resume')),
1085         range=RangeArgument(
1086             'Download only that range of data e.g., 5,10-20,-5', '--range'),
1087         matching_etag=ValueArgument('download iff ETag match', '--if-match'),
1088         non_matching_etag=ValueArgument(
1089             'download iff ETags DO NOT match', '--if-none-match'),
1090         modified_since_date=DateArgument(
1091             'download iff remote file is modified since then',
1092             '--if-modified-since'),
1093         unmodified_since_date=DateArgument(
1094             'show output iff remote file is unmodified since then',
1095             '--if-unmodified-since'),
1096         object_version=ValueArgument(
1097             'download a file of a specific version', '--object-version'),
1098         max_threads=IntArgument('default: 5', '--threads'),
1099         progress_bar=ProgressBarArgument(
1100             'do not show progress bar', ('-N', '--no-progress-bar'),
1101             default=False),
1102         recursive=FlagArgument(
1103             'Download a remote directory object and its contents',
1104             ('-r', '--recursive'))
1105         )
1106
1107     def _src_dst(self, local_path):
1108         """Create a list of (src, dst) where src is a remote location and dst
1109         is an open file descriptor. Directories are denoted as (None, dirpath)
1110         and they are pretended to other objects in a very strict order (shorter
1111         to longer path)."""
1112         ret = []
1113         try:
1114             if self.path:
1115                 obj = self.client.get_object_info(
1116                     self.path, version=self['object_version'])
1117                 obj.setdefault('name', self.path.strip('/'))
1118             else:
1119                 obj = None
1120         except ClientError as ce:
1121             if ce.status in (404, ):
1122                 raiseCLIError(ce, details=[
1123                     'To download an object, it must exist either as a file or'
1124                     ' as a directory.',
1125                     'For example, to download everything under prefix/ the '
1126                     'directory "prefix" must exist.',
1127                     'To see if an remote object is actually there:',
1128                     '  /file info [/CONTAINER/]OBJECT',
1129                     'To create a directory object:',
1130                     '  /file mkdir [/CONTAINER/]OBJECT'])
1131             if ce.status in (204, ):
1132                 raise CLIError(
1133                     'No file or directory objects to download',
1134                     details=[
1135                         'To download a container (e.g., %s):' % self.container,
1136                         '  [kamaki] container download %s [LOCAL_PATH]' % (
1137                             self.container)])
1138             raise
1139         rpath = self.path.strip('/')
1140         if local_path and self.path and local_path.endswith('/'):
1141             local_path = local_path[-1:]
1142
1143         if (not obj) or self._is_dir(obj):
1144             if self['recursive']:
1145                 if not (self.path or local_path.endswith('/')):
1146                     #  Download the whole container
1147                     local_path = '' if local_path in ('.', ) else local_path
1148                     local_path = '%s/' % (local_path or self.container)
1149                 obj = obj or dict(
1150                     name='', content_type='application/directory')
1151                 dirs, files = [obj, ], []
1152                 objects = self.client.container_get(
1153                     path=self.path,
1154                     if_modified_since=self['modified_since_date'],
1155                     if_unmodified_since=self['unmodified_since_date'])
1156                 for o in objects.json:
1157                     (dirs if self._is_dir(o) else files).append(o)
1158
1159                 #  Put the directories on top of the list
1160                 for dpath in sorted(['%s%s' % (
1161                         local_path, d['name'][len(rpath):]) for d in dirs]):
1162                     if path.exists(dpath):
1163                         if path.isdir(dpath):
1164                             continue
1165                         raise CLIError(
1166                             'Cannot replace local file %s with a directory '
1167                             'of the same name' % dpath,
1168                             details=[
1169                                 'Either remove the file or specify a'
1170                                 'different target location'])
1171                     ret.append((None, dpath, None))
1172
1173                 #  Append the file objects
1174                 for opath in [o['name'] for o in files]:
1175                     lpath = '%s%s' % (local_path, opath[len(rpath):])
1176                     if self['resume']:
1177                         fxists = path.exists(lpath)
1178                         if fxists and path.isdir(lpath):
1179                             raise CLIError(
1180                                 'Cannot change local dir %s info file' % (
1181                                     lpath),
1182                                 details=[
1183                                     'Either remove the file or specify a'
1184                                     'different target location'])
1185                         ret.append((opath, lpath, fxists))
1186                     elif path.exists(lpath):
1187                         raise CLIError(
1188                             'Cannot overwrite %s' % lpath,
1189                             details=['To overwrite/resume, use  %s' % (
1190                                 self.arguments['resume'].lvalue)])
1191                     else:
1192                         ret.append((opath, lpath, None))
1193             elif self.path:
1194                 raise CLIError(
1195                     'Remote object /%s/%s is a directory' % (
1196                         self.container, local_path),
1197                     details=['Use %s to download directories' % (
1198                         self.arguments['recursive'].lvalue)])
1199             else:
1200                 parsed_name = self.arguments['recursive'].lvalue
1201                 raise CLIError(
1202                     'Cannot download container %s' % self.container,
1203                     details=[
1204                         'Use %s to download containers' % parsed_name,
1205                         '  [kamaki] file download %s /%s [LOCAL_PATH]' % (
1206                             parsed_name, self.container)])
1207         else:
1208             #  Remote object is just a file
1209             if path.exists(local_path) and not self['resume']:
1210                 raise CLIError(
1211                     'Cannot overwrite local file %s' % (lpath),
1212                     details=['To overwrite/resume, use  %s' % (
1213                         self.arguments['resume'].lvalue)])
1214             ret.append((rpath, local_path, self['resume']))
1215         for r, l, resume in ret:
1216             if r:
1217                 with open(l, 'rwb+' if resume else 'wb+') as f:
1218                     yield (r, f)
1219             else:
1220                 yield (r, l)
1221
1222     @errors.generic.all
1223     @errors.pithos.connection
1224     @errors.pithos.container
1225     @errors.pithos.object_path
1226     @errors.pithos.local_path
1227     @errors.pithos.local_path_download
1228     def _run(self, local_path):
1229         self.client.MAX_THREADS = int(self['max_threads'] or 5)
1230         progress_bar = None
1231         try:
1232             for rpath, output_file in self._src_dst(local_path):
1233                 if not rpath:
1234                     self.error('Create local directory %s' % output_file)
1235                     makedirs(output_file)
1236                     continue
1237                 self.error('/%s/%s --> %s' % (
1238                     self.container, rpath, output_file.name))
1239                 progress_bar, download_cb = self._safe_progress_bar(
1240                     '  download')
1241                 self.client.download_object(
1242                     rpath, output_file,
1243                     download_cb=download_cb,
1244                     range_str=self['range'],
1245                     version=self['object_version'],
1246                     if_match=self['matching_etag'],
1247                     resume=self['resume'],
1248                     if_none_match=self['non_matching_etag'],
1249                     if_modified_since=self['modified_since_date'],
1250                     if_unmodified_since=self['unmodified_since_date'])
1251         except KeyboardInterrupt:
1252             from threading import activeCount, enumerate as activethreads
1253             timeout = 0.5
1254             while activeCount() > 1:
1255                 self._out.write('\nCancel %s threads: ' % (activeCount() - 1))
1256                 self._out.flush()
1257                 for thread in activethreads():
1258                     try:
1259                         thread.join(timeout)
1260                         self._out.write('.' if thread.isAlive() else '*')
1261                     except RuntimeError:
1262                         continue
1263                     finally:
1264                         self._out.flush()
1265                         timeout += 0.1
1266             self.error('\nDownload canceled by user')
1267             if local_path is not None:
1268                 self.error('to resume, re-run with --resume')
1269         except Exception:
1270             self._safe_progress_bar_finish(progress_bar)
1271             raise
1272         finally:
1273             self._safe_progress_bar_finish(progress_bar)
1274
1275     def main(self, remote_path_or_url, local_path=None):
1276         super(self.__class__, self)._run(remote_path_or_url)
1277         local_path = local_path or self.path or '.'
1278         self._run(local_path=local_path)
1279
1280
1281 @command(container_cmds)
1282 class container_info(_pithos_account, _optional_json):
1283     """Get information about a container"""
1284
1285     arguments = dict(
1286         until_date=DateArgument('show metadata until then', '--until'),
1287         metadata=FlagArgument('Show only container metadata', '--metadata'),
1288         sizelimit=FlagArgument(
1289             'Show the maximum size limit for container', '--size-limit'),
1290         in_bytes=FlagArgument('Show size limit in bytes', ('-b', '--bytes'))
1291     )
1292
1293     @errors.generic.all
1294     @errors.pithos.connection
1295     @errors.pithos.container
1296     @errors.pithos.object_path
1297     def _run(self):
1298         if self['metadata']:
1299             r, preflen = dict(), len('x-container-meta-')
1300             for k, v in self.client.get_container_meta(
1301                     until=self['until_date']).items():
1302                 r[k[preflen:]] = v
1303         elif self['sizelimit']:
1304             r = self.client.get_container_limit(
1305                 self.container)['x-container-policy-quota']
1306             r = {'size limit': 'unlimited' if r in ('0', ) else (
1307                 int(r) if self['in_bytes'] else format_size(r))}
1308         else:
1309             r = self.client.get_container_info(self.container)
1310         self._print(r, self.print_dict)
1311
1312     def main(self, container):
1313         super(self.__class__, self)._run()
1314         self.container, self.client.container = container, container
1315         self._run()
1316
1317
1318 class VersioningArgument(ValueArgument):
1319
1320     schemes = ('auto', 'none')
1321
1322     @property
1323     def value(self):
1324         return getattr(self, '_value', None)
1325
1326     @value.setter
1327     def value(self, new_scheme):
1328         if new_scheme:
1329             new_scheme = new_scheme.lower()
1330             if new_scheme not in self.schemes:
1331                 raise CLIInvalidArgument('Invalid versioning value', details=[
1332                     'Valid versioning values are %s' % ', '.join(
1333                         self.schemes)])
1334             self._value = new_scheme
1335
1336
1337 @command(container_cmds)
1338 class container_modify(_pithos_account, _optional_json):
1339     """Modify the properties of a container"""
1340
1341     arguments = dict(
1342         metadata_to_add=KeyValueArgument(
1343             'Add metadata in the form KEY=VALUE (can be repeated)',
1344             '--metadata-add'),
1345         metadata_to_delete=RepeatableArgument(
1346             'Delete metadata by KEY (can be repeated)', '--metadata-del'),
1347         sizelimit=DataSizeArgument(
1348             'Set max size limit (0 for unlimited, '
1349             'use units B, KiB, KB, etc.)', '--size-limit'),
1350         versioning=VersioningArgument(
1351             'Set a versioning scheme (%s)' % ', '.join(
1352                 VersioningArgument.schemes), '--versioning')
1353     )
1354     required = [
1355         'metadata_to_add', 'metadata_to_delete', 'sizelimit', 'versioning']
1356
1357     @errors.generic.all
1358     @errors.pithos.connection
1359     @errors.pithos.container
1360     def _run(self, container):
1361         metadata = self['metadata_to_add']
1362         for k in (self['metadata_to_delete'] or []):
1363             metadata[k] = ''
1364         if metadata:
1365             self.client.set_container_meta(metadata)
1366             self._print(self.client.get_container_meta(), self.print_dict)
1367         if self['sizelimit'] is not None:
1368             self.client.set_container_limit(self['sizelimit'])
1369             r = self.client.get_container_limit()['x-container-policy-quota']
1370             r = 'unlimited' if r in ('0', ) else format_size(r)
1371             self.writeln('new size limit: %s' % r)
1372         if self['versioning']:
1373             self.client.set_container_versioning(self['versioning'])
1374             self.writeln('new versioning scheme: %s' % (
1375                 self.client.get_container_versioning(self.container)[
1376                     'x-container-policy-versioning']))
1377
1378     def main(self, container):
1379         super(self.__class__, self)._run()
1380         self.client.container, self.container = container, container
1381         self._run(container=container)
1382
1383
1384 @command(container_cmds)
1385 class container_list(_pithos_account, _optional_json, _name_filter):
1386     """List all containers, or their contents"""
1387
1388     arguments = dict(
1389         detail=FlagArgument('Containers with details', ('-l', '--list')),
1390         limit=IntArgument('limit number of listed items', ('-n', '--number')),
1391         marker=ValueArgument('output greater that marker', '--marker'),
1392         modified_since_date=ValueArgument(
1393             'show output modified since then', '--if-modified-since'),
1394         unmodified_since_date=ValueArgument(
1395             'show output not modified since then', '--if-unmodified-since'),
1396         until_date=DateArgument('show metadata until then', '--until'),
1397         shared=FlagArgument('show only shared', '--shared'),
1398         more=FlagArgument('read long results', '--more'),
1399         enum=FlagArgument('Enumerate results', '--enumerate'),
1400         recursive=FlagArgument(
1401             'Recursively list containers and their contents',
1402             ('-r', '--recursive')),
1403         shared_by_me=FlagArgument(
1404             'show only files shared to other users', '--shared-by-me'),
1405         public=FlagArgument('show only published objects', '--public'),
1406     )
1407
1408     def print_containers(self, container_list):
1409         for index, container in enumerate(container_list):
1410             if 'bytes' in container:
1411                 size = format_size(container['bytes'])
1412             prfx = ('%s. ' % (index + 1)) if self['enum'] else ''
1413             _cname = container['name'] if (
1414                 self['more']) else bold(container['name'])
1415             cname = u'%s%s' % (prfx, _cname)
1416             if self['detail']:
1417                 self.writeln(cname)
1418                 pretty_c = container.copy()
1419                 if 'bytes' in container:
1420                     pretty_c['bytes'] = '%s (%s)' % (container['bytes'], size)
1421                 self.print_dict(pretty_c, exclude=('name'))
1422                 self.writeln()
1423             else:
1424                 if 'count' in container and 'bytes' in container:
1425                     self.writeln('%s (%s, %s objects)' % (
1426                         cname, size, container['count']))
1427                 else:
1428                     self.writeln(cname)
1429             objects = container.get('objects', [])
1430             if objects:
1431                 self.print_objects(objects)
1432                 self.writeln('')
1433
1434     def _create_object_forest(self, container_list):
1435         try:
1436             for container in container_list:
1437                 self.client.container = container['name']
1438                 objects = self.client.container_get(
1439                     limit=False if self['more'] else self['limit'],
1440                     if_modified_since=self['modified_since_date'],
1441                     if_unmodified_since=self['unmodified_since_date'],
1442                     until=self['until_date'],
1443                     show_only_shared=self['shared_by_me'],
1444                     public=self['public'])
1445                 container['objects'] = objects.json
1446         finally:
1447             self.client.container = None
1448
1449     @errors.generic.all
1450     @errors.pithos.connection
1451     @errors.pithos.object_path
1452     @errors.pithos.container
1453     def _run(self, container):
1454         if container:
1455             r = self.client.container_get(
1456                 limit=False if self['more'] else self['limit'],
1457                 marker=self['marker'],
1458                 if_modified_since=self['modified_since_date'],
1459                 if_unmodified_since=self['unmodified_since_date'],
1460                 until=self['until_date'],
1461                 show_only_shared=self['shared_by_me'],
1462                 public=self['public'])
1463         else:
1464             r = self.client.account_get(
1465                 limit=False if self['more'] else self['limit'],
1466                 marker=self['marker'],
1467                 if_modified_since=self['modified_since_date'],
1468                 if_unmodified_since=self['unmodified_since_date'],
1469                 until=self['until_date'],
1470                 show_only_shared=self['shared_by_me'],
1471                 public=self['public'])
1472         files = self._filter_by_name(r.json)
1473         if self['recursive'] and not container:
1474             self._create_object_forest(files)
1475         if self['more']:
1476             outbu, self._out = self._out, StringIO()
1477         try:
1478             if self['json_output'] or self['output_format']:
1479                 self._print(files)
1480             else:
1481                 (self.print_objects if container else self.print_containers)(
1482                     files)
1483         finally:
1484             if self['more']:
1485                 pager(self._out.getvalue())
1486                 self._out = outbu
1487
1488     def main(self, container=None):
1489         super(self.__class__, self)._run()
1490         self.client.container, self.container = container, container
1491         self._run(container)
1492
1493
1494 @command(container_cmds)
1495 class container_create(_pithos_account):
1496     """Create a new container"""
1497
1498     arguments = dict(
1499         versioning=ValueArgument(
1500             'set container versioning (auto/none)', '--versioning'),
1501         limit=IntArgument('set default container limit', '--limit'),
1502         meta=KeyValueArgument(
1503             'set container metadata (can be repeated)', '--meta')
1504     )
1505
1506     @errors.generic.all
1507     @errors.pithos.connection
1508     @errors.pithos.container
1509     def _run(self, container):
1510         try:
1511             self.client.create_container(
1512                 container=container,
1513                 sizelimit=self['limit'],
1514                 versioning=self['versioning'],
1515                 metadata=self['meta'],
1516                 success=(201, ))
1517         except ClientError as ce:
1518             if ce.status in (202, ):
1519                 raise CLIError(
1520                     'Container %s alread exists' % container, details=[
1521                     'Either delete %s or choose another name' % (container)])
1522             raise
1523
1524     def main(self, new_container):
1525         super(self.__class__, self)._run()
1526         self._run(container=new_container)
1527
1528
1529 @command(container_cmds)
1530 class container_delete(_pithos_account):
1531     """Delete a container"""
1532
1533     arguments = dict(
1534         yes=FlagArgument('Do not prompt for permission', '--yes'),
1535         recursive=FlagArgument(
1536             'delete container even if not empty', ('-r', '--recursive'))
1537     )
1538
1539     @errors.generic.all
1540     @errors.pithos.connection
1541     @errors.pithos.container
1542     def _run(self, container):
1543         num_of_contents = int(self.client.get_container_info(container)[
1544             'x-container-object-count'])
1545         delimiter, msg = None, 'Delete container %s ?' % container
1546         if self['recursive']:
1547             delimiter, msg = '/', 'Empty and d%s' % msg[1:]
1548         elif num_of_contents:
1549             raise CLIError('Container %s is not empty' % container, details=[
1550                 'Use %s to delete non-empty containers' % (
1551                     self.arguments['recursive'].lvalue)])
1552         if self['yes'] or self.ask_user(msg):
1553             if num_of_contents:
1554                 self.client.del_container(delimiter=delimiter)
1555             self.client.purge_container()
1556
1557     def main(self, container):
1558         super(self.__class__, self)._run()
1559         self.container, self.client.container = container, container
1560         self._run(container)
1561
1562
1563 @command(container_cmds)
1564 class container_empty(_pithos_account):
1565     """Empty a container"""
1566
1567     arguments = dict(yes=FlagArgument('Do not prompt for permission', '--yes'))
1568
1569     @errors.generic.all
1570     @errors.pithos.connection
1571     @errors.pithos.container
1572     def _run(self, container):
1573         if self['yes'] or self.ask_user('Empty container %s ?' % container):
1574             self.client.del_container(delimiter='/')
1575
1576     def main(self, container):
1577         super(self.__class__, self)._run()
1578         self.container, self.client.container = container, container
1579         self._run(container)
1580
1581
1582 @command(sharer_cmds)
1583 class sharer_list(_pithos_account, _optional_json):
1584     """List accounts who share file objects with current user"""
1585
1586     arguments = dict(
1587         detail=FlagArgument('show detailed output', ('-l', '--details')),
1588         marker=ValueArgument('show output greater then marker', '--marker')
1589     )
1590
1591     @errors.generic.all
1592     @errors.pithos.connection
1593     def _run(self):
1594         accounts = self.client.get_sharing_accounts(marker=self['marker'])
1595         if not (self['json_output'] or self['output_format']):
1596             usernames = self._uuids2usernames(
1597                 [acc['name'] for acc in accounts])
1598             for item in accounts:
1599                 uuid = item['name']
1600                 item['id'], item['name'] = uuid, usernames[uuid]
1601                 if not self['detail']:
1602                     item.pop('last_modified')
1603         self._print(accounts)
1604
1605     def main(self):
1606         super(self.__class__, self)._run()
1607         self._run()
1608
1609
1610 @command(sharer_cmds)
1611 class sharer_info(_pithos_account, _optional_json):
1612     """Details on a Pithos+ sharer account (default: current account)"""
1613
1614     @errors.generic.all
1615     @errors.pithos.connection
1616     def _run(self):
1617         self._print(self.client.get_account_info(), self.print_dict)
1618
1619     def main(self, account_uuid=None):
1620         super(self.__class__, self)._run()
1621         if account_uuid:
1622             self.client.account, self.account = account_uuid, account_uuid
1623         self._run()
1624
1625
1626 class _pithos_group(_pithos_account):
1627     prefix = 'x-account-group-'
1628     preflen = len(prefix)
1629
1630     def _groups(self):
1631         groups = dict()
1632         for k, v in self.client.get_account_group().items():
1633             groups[k[self.preflen:]] = v
1634         return groups
1635
1636
1637 @command(group_cmds)
1638 class group_list(_pithos_group, _optional_json):
1639     """list all groups and group members"""
1640
1641     @errors.generic.all
1642     @errors.pithos.connection
1643     def _run(self):
1644         self._print(self._groups(), self.print_dict)
1645
1646     def main(self):
1647         super(self.__class__, self)._run()
1648         self._run()
1649
1650
1651 @command(group_cmds)
1652 class group_create(_pithos_group, _optional_json):
1653     """Create a group of users"""
1654
1655     arguments = dict(
1656         user_uuid=RepeatableArgument('Add a user to the group', '--uuid'),
1657         username=RepeatableArgument('Add a user to the group', '--username')
1658     )
1659     required = ['user_uuid', 'user_name']
1660
1661     @errors.generic.all
1662     @errors.pithos.connection
1663     def _run(self, groupname, *users):
1664         if groupname in self._groups() and not self.ask_user(
1665                 'Group %s already exists, overwrite?' % groupname):
1666             self.error('Aborted')
1667             return
1668         self.client.set_account_group(groupname, users)
1669         self._print(self._groups(), self.print_dict)
1670
1671     def main(self, groupname):
1672         super(self.__class__, self)._run()
1673         users = (self['user_uuid'] or []) + self._usernames2uuids(
1674             self['username'] or []).values()
1675         if users:
1676             self._run(groupname, *users)
1677         else:
1678             raise CLISyntaxError(
1679                 'No valid users specified, use %s or %s' % (
1680                     self.arguments['user_uuid'].lvalue,
1681                     self.arguments['username'].lvalue),
1682                 details=[
1683                     'Check if a username or uuid is valid with',
1684                     '  user uuid2username', 'OR', '  user username2uuid'])
1685
1686
1687 @command(group_cmds)
1688 class group_delete(_pithos_group, _optional_json):
1689     """Delete a user group"""
1690
1691     @errors.generic.all
1692     @errors.pithos.connection
1693     def _run(self, groupname):
1694         self.client.del_account_group(groupname)
1695         self._print(self._groups(), self.print_dict)
1696
1697     def main(self, groupname):
1698         super(self.__class__, self)._run()
1699         self._run(groupname)
1700
1701
1702 #  Deprecated commands
1703
1704 @command(file_cmds)
1705 class file_publish(_pithos_init):
1706     """DEPRECATED, replaced by [kamaki] file modify OBJECT --publish"""
1707
1708     def main(self, *args):
1709         raise CLISyntaxError('DEPRECATED', details=[
1710             'This command is replaced by:',
1711             '  [kamaki] file modify OBJECT --publish'])
1712
1713
1714 @command(file_cmds)
1715 class file_unpublish(_pithos_init):
1716     """DEPRECATED, replaced by [kamaki] file modify OBJECT --unpublish"""
1717
1718     def main(self, *args):
1719         raise CLISyntaxError('DEPRECATED', details=[
1720             'This command is replaced by:',
1721             '  [kamaki] file modify OBJECT --unpublish'])