code.grnet.gr Git - kamaki/blob - kamaki/cli/commands/pithos.py

   1 # Copyright 2011-2013 GRNET S.A. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or
   4 # without modification, are permitted provided that the following
   5 # conditions are met:
   6 #
   7 #   1. Redistributions of source code must retain the above
   8 #      copyright notice, this list of conditions and the following
   9 #      disclaimer.
  10 #
  11 #   2. Redistributions in binary form must reproduce the above
  12 #      copyright notice, this list of conditions and the following
  13 #      disclaimer in the documentation and/or other materials
  14 #      provided with the distribution.
  15 #
  16 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  17 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  23 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  24 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  26 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27 # POSSIBILITY OF SUCH DAMAGE.
  28 #
  29 # The views and conclusions contained in the software and
  30 # documentation are those of the authors and should not be
  31 # interpreted as representing official policies, either expressed
  32 # or implied, of GRNET S.A.command
  33
  34 from time import localtime, strftime
  35 from io import StringIO
  36 from pydoc import pager
  37 from os import path, walk, makedirs
  38
  39 from kamaki.clients.pithos import PithosClient, ClientError
  40
  41 from kamaki.cli import command
  42 from kamaki.cli.command_tree import CommandTree
  43 from kamaki.cli.commands import (
  44     _command_init, errors, addLogSettings, DontRaiseKeyError, _optional_json,
  45     _name_filter, _optional_output_cmd)
  46 from kamaki.cli.errors import (
  47     CLIBaseUrlError, CLIError, CLIInvalidArgument, raiseCLIError,
  48     CLISyntaxError)
  49 from kamaki.cli.argument import (
  50     FlagArgument, IntArgument, ValueArgument, DateArgument, KeyValueArgument,
  51     ProgressBarArgument, RepeatableArgument, DataSizeArgument)
  52 from kamaki.cli.utils import (
  53     format_size, bold, get_path_size, guess_mime_type)
  54
  55 file_cmds = CommandTree('file', 'Pithos+/Storage object level API commands')
  56 container_cmds = CommandTree(
  57     'container', 'Pithos+/Storage container level API commands')
  58 sharer_cmds = CommandTree('sharer', 'Pithos+/Storage sharers')
  59 group_cmds = CommandTree('group', 'Pithos+/Storage user groups')
  60 _commands = [file_cmds, container_cmds, sharer_cmds, group_cmds]
  61
  62
  63 class _pithos_init(_command_init):
  64     """Initilize a pithos+ client
  65     There is always a default account (current user uuid)
  66     There is always a default container (pithos)
  67     """
  68
  69     @DontRaiseKeyError
  70     def _custom_container(self):
  71         return self.config.get_cloud(self.cloud, 'pithos_container')
  72
  73     @DontRaiseKeyError
  74     def _custom_uuid(self):
  75         return self.config.get_cloud(self.cloud, 'pithos_uuid')
  76
  77     def _set_account(self):
  78         self.account = self._custom_uuid()
  79         if self.account:
  80             return
  81         astakos = getattr(self, 'auth_base', None)
  82         if astakos:
  83             self.account = astakos.user_term('id', self.token)
  84         else:
  85             raise CLIBaseUrlError(service='astakos')
  86
  87     @errors.generic.all
  88     @addLogSettings
  89     def _run(self):
  90         cloud = getattr(self, 'cloud', None)
  91         if cloud:
  92             self.base_url = self._custom_url('pithos')
  93         else:
  94             self.cloud = 'default'
  95         self.token = self._custom_token('pithos')
  96         self.container = self._custom_container() or 'pithos'
  97
  98         astakos = getattr(self, 'auth_base', None)
  99         if astakos:
 100             self.token = self.token or astakos.token
 101             if not self.base_url:
 102                 pithos_endpoints = astakos.get_service_endpoints(
 103                     self._custom_type('pithos') or 'object-store',
 104                     self._custom_version('pithos') or '')
 105                 self.base_url = pithos_endpoints['publicURL']
 106         else:
 107             raise CLIBaseUrlError(service='astakos')
 108
 109         self._set_account()
 110         self.client = PithosClient(
 111             self.base_url, self.token, self.account, self.container)
 112
 113     def main(self):
 114         self._run()
 115
 116
 117 class _pithos_account(_pithos_init):
 118     """Setup account"""
 119
 120     def __init__(self, arguments={}, auth_base=None, cloud=None):
 121         super(_pithos_account, self).__init__(arguments, auth_base, cloud)
 122         self['account'] = ValueArgument(
 123             'Use (a different) user uuid', ('-A', '--account'))
 124
 125     def print_objects(self, object_list):
 126         for index, obj in enumerate(object_list):
 127             pretty_obj = obj.copy()
 128             index += 1
 129             empty_space = ' ' * (len(str(len(object_list))) - len(str(index)))
 130             if 'subdir' in obj:
 131                 continue
 132             if self._is_dir(obj):
 133                 size = 'D'
 134             else:
 135                 size = format_size(obj['bytes'])
 136                 pretty_obj['bytes'] = '%s (%s)' % (obj['bytes'], size)
 137             oname = obj['name'] if self['more'] else bold(obj['name'])
 138             prfx = ('%s%s. ' % (empty_space, index)) if self['enum'] else ''
 139             if self['detail']:
 140                 self.writeln('%s%s' % (prfx, oname))
 141                 self.print_dict(pretty_obj, exclude=('name'))
 142                 self.writeln()
 143             else:
 144                 oname = '%s%9s %s' % (prfx, size, oname)
 145                 oname += '/' if self._is_dir(obj) else u''
 146                 self.writeln(oname)
 147
 148     @staticmethod
 149     def _is_dir(remote_dict):
 150         return 'application/directory' in remote_dict.get(
 151             'content_type', remote_dict.get('content-type', ''))
 152
 153     def _run(self):
 154         super(_pithos_account, self)._run()
 155         self.client.account = self['account'] or getattr(
 156             self, 'account', getattr(self.client, 'account', None))
 157
 158
 159 class _pithos_container(_pithos_account):
 160     """Setup container"""
 161
 162     def __init__(self, arguments={}, auth_base=None, cloud=None):
 163         super(_pithos_container, self).__init__(arguments, auth_base, cloud)
 164         self['container'] = ValueArgument(
 165             'Use this container (default: pithos)', ('-C', '--container'))
 166
 167     @staticmethod
 168     def _resolve_pithos_url(url):
 169         """Match urls of one of the following formats:
 170         pithos://ACCOUNT/CONTAINER/OBJECT_PATH
 171         /CONTAINER/OBJECT_PATH
 172         return account, container, path
 173         """
 174         account, container, obj_path, prefix = '', '', url, 'pithos://'
 175         if url.startswith(prefix):
 176             account, sep, url = url[len(prefix):].partition('/')
 177             url = '/%s' % url
 178         if url.startswith('/'):
 179             container, sep, obj_path = url[1:].partition('/')
 180         return account, container, obj_path
 181
 182     def _run(self, url=None):
 183         acc, con, self.path = self._resolve_pithos_url(url or '')
 184         #  self.account = acc or getattr(self, 'account', '')
 185         super(_pithos_container, self)._run()
 186         self.container = con or self['container'] or getattr(
 187             self, 'container', None) or getattr(self.client, 'container', '')
 188         self.client.account = acc or self.client.account
 189         self.client.container = self.container
 190
 191
 192 @command(file_cmds)
 193 class file_info(_pithos_container, _optional_json):
 194     """Get information/details about a file"""
 195
 196     arguments = dict(
 197         object_version=ValueArgument(
 198             'download a file of a specific version', '--object-version'),
 199         hashmap=FlagArgument(
 200             'Get file hashmap instead of details', '--hashmap'),
 201         matching_etag=ValueArgument(
 202             'show output if ETags match', '--if-match'),
 203         non_matching_etag=ValueArgument(
 204             'show output if ETags DO NOT match', '--if-none-match'),
 205         modified_since_date=DateArgument(
 206             'show output modified since then', '--if-modified-since'),
 207         unmodified_since_date=DateArgument(
 208             'show output unmodified since then', '--if-unmodified-since'),
 209         sharing=FlagArgument(
 210             'show object permissions and sharing information', '--sharing'),
 211         metadata=FlagArgument('show only object metadata', '--metadata'),
 212         versions=FlagArgument(
 213             'show the list of versions for the file', '--object-versions')
 214     )
 215
 216     def version_print(self, versions):
 217         return {'/%s/%s' % (self.container, self.path): [
 218             dict(version_id=vitem[0], created=strftime(
 219                 '%d-%m-%Y %H:%M:%S',
 220                 localtime(float(vitem[1])))) for vitem in versions]}
 221
 222     @errors.generic.all
 223     @errors.pithos.connection
 224     @errors.pithos.container
 225     @errors.pithos.object_path
 226     def _run(self):
 227         if self['hashmap']:
 228             r = self.client.get_object_hashmap(
 229                 self.path,
 230                 version=self['object_version'],
 231                 if_match=self['matching_etag'],
 232                 if_none_match=self['non_matching_etag'],
 233                 if_modified_since=self['modified_since_date'],
 234                 if_unmodified_since=self['unmodified_since_date'])
 235         elif self['sharing']:
 236             r = self.client.get_object_sharing(self.path)
 237             r['public url'] = self.client.get_object_info(
 238                 self.path, version=self['object_version']).get(
 239                     'x-object-public', None)
 240         elif self['metadata']:
 241             r, preflen = dict(), len('x-object-meta-')
 242             for k, v in self.client.get_object_meta(self.path).items():
 243                 r[k[preflen:]] = v
 244         elif self['versions']:
 245             r = self.version_print(
 246                 self.client.get_object_versionlist(self.path))
 247         else:
 248             r = self.client.get_object_info(
 249                 self.path, version=self['object_version'])
 250         self._print(r, self.print_dict)
 251
 252     def main(self, path_or_url):
 253         super(self.__class__, self)._run(path_or_url)
 254         self._run()
 255
 256
 257 @command(file_cmds)
 258 class file_list(_pithos_container, _optional_json, _name_filter):
 259     """List all objects in a container or a directory object"""
 260
 261     arguments = dict(
 262         detail=FlagArgument('detailed output', ('-l', '--list')),
 263         limit=IntArgument('limit number of listed items', ('-n', '--number')),
 264         marker=ValueArgument('output greater that marker', '--marker'),
 265         delimiter=ValueArgument('show output up to delimiter', '--delimiter'),
 266         meta=ValueArgument(
 267             'show output with specified meta keys', '--meta',
 268             default=[]),
 269         if_modified_since=ValueArgument(
 270             'show output modified since then', '--if-modified-since'),
 271         if_unmodified_since=ValueArgument(
 272             'show output not modified since then', '--if-unmodified-since'),
 273         until=DateArgument('show metadata until then', '--until'),
 274         format=ValueArgument(
 275             'format to parse until data (default: d/m/Y H:M:S )', '--format'),
 276         shared_by_me=FlagArgument(
 277             'show only files shared to other users', '--shared-by-me'),
 278         public=FlagArgument('show only published objects', '--public'),
 279         more=FlagArgument('read long results', '--more'),
 280         enum=FlagArgument('Enumerate results', '--enumerate'),
 281         recursive=FlagArgument(
 282             'Recursively list containers and their contents',
 283             ('-R', '--recursive'))
 284     )
 285
 286     @errors.generic.all
 287     @errors.pithos.connection
 288     @errors.pithos.container
 289     @errors.pithos.object_path
 290     def _run(self):
 291         r = self.client.container_get(
 292             limit=False if self['more'] else self['limit'],
 293             marker=self['marker'],
 294             prefix=self['name_pref'],
 295             delimiter=self['delimiter'],
 296             path=self.path or '',
 297             show_only_shared=self['shared_by_me'],
 298             public=self['public'],
 299             if_modified_since=self['if_modified_since'],
 300             if_unmodified_since=self['if_unmodified_since'],
 301             until=self['until'],
 302             meta=self['meta'])
 303
 304         #  REMOVE THIS if version >> 0.12
 305         if not r.json:
 306             self.error('  NOTE: Since v0.12, use / for containers e.g.,')
 307             self.error('    [kamaki] file list /pithos')
 308
 309         files = self._filter_by_name(r.json)
 310         if self['more']:
 311             outbu, self._out = self._out, StringIO()
 312         try:
 313             if self['json_output'] or self['output_format']:
 314                 self._print(files)
 315             else:
 316                 self.print_objects(files)
 317         finally:
 318             if self['more']:
 319                 pager(self._out.getvalue())
 320                 self._out = outbu
 321
 322     def main(self, path_or_url=''):
 323         super(self.__class__, self)._run(path_or_url)
 324         self._run()
 325
 326
 327 @command(file_cmds)
 328 class file_modify(_pithos_container):
 329     """Modify the attributes of a file or directory object"""
 330
 331     arguments = dict(
 332         publish=FlagArgument(
 333             'Make an object public (returns the public URL)', '--publish'),
 334         unpublish=FlagArgument(
 335             'Make an object unpublic', '--unpublish'),
 336         uuid_for_read_permission=RepeatableArgument(
 337             'Give read access to user/group (can be repeated, accumulative). '
 338             'Format for users: UUID . Format for groups: UUID:GROUP . '
 339             'Use * for all users/groups', '--read-permission'),
 340         uuid_for_write_permission=RepeatableArgument(
 341             'Give write access to user/group (can be repeated, accumulative). '
 342             'Format for users: UUID . Format for groups: UUID:GROUP . '
 343             'Use * for all users/groups', '--write-permission'),
 344         no_permissions=FlagArgument('Remove permissions', '--no-permissions'),
 345         metadata_to_set=KeyValueArgument(
 346             'Add metadata (KEY=VALUE) to an object (can be repeated)',
 347             '--metadata-add'),
 348         metadata_key_to_delete=RepeatableArgument(
 349             'Delete object metadata (can be repeated)', '--metadata-del'),
 350     )
 351     required = [
 352         'publish', 'unpublish', 'uuid_for_read_permission', 'metadata_to_set',
 353         'uuid_for_write_permission', 'no_permissions',
 354         'metadata_key_to_delete']
 355
 356     @errors.generic.all
 357     @errors.pithos.connection
 358     @errors.pithos.container
 359     @errors.pithos.object_path
 360     def _run(self):
 361         if self['publish']:
 362             self.writeln(self.client.publish_object(self.path))
 363         if self['unpublish']:
 364             self.client.unpublish_object(self.path)
 365         if self['uuid_for_read_permission'] or self[
 366                 'uuid_for_write_permission']:
 367             perms = self.client.get_object_sharing(self.path)
 368             read, write = perms.get('read', ''), perms.get('write', '')
 369             read = read.split(',') if read else []
 370             write = write.split(',') if write else []
 371             read += (self['uuid_for_read_permission'] or [])
 372             write += (self['uuid_for_write_permission'] or [])
 373             self.client.set_object_sharing(
 374                 self.path, read_permission=read, write_permission=write)
 375             self.print_dict(self.client.get_object_sharing(self.path))
 376         if self['no_permissions']:
 377             self.client.del_object_sharing(self.path)
 378         metadata = self['metadata_to_set'] or dict()
 379         for k in (self['metadata_key_to_delete'] or []):
 380             metadata[k] = ''
 381         if metadata:
 382             self.client.set_object_meta(self.path, metadata)
 383             self.print_dict(self.client.get_object_meta(self.path))
 384
 385     def main(self, path_or_url):
 386         super(self.__class__, self)._run(path_or_url)
 387         if self['publish'] and self['unpublish']:
 388             raise CLIInvalidArgument(
 389                 'Arguments %s and %s cannot be used together' % (
 390                     self.arguments['publish'].lvalue,
 391                     self.arguments['publish'].lvalue))
 392         if self['no_permissions'] and (
 393                 self['uuid_for_read_permission'] or self[
 394                     'uuid_for_write_permission']):
 395             raise CLIInvalidArgument(
 396                 '%s cannot be used with other permission arguments' % (
 397                     self.arguments['no_permissions'].lvalue))
 398         self._run()
 399
 400
 401 def _assert_path(self, path_or_url):
 402     if not self.path:
 403         raiseCLIError(
 404             'Directory path is missing in location %s' % path_or_url,
 405             details=['Location format:    [[pithos://UUID]/CONTAINER/]PATH'])
 406
 407
 408 @command(file_cmds)
 409 class file_create(_pithos_container, _optional_output_cmd):
 410     """Create an empty file"""
 411
 412     arguments = dict(
 413         content_type=ValueArgument(
 414             'Set content type (default: application/octet-stream)',
 415             '--content-type',
 416             default='application/octet-stream')
 417     )
 418
 419     @errors.generic.all
 420     @errors.pithos.connection
 421     @errors.pithos.container
 422     def _run(self):
 423         self._optional_output(
 424             self.client.create_object(self.path, self['content_type']))
 425
 426     def main(self, path_or_url):
 427         super(self.__class__, self)._run(path_or_url)
 428         _assert_path(self, path_or_url)
 429         self._run()
 430
 431
 432 @command(file_cmds)
 433 class file_mkdir(_pithos_container, _optional_output_cmd):
 434     """Create a directory: /file create --content-type='applcation/directory'
 435     """
 436
 437     @errors.generic.all
 438     @errors.pithos.connection
 439     @errors.pithos.container
 440     def _run(self, path):
 441         self._optional_output(self.client.create_directory(self.path))
 442
 443     def main(self, path_or_url):
 444         super(self.__class__, self)._run(path_or_url)
 445         _assert_path(self, path_or_url)
 446         self._run(self.path)
 447
 448
 449 @command(file_cmds)
 450 class file_delete(_pithos_container):
 451     """Delete a file or directory object"""
 452
 453     arguments = dict(
 454         until_date=DateArgument('remove history until then', '--until'),
 455         yes=FlagArgument('Do not prompt for permission', '--yes'),
 456         recursive=FlagArgument(
 457             'If a directory, empty first', ('-r', '--recursive')),
 458         delimiter=ValueArgument(
 459             'delete objects prefixed with <object><delimiter>', '--delimiter')
 460     )
 461
 462     @errors.generic.all
 463     @errors.pithos.connection
 464     @errors.pithos.container
 465     @errors.pithos.object_path
 466     def _run(self):
 467         if self.path:
 468             if self['yes'] or self.ask_user(
 469                     'Delete /%s/%s ?' % (self.container, self.path)):
 470                 self.client.del_object(
 471                     self.path,
 472                     until=self['until_date'],
 473                     delimiter='/' if self['recursive'] else self['delimiter'])
 474             else:
 475                 self.error('Aborted')
 476         else:
 477             if self['yes'] or self.ask_user(
 478                     'Empty container /%s ?' % self.container):
 479                 self.client.container_delete(self.container, delimiter='/')
 480             else:
 481                 self.error('Aborted')
 482
 483     def main(self, path_or_url):
 484         super(self.__class__, self)._run(path_or_url)
 485         self._run()
 486
 487
 488 class _source_destination(_pithos_container, _optional_output_cmd):
 489
 490     sd_arguments = dict(
 491         destination_user_uuid=ValueArgument(
 492             'default: current user uuid', '--to-account'),
 493         destination_container=ValueArgument(
 494             'default: pithos', '--to-container'),
 495         source_prefix=FlagArgument(
 496             'Transfer all files that are prefixed with SOURCE PATH If the '
 497             'destination path is specified, replace SOURCE_PATH with '
 498             'DESTINATION_PATH',
 499             ('-r', '--recursive')),
 500         force=FlagArgument(
 501             'Overwrite destination objects, if needed', ('-f', '--force')),
 502         source_version=ValueArgument(
 503             'The version of the source object', '--source-version')
 504     )
 505
 506     def __init__(self, arguments={}, auth_base=None, cloud=None):
 507         self.arguments.update(arguments)
 508         self.arguments.update(self.sd_arguments)
 509         super(_source_destination, self).__init__(
 510             self.arguments, auth_base, cloud)
 511
 512     def _report_transfer(self, src, dst, transfer_name):
 513         if not dst:
 514             if transfer_name in ('move', ):
 515                 self.error('  delete source directory %s' % src)
 516             return
 517         dst_prf = '' if self.account == self.dst_client.account else (
 518                 'pithos://%s' % self.dst_client.account)
 519         if src:
 520             src_prf = '' if self.account == self.dst_client.account else (
 521                     'pithos://%s' % self.account)
 522             self.error('  %s %s/%s/%s\n  -->  %s/%s/%s' % (
 523                 transfer_name,
 524                 src_prf, self.container, src,
 525                 dst_prf, self.dst_client.container, dst))
 526         else:
 527             self.error('  mkdir %s/%s/%s' % (
 528                 dst_prf, self.dst_client.container, dst))
 529
 530     @errors.generic.all
 531     @errors.pithos.account
 532     def _src_dst(self, version=None):
 533         """Preconditions:
 534         self.account, self.container, self.path
 535         self.dst_acc, self.dst_con, self.dst_path
 536         They should all be configured properly
 537         :returns: [(src_path, dst_path), ...], if src_path is None, create
 538             destination directory
 539         """
 540         src_objects, dst_objects, pairs = dict(), dict(), []
 541         try:
 542             for obj in self.dst_client.list_objects(
 543                     prefix=self.dst_path or self.path or '/'):
 544                 dst_objects[obj['name']] = obj
 545         except ClientError as ce:
 546             if ce.status in (404, ):
 547                 raise CLIError(
 548                     'Destination container pithos://%s/%s not found' % (
 549                         self.dst_client.account, self.dst_client.container))
 550             raise ce
 551         if self['source_prefix']:
 552             #  Copy and replace prefixes
 553             for src_obj in self.client.list_objects(prefix=self.path):
 554                 src_objects[src_obj['name']] = src_obj
 555             for src_path, src_obj in src_objects.items():
 556                 dst_path = '%s%s' % (
 557                     self.dst_path or self.path, src_path[len(self.path):])
 558                 dst_obj = dst_objects.get(dst_path, None)
 559                 if self['force'] or not dst_obj:
 560                     #  Just do it
 561                     pairs.append((
 562                         None if self._is_dir(src_obj) else src_path, dst_path))
 563                     if self._is_dir(src_obj):
 564                         pairs.append((self.path or dst_path, None))
 565                 elif not (self._is_dir(dst_obj) and self._is_dir(src_obj)):
 566                     raise CLIError(
 567                         'Destination object exists', importance=2, details=[
 568                             'Failed while transfering:',
 569                             '    pithos://%s/%s/%s' % (
 570                                     self.account,
 571                                     self.container,
 572                                     src_path),
 573                             '--> pithos://%s/%s/%s' % (
 574                                     self.dst_client.account,
 575                                     self.dst_client.container,
 576                                     dst_path),
 577                             'Use %s to transfer overwrite' % (
 578                                     self.arguments['force'].lvalue)])
 579         else:
 580             #  One object transfer
 581             try:
 582                 src_version_arg = self.arguments.get('source_version', None)
 583                 src_obj = self.client.get_object_info(
 584                     self.path,
 585                     version=src_version_arg.value if src_version_arg else None)
 586             except ClientError as ce:
 587                 if ce.status in (204, ):
 588                     raise CLIError(
 589                         'Missing specific path container %s' % self.container,
 590                         importance=2, details=[
 591                             'To transfer container contents %s' % (
 592                                 self.arguments['source_prefix'].lvalue)])
 593                 raise
 594             dst_path = self.dst_path or self.path
 595             dst_obj = dst_objects.get(dst_path or self.path, None)
 596             if self['force'] or not dst_obj:
 597                 pairs.append(
 598                     (None if self._is_dir(src_obj) else self.path, dst_path))
 599                 if self._is_dir(src_obj):
 600                     pairs.append((self.path or dst_path, None))
 601             elif self._is_dir(src_obj):
 602                 raise CLIError(
 603                     'Cannot transfer an application/directory object',
 604                     importance=2, details=[
 605                         'The object pithos://%s/%s/%s is a directory' % (
 606                             self.account,
 607                             self.container,
 608                             self.path),
 609                         'To recursively copy a directory, use',
 610                         '  %s' % self.arguments['source_prefix'].lvalue,
 611                         'To create a file, use',
 612                         '  /file create  (general purpose)',
 613                         '  /file mkdir   (a directory object)'])
 614             else:
 615                 raise CLIError(
 616                     'Destination object exists',
 617                     importance=2, details=[
 618                         'Failed while transfering:',
 619                         '    pithos://%s/%s/%s' % (
 620                                 self.account,
 621                                 self.container,
 622                                 self.path),
 623                         '--> pithos://%s/%s/%s' % (
 624                                 self.dst_client.account,
 625                                 self.dst_client.container,
 626                                 dst_path),
 627                         'Use %s to transfer overwrite' % (
 628                                 self.arguments['force'].lvalue)])
 629         return pairs
 630
 631     def _run(self, source_path_or_url, destination_path_or_url=''):
 632         super(_source_destination, self)._run(source_path_or_url)
 633         dst_acc, dst_con, dst_path = self._resolve_pithos_url(
 634             destination_path_or_url)
 635         self.dst_client = PithosClient(
 636             base_url=self.client.base_url, token=self.client.token,
 637             container=self[
 638                 'destination_container'] or dst_con or self.client.container,
 639             account=self[
 640                 'destination_user_uuid'] or dst_acc or self.account)
 641         self.dst_path = dst_path or self.path
 642
 643
 644 @command(file_cmds)
 645 class file_copy(_source_destination):
 646     """Copy objects, even between different accounts or containers"""
 647
 648     arguments = dict(
 649         public=ValueArgument('publish new object', '--public'),
 650         content_type=ValueArgument(
 651             'change object\'s content type', '--content-type'),
 652         source_version=ValueArgument(
 653             'The version of the source object', '--object-version')
 654     )
 655
 656     @errors.generic.all
 657     @errors.pithos.connection
 658     @errors.pithos.container
 659     @errors.pithos.account
 660     def _run(self):
 661         for src, dst in self._src_dst(self['source_version']):
 662             self._report_transfer(src, dst, 'copy')
 663             if src and dst:
 664                 self.dst_client.copy_object(
 665                     src_container=self.client.container,
 666                     src_object=src,
 667                     dst_container=self.dst_client.container,
 668                     dst_object=dst,
 669                     source_account=self.client.account,
 670                     source_version=self['source_version'],
 671                     public=self['public'],
 672                     content_type=self['content_type'])
 673             elif dst:
 674                 self.dst_client.create_directory(dst)
 675
 676     def main(self, source_path_or_url, destination_path_or_url=None):
 677         super(file_copy, self)._run(
 678             source_path_or_url, destination_path_or_url or '')
 679         self._run()
 680
 681
 682 @command(file_cmds)
 683 class file_move(_source_destination):
 684     """Move objects, even between different accounts or containers"""
 685
 686     arguments = dict(
 687         public=ValueArgument('publish new object', '--public'),
 688         content_type=ValueArgument(
 689             'change object\'s content type', '--content-type')
 690     )
 691
 692     @errors.generic.all
 693     @errors.pithos.connection
 694     @errors.pithos.container
 695     @errors.pithos.account
 696     def _run(self):
 697         for src, dst in self._src_dst():
 698             self._report_transfer(src, dst, 'move')
 699             if src and dst:
 700                 self.dst_client.move_object(
 701                     src_container=self.client.container,
 702                     src_object=src,
 703                     dst_container=self.dst_client.container,
 704                     dst_object=dst,
 705                     source_account=self.account,
 706                     public=self['public'],
 707                     content_type=self['content_type'])
 708             elif dst:
 709                 self.dst_client.create_directory(dst)
 710             else:
 711                 self.client.del_object(src)
 712
 713     def main(self, source_path_or_url, destination_path_or_url=None):
 714         super(file_move, self)._run(
 715             source_path_or_url, destination_path_or_url or '')
 716         self._run()
 717
 718
 719 @command(file_cmds)
 720 class file_append(_pithos_container, _optional_output_cmd):
 721     """Append local file to (existing) remote object
 722     The remote object should exist.
 723     If the remote object is a directory, it is transformed into a file.
 724     In the later case, objects under the directory remain intact.
 725     """
 726
 727     arguments = dict(
 728         progress_bar=ProgressBarArgument(
 729             'do not show progress bar', ('-N', '--no-progress-bar'),
 730             default=False),
 731         max_threads=IntArgument('default: 1', '--threads'),
 732     )
 733
 734     @errors.generic.all
 735     @errors.pithos.connection
 736     @errors.pithos.container
 737     @errors.pithos.object_path
 738     def _run(self, local_path):
 739         if self['max_threads'] > 0:
 740             self.client.MAX_THREADS = int(self['max_threads'])
 741         (progress_bar, upload_cb) = self._safe_progress_bar('Appending')
 742         try:
 743             with open(local_path, 'rb') as f:
 744                 self._optional_output(
 745                     self.client.append_object(self.path, f, upload_cb))
 746         finally:
 747             self._safe_progress_bar_finish(progress_bar)
 748
 749     def main(self, local_path, remote_path_or_url):
 750         super(self.__class__, self)._run(remote_path_or_url)
 751         self._run(local_path)
 752
 753
 754 @command(file_cmds)
 755 class file_truncate(_pithos_container, _optional_output_cmd):
 756     """Truncate remote file up to size"""
 757
 758     arguments = dict(
 759         size_in_bytes=IntArgument('Length of file after truncation', '--size')
 760     )
 761     required = ('size_in_bytes', )
 762
 763     @errors.generic.all
 764     @errors.pithos.connection
 765     @errors.pithos.container
 766     @errors.pithos.object_path
 767     @errors.pithos.object_size
 768     def _run(self, size):
 769         self._optional_output(self.client.truncate_object(self.path, size))
 770
 771     def main(self, path_or_url):
 772         super(self.__class__, self)._run(path_or_url)
 773         self._run(size=self['size_in_bytes'])
 774
 775
 776 @command(file_cmds)
 777 class file_overwrite(_pithos_container, _optional_output_cmd):
 778     """Overwrite part of a remote file"""
 779
 780     arguments = dict(
 781         progress_bar=ProgressBarArgument(
 782             'do not show progress bar', ('-N', '--no-progress-bar'),
 783             default=False),
 784         start_position=IntArgument('File position in bytes', '--from'),
 785         end_position=IntArgument('File position in bytes', '--to'),
 786     )
 787     required = ('start_position', 'end_position')
 788
 789     @errors.generic.all
 790     @errors.pithos.connection
 791     @errors.pithos.container
 792     @errors.pithos.object_path
 793     @errors.pithos.object_size
 794     def _run(self, local_path, start, end):
 795         start, end = int(start), int(end)
 796         (progress_bar, upload_cb) = self._safe_progress_bar(
 797             'Overwrite %s bytes' % (end - start))
 798         try:
 799             with open(path.abspath(local_path), 'rb') as f:
 800                 self._optional_output(self.client.overwrite_object(
 801                     obj=self.path,
 802                     start=start,
 803                     end=end,
 804                     source_file=f,
 805                     upload_cb=upload_cb))
 806         finally:
 807             self._safe_progress_bar_finish(progress_bar)
 808
 809     def main(self, local_path, path_or_url):
 810         super(self.__class__, self)._run(path_or_url)
 811         self.path = self.path or path.basename(local_path)
 812         self._run(
 813             local_path=local_path,
 814             start=self['start_position'],
 815             end=self['end_position'])
 816
 817
 818 @command(file_cmds)
 819 class file_upload(_pithos_container, _optional_output_cmd):
 820     """Upload a file"""
 821
 822     arguments = dict(
 823         max_threads=IntArgument('default: 5', '--threads'),
 824         content_encoding=ValueArgument(
 825             'set MIME content type', '--content-encoding'),
 826         content_disposition=ValueArgument(
 827             'specify objects presentation style', '--content-disposition'),
 828         content_type=ValueArgument('specify content type', '--content-type'),
 829         uuid_for_read_permission=RepeatableArgument(
 830             'Give read access to a user or group (can be repeated) '
 831             'Use * for all users',
 832             '--read-permission'),
 833         uuid_for_write_permission=RepeatableArgument(
 834             'Give write access to a user or group (can be repeated) '
 835             'Use * for all users',
 836             '--write-permission'),
 837         public=FlagArgument('make object publicly accessible', '--public'),
 838         progress_bar=ProgressBarArgument(
 839             'do not show progress bar',
 840             ('-N', '--no-progress-bar'),
 841             default=False),
 842         overwrite=FlagArgument('Force (over)write', ('-f', '--force')),
 843         recursive=FlagArgument(
 844             'Recursively upload directory *contents* + subdirectories',
 845             ('-r', '--recursive')),
 846         unchunked=FlagArgument(
 847             'Upload file as one block (not recommended)', '--unchunked'),
 848         md5_checksum=ValueArgument(
 849             'Confirm upload with a custom checksum (MD5)', '--etag'),
 850         use_hashes=FlagArgument(
 851             'Source file contains hashmap not data', '--source-is-hashmap'),
 852     )
 853
 854     def _sharing(self):
 855         sharing = dict()
 856         readlist = self['uuid_for_read_permission']
 857         if readlist:
 858             sharing['read'] = self['uuid_for_read_permission']
 859         writelist = self['uuid_for_write_permission']
 860         if writelist:
 861             sharing['write'] = self['uuid_for_write_permission']
 862         return sharing or None
 863
 864     def _check_container_limit(self, path):
 865         cl_dict = self.client.get_container_limit()
 866         container_limit = int(cl_dict['x-container-policy-quota'])
 867         r = self.client.container_get()
 868         used_bytes = sum(int(o['bytes']) for o in r.json)
 869         path_size = get_path_size(path)
 870         if container_limit and path_size > (container_limit - used_bytes):
 871             raise CLIError(
 872                 'Container %s (limit(%s) - used(%s)) < (size(%s) of %s)' % (
 873                     self.client.container,
 874                     format_size(container_limit),
 875                     format_size(used_bytes),
 876                     format_size(path_size),
 877                     path),
 878                 details=[
 879                     'Check accound limit: /file quota',
 880                     'Check container limit:',
 881                     '\t/file containerlimit get %s' % self.client.container,
 882                     'Increase container limit:',
 883                     '\t/file containerlimit set <new limit> %s' % (
 884                         self.client.container)])
 885
 886     def _src_dst(self, local_path, remote_path, objlist=None):
 887         lpath = path.abspath(local_path)
 888         short_path = path.basename(path.abspath(local_path))
 889         rpath = remote_path or short_path
 890         if path.isdir(lpath):
 891             if not self['recursive']:
 892                 raise CLIError('%s is a directory' % lpath, details=[
 893                     'Use %s to upload directories & contents' % (
 894                         self.arguments['recursive'].lvalue)])
 895             robj = self.client.container_get(path=rpath)
 896             if not self['overwrite']:
 897                 if robj.json:
 898                     raise CLIError(
 899                         'Objects/files prefixed as %s already exist' % rpath,
 900                         details=['Existing objects:'] + ['\t/%s/\t%s' % (
 901                             o['name'],
 902                             o['content_type'][12:]) for o in robj.json] + [
 903                             'Use -f to add, overwrite or resume'])
 904                 else:
 905                     try:
 906                         topobj = self.client.get_object_info(rpath)
 907                         if not self._is_dir(topobj):
 908                             raise CLIError(
 909                                 'Object /%s/%s exists but not a directory' % (
 910                                     self.container, rpath),
 911                                 details=['Use -f to overwrite'])
 912                     except ClientError as ce:
 913                         if ce.status not in (404, ):
 914                             raise
 915             self._check_container_limit(lpath)
 916             prev = ''
 917             for top, subdirs, files in walk(lpath):
 918                 if top != prev:
 919                     prev = top
 920                     try:
 921                         rel_path = rpath + top.split(lpath)[1]
 922                     except IndexError:
 923                         rel_path = rpath
 924                     self.error('mkdir /%s/%s' % (
 925                         self.client.container, rel_path))
 926                     self.client.create_directory(rel_path)
 927                 for f in files:
 928                     fpath = path.join(top, f)
 929                     if path.isfile(fpath):
 930                         rel_path = rel_path.replace(path.sep, '/')
 931                         pathfix = f.replace(path.sep, '/')
 932                         yield open(fpath, 'rb'), '%s/%s' % (rel_path, pathfix)
 933                     else:
 934                         self.error('%s is not a regular file' % fpath)
 935         else:
 936             if not path.isfile(lpath):
 937                 raise CLIError(('%s is not a regular file' % lpath) if (
 938                     path.exists(lpath)) else '%s does not exist' % lpath)
 939             try:
 940                 robj = self.client.get_object_info(rpath)
 941                 if remote_path and self._is_dir(robj):
 942                     rpath += '/%s' % (short_path.replace(path.sep, '/'))
 943                     self.client.get_object_info(rpath)
 944                 if not self['overwrite']:
 945                     raise CLIError(
 946                         'Object /%s/%s already exists' % (
 947                             self.container, rpath),
 948                         details=['use -f to overwrite / resume'])
 949             except ClientError as ce:
 950                 if ce.status not in (404, ):
 951                     raise
 952             self._check_container_limit(lpath)
 953             yield open(lpath, 'rb'), rpath
 954
 955     def _run(self, local_path, remote_path):
 956         self.client.MAX_THREADS = int(self['max_threads'] or 5)
 957         params = dict(
 958             content_encoding=self['content_encoding'],
 959             content_type=self['content_type'],
 960             content_disposition=self['content_disposition'],
 961             sharing=self._sharing(),
 962             public=self['public'])
 963         uploaded, container_info_cache = list, dict()
 964         rpref = 'pithos://%s' if self['account'] else ''
 965         for f, rpath in self._src_dst(local_path, remote_path):
 966             self.error('%s --> %s/%s/%s' % (
 967                 f.name, rpref, self.client.container, rpath))
 968             if not (self['content_type'] and self['content_encoding']):
 969                 ctype, cenc = guess_mime_type(f.name)
 970                 params['content_type'] = self['content_type'] or ctype
 971                 params['content_encoding'] = self['content_encoding'] or cenc
 972             if self['unchunked']:
 973                 r = self.client.upload_object_unchunked(
 974                     rpath, f,
 975                     etag=self['md5_checksum'], withHashFile=self['use_hashes'],
 976                     **params)
 977                 if self['with_output'] or self['json_output']:
 978                     r['name'] = '/%s/%s' % (self.client.container, rpath)
 979                     uploaded.append(r)
 980             else:
 981                 try:
 982                     (progress_bar, upload_cb) = self._safe_progress_bar(
 983                         'Uploading %s' % f.name.split(path.sep)[-1])
 984                     if progress_bar:
 985                         hash_bar = progress_bar.clone()
 986                         hash_cb = hash_bar.get_generator(
 987                             'Calculating block hashes')
 988                     else:
 989                         hash_cb = None
 990                     r = self.client.upload_object(
 991                         rpath, f,
 992                         hash_cb=hash_cb,
 993                         upload_cb=upload_cb,
 994                         container_info_cache=container_info_cache,
 995                         **params)
 996                     if self['with_output'] or self['json_output']:
 997                         r['name'] = '/%s/%s' % (self.client.container, rpath)
 998                         uploaded.append(r)
 999                 except Exception:
1000                     self._safe_progress_bar_finish(progress_bar)
1001                     raise
1002                 finally:
1003                     self._safe_progress_bar_finish(progress_bar)
1004         self._optional_output(uploaded)
1005         self.error('Upload completed')
1006
1007     def main(self, local_path, remote_path_or_url):
1008         super(self.__class__, self)._run(remote_path_or_url)
1009         remote_path = self.path or path.basename(path.abspath(local_path))
1010         self._run(local_path=local_path, remote_path=remote_path)
1011
1012
1013 class RangeArgument(ValueArgument):
1014     """
1015     :value type: string of the form <start>-<end> where <start> and <end> are
1016         integers
1017     :value returns: the input string, after type checking <start> and <end>
1018     """
1019
1020     @property
1021     def value(self):
1022         return getattr(self, '_value', self.default)
1023
1024     @value.setter
1025     def value(self, newvalues):
1026         if newvalues:
1027             self._value = getattr(self, '_value', self.default)
1028             for newvalue in newvalues.split(','):
1029                 self._value = ('%s,' % self._value) if self._value else ''
1030                 start, sep, end = newvalue.partition('-')
1031                 if sep:
1032                     if start:
1033                         start, end = (int(start), int(end))
1034                         if start > end:
1035                             raise CLIInvalidArgument(
1036                                 'Invalid range %s' % newvalue, details=[
1037                                 'Valid range formats',
1038                                 '  START-END', '  UP_TO', '  -FROM',
1039                                 'where all values are integers',
1040                                 'OR a compination (csv), e.g.,',
1041                                 '  %s=5,10-20,-5' % self.lvalue])
1042                         self._value += '%s-%s' % (start, end)
1043                     else:
1044                         self._value += '-%s' % int(end)
1045                 else:
1046                     self._value += '%s' % int(start)
1047
1048
1049 @command(file_cmds)
1050 class file_cat(_pithos_container):
1051     """Fetch remote file contents"""
1052
1053     arguments = dict(
1054         range=RangeArgument('show range of data e.g., 5,10-20,-5', '--range'),
1055         if_match=ValueArgument('show output if ETags match', '--if-match'),
1056         if_none_match=ValueArgument(
1057             'show output if ETags match', '--if-none-match'),
1058         if_modified_since=DateArgument(
1059             'show output modified since then', '--if-modified-since'),
1060         if_unmodified_since=DateArgument(
1061             'show output unmodified since then', '--if-unmodified-since'),
1062         object_version=ValueArgument(
1063             'Get contents of the chosen version', '--object-version')
1064     )
1065
1066     @errors.generic.all
1067     @errors.pithos.connection
1068     @errors.pithos.container
1069     @errors.pithos.object_path
1070     def _run(self):
1071         self.client.download_object(
1072             self.path, self._out,
1073             range_str=self['range'],
1074             version=self['object_version'],
1075             if_match=self['if_match'],
1076             if_none_match=self['if_none_match'],
1077             if_modified_since=self['if_modified_since'],
1078             if_unmodified_since=self['if_unmodified_since'])
1079         self._out.flush()
1080
1081     def main(self, path_or_url):
1082         super(self.__class__, self)._run(path_or_url)
1083         self._run()
1084
1085
1086 @command(file_cmds)
1087 class file_download(_pithos_container):
1088     """Download a remove file or directory object to local file system"""
1089
1090     arguments = dict(
1091         resume=FlagArgument(
1092             'Resume/Overwrite (attempt resume, else overwrite)',
1093             ('-f', '--resume')),
1094         range=RangeArgument(
1095             'Download only that range of data e.g., 5,10-20,-5', '--range'),
1096         matching_etag=ValueArgument('download iff ETag match', '--if-match'),
1097         non_matching_etag=ValueArgument(
1098             'download iff ETags DO NOT match', '--if-none-match'),
1099         modified_since_date=DateArgument(
1100             'download iff remote file is modified since then',
1101             '--if-modified-since'),
1102         unmodified_since_date=DateArgument(
1103             'show output iff remote file is unmodified since then',
1104             '--if-unmodified-since'),
1105         object_version=ValueArgument(
1106             'download a file of a specific version', '--object-version'),
1107         max_threads=IntArgument('default: 5', '--threads'),
1108         progress_bar=ProgressBarArgument(
1109             'do not show progress bar', ('-N', '--no-progress-bar'),
1110             default=False),
1111         recursive=FlagArgument(
1112             'Download a remote directory object and its contents',
1113             ('-r', '--recursive'))
1114         )
1115
1116     def _src_dst(self, local_path):
1117         """Create a list of (src, dst) where src is a remote location and dst
1118         is an open file descriptor. Directories are denoted as (None, dirpath)
1119         and they are pretended to other objects in a very strict order (shorter
1120         to longer path)."""
1121         ret = []
1122         try:
1123             if self.path:
1124                 obj = self.client.get_object_info(
1125                     self.path, version=self['object_version'])
1126                 obj.setdefault('name', self.path.strip('/'))
1127             else:
1128                 obj = None
1129         except ClientError as ce:
1130             if ce.status in (404, ):
1131                 raiseCLIError(ce, details=[
1132                     'To download an object, it must exist either as a file or'
1133                     ' as a directory.',
1134                     'For example, to download everything under prefix/ the '
1135                     'directory "prefix" must exist.',
1136                     'To see if an remote object is actually there:',
1137                     '  /file info [/CONTAINER/]OBJECT',
1138                     'To create a directory object:',
1139                     '  /file mkdir [/CONTAINER/]OBJECT'])
1140             if ce.status in (204, ):
1141                 raise CLIError(
1142                     'No file or directory objects to download',
1143                     details=[
1144                         'To download a container (e.g., %s):' % self.container,
1145                         '  [kamaki] container download %s [LOCAL_PATH]' % (
1146                             self.container)])
1147             raise
1148         rpath = self.path.strip('/')
1149         if local_path and self.path and local_path.endswith('/'):
1150             local_path = local_path[-1:]
1151
1152         if (not obj) or self._is_dir(obj):
1153             if self['recursive']:
1154                 if not (self.path or local_path.endswith('/')):
1155                     #  Download the whole container
1156                     local_path = '' if local_path in ('.', ) else local_path
1157                     local_path = '%s/' % (local_path or self.container)
1158                 obj = obj or dict(
1159                     name='', content_type='application/directory')
1160                 dirs, files = [obj, ], []
1161                 objects = self.client.container_get(
1162                     path=self.path,
1163                     if_modified_since=self['modified_since_date'],
1164                     if_unmodified_since=self['unmodified_since_date'])
1165                 for o in objects.json:
1166                     (dirs if self._is_dir(o) else files).append(o)
1167
1168                 #  Put the directories on top of the list
1169                 for dpath in sorted(['%s%s' % (
1170                         local_path, d['name'][len(rpath):]) for d in dirs]):
1171                     if path.exists(dpath):
1172                         if path.isdir(dpath):
1173                             continue
1174                         raise CLIError(
1175                             'Cannot replace local file %s with a directory '
1176                             'of the same name' % dpath,
1177                             details=[
1178                                 'Either remove the file or specify a'
1179                                 'different target location'])
1180                     ret.append((None, dpath, None))
1181
1182                 #  Append the file objects
1183                 for opath in [o['name'] for o in files]:
1184                     lpath = '%s%s' % (local_path, opath[len(rpath):])
1185                     if self['resume']:
1186                         fxists = path.exists(lpath)
1187                         if fxists and path.isdir(lpath):
1188                             raise CLIError(
1189                                 'Cannot change local dir %s info file' % (
1190                                     lpath),
1191                                 details=[
1192                                     'Either remove the file or specify a'
1193                                     'different target location'])
1194                         ret.append((opath, lpath, fxists))
1195                     elif path.exists(lpath):
1196                         raise CLIError(
1197                             'Cannot overwrite %s' % lpath,
1198                             details=['To overwrite/resume, use  %s' % (
1199                                 self.arguments['resume'].lvalue)])
1200                     else:
1201                         ret.append((opath, lpath, None))
1202             elif self.path:
1203                 raise CLIError(
1204                     'Remote object /%s/%s is a directory' % (
1205                         self.container, local_path),
1206                     details=['Use %s to download directories' % (
1207                         self.arguments['recursive'].lvalue)])
1208             else:
1209                 parsed_name = self.arguments['recursive'].lvalue
1210                 raise CLIError(
1211                     'Cannot download container %s' % self.container,
1212                     details=[
1213                         'Use %s to download containers' % parsed_name,
1214                         '  [kamaki] file download %s /%s [LOCAL_PATH]' % (
1215                             parsed_name, self.container)])
1216         else:
1217             #  Remote object is just a file
1218             if path.exists(local_path):
1219                 if not self['resume']:
1220                     raise CLIError(
1221                         'Cannot overwrite local file %s' % (local_path),
1222                         details=['To overwrite/resume, use  %s' % (
1223                             self.arguments['resume'].lvalue)])
1224             elif '/' in local_path[1:-1]:
1225                 dirs = [p for p in local_path.split('/') if p]
1226                 pref = '/' if local_path.startswith('/') else ''
1227                 for d in dirs[:-1]:
1228                     pref += d
1229                     if not path.exists(pref):
1230                         ret.append((None, d, None))
1231                     elif not path.isdir(pref):
1232                         raise CLIError(
1233                             'Failed to use %s as a destination' % local_path,
1234                             importance=3,
1235                             details=[
1236                                 'Local file %s is not a directory' % pref,
1237                                 'Destination prefix must consist of '
1238                                 'directories or non-existing names',
1239                                 'Either remove the file, or choose another '
1240                                 'destination'])
1241             ret.append((rpath, local_path, self['resume']))
1242         for r, l, resume in ret:
1243             if r:
1244                 with open(l, 'rwb+' if resume else 'wb+') as f:
1245                     yield (r, f)
1246             else:
1247                 yield (r, l)
1248
1249     @errors.generic.all
1250     @errors.pithos.connection
1251     @errors.pithos.container
1252     @errors.pithos.object_path
1253     @errors.pithos.local_path
1254     @errors.pithos.local_path_download
1255     def _run(self, local_path):
1256         self.client.MAX_THREADS = int(self['max_threads'] or 5)
1257         progress_bar = None
1258         try:
1259             for rpath, output_file in self._src_dst(local_path):
1260                 if not rpath:
1261                     self.error('Create local directory %s' % output_file)
1262                     makedirs(output_file)
1263                     continue
1264                 self.error('/%s/%s --> %s' % (
1265                     self.container, rpath, output_file.name))
1266                 progress_bar, download_cb = self._safe_progress_bar(
1267                     '  download')
1268                 self.client.download_object(
1269                     rpath, output_file,
1270                     download_cb=download_cb,
1271                     range_str=self['range'],
1272                     version=self['object_version'],
1273                     if_match=self['matching_etag'],
1274                     resume=self['resume'],
1275                     if_none_match=self['non_matching_etag'],
1276                     if_modified_since=self['modified_since_date'],
1277                     if_unmodified_since=self['unmodified_since_date'])
1278         except KeyboardInterrupt:
1279             from threading import activeCount, enumerate as activethreads
1280             timeout = 0.5
1281             while activeCount() > 1:
1282                 self._out.write('\nCancel %s threads: ' % (activeCount() - 1))
1283                 self._out.flush()
1284                 for thread in activethreads():
1285                     try:
1286                         thread.join(timeout)
1287                         self._out.write('.' if thread.isAlive() else '*')
1288                     except RuntimeError:
1289                         continue
1290                     finally:
1291                         self._out.flush()
1292                         timeout += 0.1
1293             self.error('\nDownload canceled by user')
1294             if local_path is not None:
1295                 self.error('to resume, re-run with --resume')
1296         finally:
1297             self._safe_progress_bar_finish(progress_bar)
1298
1299     def main(self, remote_path_or_url, local_path=None):
1300         super(self.__class__, self)._run(remote_path_or_url)
1301         local_path = local_path or self.path or '.'
1302         self._run(local_path=local_path)
1303
1304
1305 @command(container_cmds)
1306 class container_info(_pithos_account, _optional_json):
1307     """Get information about a container"""
1308
1309     arguments = dict(
1310         until_date=DateArgument('show metadata until then', '--until'),
1311         metadata=FlagArgument('Show only container metadata', '--metadata'),
1312         sizelimit=FlagArgument(
1313             'Show the maximum size limit for container', '--size-limit'),
1314         in_bytes=FlagArgument('Show size limit in bytes', ('-b', '--bytes'))
1315     )
1316
1317     @errors.generic.all
1318     @errors.pithos.connection
1319     @errors.pithos.container
1320     @errors.pithos.object_path
1321     def _run(self):
1322         if self['metadata']:
1323             r, preflen = dict(), len('x-container-meta-')
1324             for k, v in self.client.get_container_meta(
1325                     until=self['until_date']).items():
1326                 r[k[preflen:]] = v
1327         elif self['sizelimit']:
1328             r = self.client.get_container_limit(
1329                 self.container)['x-container-policy-quota']
1330             r = {'size limit': 'unlimited' if r in ('0', ) else (
1331                 int(r) if self['in_bytes'] else format_size(r))}
1332         else:
1333             r = self.client.get_container_info(self.container)
1334         self._print(r, self.print_dict)
1335
1336     def main(self, container):
1337         super(self.__class__, self)._run()
1338         self.container, self.client.container = container, container
1339         self._run()
1340
1341
1342 class VersioningArgument(ValueArgument):
1343
1344     schemes = ('auto', 'none')
1345
1346     @property
1347     def value(self):
1348         return getattr(self, '_value', None)
1349
1350     @value.setter
1351     def value(self, new_scheme):
1352         if new_scheme:
1353             new_scheme = new_scheme.lower()
1354             if new_scheme not in self.schemes:
1355                 raise CLIInvalidArgument('Invalid versioning value', details=[
1356                     'Valid versioning values are %s' % ', '.join(
1357                         self.schemes)])
1358             self._value = new_scheme
1359
1360
1361 @command(container_cmds)
1362 class container_modify(_pithos_account, _optional_json):
1363     """Modify the properties of a container"""
1364
1365     arguments = dict(
1366         metadata_to_add=KeyValueArgument(
1367             'Add metadata in the form KEY=VALUE (can be repeated)',
1368             '--metadata-add'),
1369         metadata_to_delete=RepeatableArgument(
1370             'Delete metadata by KEY (can be repeated)', '--metadata-del'),
1371         sizelimit=DataSizeArgument(
1372             'Set max size limit (0 for unlimited, '
1373             'use units B, KiB, KB, etc.)', '--size-limit'),
1374         versioning=VersioningArgument(
1375             'Set a versioning scheme (%s)' % ', '.join(
1376                 VersioningArgument.schemes), '--versioning')
1377     )
1378     required = [
1379         'metadata_to_add', 'metadata_to_delete', 'sizelimit', 'versioning']
1380
1381     @errors.generic.all
1382     @errors.pithos.connection
1383     @errors.pithos.container
1384     def _run(self, container):
1385         metadata = self['metadata_to_add']
1386         for k in (self['metadata_to_delete'] or []):
1387             metadata[k] = ''
1388         if metadata:
1389             self.client.set_container_meta(metadata)
1390             self._print(self.client.get_container_meta(), self.print_dict)
1391         if self['sizelimit'] is not None:
1392             self.client.set_container_limit(self['sizelimit'])
1393             r = self.client.get_container_limit()['x-container-policy-quota']
1394             r = 'unlimited' if r in ('0', ) else format_size(r)
1395             self.writeln('new size limit: %s' % r)
1396         if self['versioning']:
1397             self.client.set_container_versioning(self['versioning'])
1398             self.writeln('new versioning scheme: %s' % (
1399                 self.client.get_container_versioning(self.container)[
1400                     'x-container-policy-versioning']))
1401
1402     def main(self, container):
1403         super(self.__class__, self)._run()
1404         self.client.container, self.container = container, container
1405         self._run(container=container)
1406
1407
1408 @command(container_cmds)
1409 class container_list(_pithos_account, _optional_json, _name_filter):
1410     """List all containers, or their contents"""
1411
1412     arguments = dict(
1413         detail=FlagArgument('Containers with details', ('-l', '--list')),
1414         limit=IntArgument('limit number of listed items', ('-n', '--number')),
1415         marker=ValueArgument('output greater that marker', '--marker'),
1416         modified_since_date=ValueArgument(
1417             'show output modified since then', '--if-modified-since'),
1418         unmodified_since_date=ValueArgument(
1419             'show output not modified since then', '--if-unmodified-since'),
1420         until_date=DateArgument('show metadata until then', '--until'),
1421         shared=FlagArgument('show only shared', '--shared'),
1422         more=FlagArgument('read long results', '--more'),
1423         enum=FlagArgument('Enumerate results', '--enumerate'),
1424         recursive=FlagArgument(
1425             'Recursively list containers and their contents',
1426             ('-r', '--recursive')),
1427         shared_by_me=FlagArgument(
1428             'show only files shared to other users', '--shared-by-me'),
1429         public=FlagArgument('show only published objects', '--public'),
1430     )
1431
1432     def print_containers(self, container_list):
1433         for index, container in enumerate(container_list):
1434             if 'bytes' in container:
1435                 size = format_size(container['bytes'])
1436             prfx = ('%s. ' % (index + 1)) if self['enum'] else ''
1437             _cname = container['name'] if (
1438                 self['more']) else bold(container['name'])
1439             cname = u'%s%s' % (prfx, _cname)
1440             if self['detail']:
1441                 self.writeln(cname)
1442                 pretty_c = container.copy()
1443                 if 'bytes' in container:
1444                     pretty_c['bytes'] = '%s (%s)' % (container['bytes'], size)
1445                 self.print_dict(pretty_c, exclude=('name'))
1446                 self.writeln()
1447             else:
1448                 if 'count' in container and 'bytes' in container:
1449                     self.writeln('%s (%s, %s objects)' % (
1450                         cname, size, container['count']))
1451                 else:
1452                     self.writeln(cname)
1453             objects = container.get('objects', [])
1454             if objects:
1455                 self.print_objects(objects)
1456                 self.writeln('')
1457
1458     def _create_object_forest(self, container_list):
1459         try:
1460             for container in container_list:
1461                 self.client.container = container['name']
1462                 objects = self.client.container_get(
1463                     limit=False if self['more'] else self['limit'],
1464                     if_modified_since=self['modified_since_date'],
1465                     if_unmodified_since=self['unmodified_since_date'],
1466                     until=self['until_date'],
1467                     show_only_shared=self['shared_by_me'],
1468                     public=self['public'])
1469                 container['objects'] = objects.json
1470         finally:
1471             self.client.container = None
1472
1473     @errors.generic.all
1474     @errors.pithos.connection
1475     @errors.pithos.object_path
1476     @errors.pithos.container
1477     def _run(self, container):
1478         if container:
1479             r = self.client.container_get(
1480                 limit=False if self['more'] else self['limit'],
1481                 marker=self['marker'],
1482                 if_modified_since=self['modified_since_date'],
1483                 if_unmodified_since=self['unmodified_since_date'],
1484                 until=self['until_date'],
1485                 show_only_shared=self['shared_by_me'],
1486                 public=self['public'])
1487         else:
1488             r = self.client.account_get(
1489                 limit=False if self['more'] else self['limit'],
1490                 marker=self['marker'],
1491                 if_modified_since=self['modified_since_date'],
1492                 if_unmodified_since=self['unmodified_since_date'],
1493                 until=self['until_date'],
1494                 show_only_shared=self['shared_by_me'],
1495                 public=self['public'])
1496         files = self._filter_by_name(r.json)
1497         if self['recursive'] and not container:
1498             self._create_object_forest(files)
1499         if self['more']:
1500             outbu, self._out = self._out, StringIO()
1501         try:
1502             if self['json_output'] or self['output_format']:
1503                 self._print(files)
1504             else:
1505                 (self.print_objects if container else self.print_containers)(
1506                     files)
1507         finally:
1508             if self['more']:
1509                 pager(self._out.getvalue())
1510                 self._out = outbu
1511
1512     def main(self, container=None):
1513         super(self.__class__, self)._run()
1514         self.client.container, self.container = container, container
1515         self._run(container)
1516
1517
1518 @command(container_cmds)
1519 class container_create(_pithos_account):
1520     """Create a new container"""
1521
1522     arguments = dict(
1523         versioning=ValueArgument(
1524             'set container versioning (auto/none)', '--versioning'),
1525         limit=IntArgument('set default container limit', '--limit'),
1526         meta=KeyValueArgument(
1527             'set container metadata (can be repeated)', '--meta')
1528     )
1529
1530     @errors.generic.all
1531     @errors.pithos.connection
1532     @errors.pithos.container
1533     def _run(self, container):
1534         try:
1535             self.client.create_container(
1536                 container=container,
1537                 sizelimit=self['limit'],
1538                 versioning=self['versioning'],
1539                 metadata=self['meta'],
1540                 success=(201, ))
1541         except ClientError as ce:
1542             if ce.status in (202, ):
1543                 raise CLIError(
1544                     'Container %s alread exists' % container, details=[
1545                     'Either delete %s or choose another name' % (container)])
1546             raise
1547
1548     def main(self, new_container):
1549         super(self.__class__, self)._run()
1550         self._run(container=new_container)
1551
1552
1553 @command(container_cmds)
1554 class container_delete(_pithos_account):
1555     """Delete a container"""
1556
1557     arguments = dict(
1558         yes=FlagArgument('Do not prompt for permission', '--yes'),
1559         recursive=FlagArgument(
1560             'delete container even if not empty', ('-r', '--recursive'))
1561     )
1562
1563     @errors.generic.all
1564     @errors.pithos.connection
1565     @errors.pithos.container
1566     def _run(self, container):
1567         num_of_contents = int(self.client.get_container_info(container)[
1568             'x-container-object-count'])
1569         delimiter, msg = None, 'Delete container %s ?' % container
1570         if self['recursive']:
1571             delimiter, msg = '/', 'Empty and d%s' % msg[1:]
1572         elif num_of_contents:
1573             raise CLIError('Container %s is not empty' % container, details=[
1574                 'Use %s to delete non-empty containers' % (
1575                     self.arguments['recursive'].lvalue)])
1576         if self['yes'] or self.ask_user(msg):
1577             if num_of_contents:
1578                 self.client.del_container(delimiter=delimiter)
1579             self.client.purge_container()
1580
1581     def main(self, container):
1582         super(self.__class__, self)._run()
1583         self.container, self.client.container = container, container
1584         self._run(container)
1585
1586
1587 @command(container_cmds)
1588 class container_empty(_pithos_account):
1589     """Empty a container"""
1590
1591     arguments = dict(yes=FlagArgument('Do not prompt for permission', '--yes'))
1592
1593     @errors.generic.all
1594     @errors.pithos.connection
1595     @errors.pithos.container
1596     def _run(self, container):
1597         if self['yes'] or self.ask_user('Empty container %s ?' % container):
1598             self.client.del_container(delimiter='/')
1599
1600     def main(self, container):
1601         super(self.__class__, self)._run()
1602         self.container, self.client.container = container, container
1603         self._run(container)
1604
1605
1606 @command(sharer_cmds)
1607 class sharer_list(_pithos_account, _optional_json):
1608     """List accounts who share file objects with current user"""
1609
1610     arguments = dict(
1611         detail=FlagArgument('show detailed output', ('-l', '--details')),
1612         marker=ValueArgument('show output greater then marker', '--marker')
1613     )
1614
1615     @errors.generic.all
1616     @errors.pithos.connection
1617     def _run(self):
1618         accounts = self.client.get_sharing_accounts(marker=self['marker'])
1619         if not (self['json_output'] or self['output_format']):
1620             usernames = self._uuids2usernames(
1621                 [acc['name'] for acc in accounts])
1622             for item in accounts:
1623                 uuid = item['name']
1624                 item['id'], item['name'] = uuid, usernames[uuid]
1625                 if not self['detail']:
1626                     item.pop('last_modified')
1627         self._print(accounts)
1628
1629     def main(self):
1630         super(self.__class__, self)._run()
1631         self._run()
1632
1633
1634 @command(sharer_cmds)
1635 class sharer_info(_pithos_account, _optional_json):
1636     """Details on a Pithos+ sharer account (default: current account)"""
1637
1638     @errors.generic.all
1639     @errors.pithos.connection
1640     def _run(self):
1641         self._print(self.client.get_account_info(), self.print_dict)
1642
1643     def main(self, account_uuid=None):
1644         super(self.__class__, self)._run()
1645         if account_uuid:
1646             self.client.account, self.account = account_uuid, account_uuid
1647         self._run()
1648
1649
1650 class _pithos_group(_pithos_account):
1651     prefix = 'x-account-group-'
1652     preflen = len(prefix)
1653
1654     def _groups(self):
1655         groups = dict()
1656         for k, v in self.client.get_account_group().items():
1657             groups[k[self.preflen:]] = v
1658         return groups
1659
1660
1661 @command(group_cmds)
1662 class group_list(_pithos_group, _optional_json):
1663     """list all groups and group members"""
1664
1665     @errors.generic.all
1666     @errors.pithos.connection
1667     def _run(self):
1668         self._print(self._groups(), self.print_dict)
1669
1670     def main(self):
1671         super(self.__class__, self)._run()
1672         self._run()
1673
1674
1675 @command(group_cmds)
1676 class group_create(_pithos_group, _optional_json):
1677     """Create a group of users"""
1678
1679     arguments = dict(
1680         user_uuid=RepeatableArgument('Add a user to the group', '--uuid'),
1681         username=RepeatableArgument('Add a user to the group', '--username')
1682     )
1683     required = ['user_uuid', 'username']
1684
1685     @errors.generic.all
1686     @errors.pithos.connection
1687     def _run(self, groupname, *users):
1688         if groupname in self._groups() and not self.ask_user(
1689                 'Group %s already exists, overwrite?' % groupname):
1690             self.error('Aborted')
1691             return
1692         self.client.set_account_group(groupname, users)
1693         self._print(self._groups(), self.print_dict)
1694
1695     def main(self, groupname):
1696         super(self.__class__, self)._run()
1697         users = (self['user_uuid'] or []) + self._usernames2uuids(
1698             self['username'] or []).values()
1699         if users:
1700             self._run(groupname, *users)
1701         else:
1702             raise CLISyntaxError(
1703                 'No valid users specified, use %s or %s' % (
1704                     self.arguments['user_uuid'].lvalue,
1705                     self.arguments['username'].lvalue),
1706                 details=[
1707                     'Check if a username or uuid is valid with',
1708                     '  user uuid2username', 'OR', '  user username2uuid'])
1709
1710
1711 @command(group_cmds)
1712 class group_delete(_pithos_group, _optional_json):
1713     """Delete a user group"""
1714
1715     @errors.generic.all
1716     @errors.pithos.connection
1717     def _run(self, groupname):
1718         self.client.del_account_group(groupname)
1719         self._print(self._groups(), self.print_dict)
1720
1721     def main(self, groupname):
1722         super(self.__class__, self)._run()
1723         self._run(groupname)
1724
1725
1726 #  Deprecated commands
1727
1728 @command(file_cmds)
1729 class file_publish(_pithos_init):
1730     """DEPRECATED, replaced by [kamaki] file modify OBJECT --publish"""
1731
1732     def main(self, *args):
1733         raise CLISyntaxError('DEPRECATED', details=[
1734             'This command is replaced by:',
1735             '  [kamaki] file modify OBJECT --publish'])
1736
1737
1738 @command(file_cmds)
1739 class file_unpublish(_pithos_init):
1740     """DEPRECATED, replaced by [kamaki] file modify OBJECT --unpublish"""
1741
1742     def main(self, *args):
1743         raise CLISyntaxError('DEPRECATED', details=[
1744             'This command is replaced by:',
1745             '  [kamaki] file modify OBJECT --unpublish'])