code.grnet.gr Git - kamaki/blob - kamaki/cli/commands/pithos.py

   1 # Copyright 2011-2013 GRNET S.A. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or
   4 # without modification, are permitted provided that the following
   5 # conditions are met:
   6 #
   7 #   1. Redistributions of source code must retain the above
   8 #      copyright notice, this list of conditions and the following
   9 #      disclaimer.
  10 #
  11 #   2. Redistributions in binary form must reproduce the above
  12 #      copyright notice, this list of conditions and the following
  13 #      disclaimer in the documentation and/or other materials
  14 #      provided with the distribution.
  15 #
  16 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  17 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  23 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  24 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  26 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27 # POSSIBILITY OF SUCH DAMAGE.
  28 #
  29 # The views and conclusions contained in the software and
  30 # documentation are those of the authors and should not be
  31 # interpreted as representing official policies, either expressed
  32 # or implied, of GRNET S.A.command
  33
  34 from time import localtime, strftime
  35 from io import StringIO
  36 from pydoc import pager
  37 from os import path, walk, makedirs
  38
  39 from kamaki.clients.pithos import PithosClient, ClientError
  40
  41 from kamaki.cli import command
  42 from kamaki.cli.command_tree import CommandTree
  43 from kamaki.cli.commands import (
  44     _command_init, errors, addLogSettings, DontRaiseKeyError, _optional_json,
  45     _name_filter, _optional_output_cmd)
  46 from kamaki.cli.errors import (
  47     CLIBaseUrlError, CLIError, CLIInvalidArgument, raiseCLIError)
  48 from kamaki.cli.argument import (
  49     FlagArgument, IntArgument, ValueArgument, DateArgument, KeyValueArgument,
  50     ProgressBarArgument, RepeatableArgument, DataSizeArgument)
  51 from kamaki.cli.utils import (
  52     format_size, bold, get_path_size, guess_mime_type)
  53
  54 file_cmds = CommandTree('file', 'Pithos+/Storage object level API commands')
  55 container_cmds = CommandTree(
  56     'container', 'Pithos+/Storage container level API commands')
  57 sharers_commands = CommandTree('sharers', 'Pithos+/Storage sharers')
  58 _commands = [file_cmds, container_cmds, sharers_commands]
  59
  60
  61 class _pithos_init(_command_init):
  62     """Initilize a pithos+ client
  63     There is always a default account (current user uuid)
  64     There is always a default container (pithos)
  65     """
  66
  67     @DontRaiseKeyError
  68     def _custom_container(self):
  69         return self.config.get_cloud(self.cloud, 'pithos_container')
  70
  71     @DontRaiseKeyError
  72     def _custom_uuid(self):
  73         return self.config.get_cloud(self.cloud, 'pithos_uuid')
  74
  75     def _set_account(self):
  76         self.account = self._custom_uuid()
  77         if self.account:
  78             return
  79         astakos = getattr(self, 'auth_base', None)
  80         if astakos:
  81             self.account = astakos.user_term('id', self.token)
  82         else:
  83             raise CLIBaseUrlError(service='astakos')
  84
  85     @errors.generic.all
  86     @addLogSettings
  87     def _run(self):
  88         cloud = getattr(self, 'cloud', None)
  89         if cloud:
  90             self.base_url = self._custom_url('pithos')
  91         else:
  92             self.cloud = 'default'
  93         self.token = self._custom_token('pithos')
  94         self.container = self._custom_container() or 'pithos'
  95
  96         astakos = getattr(self, 'auth_base', None)
  97         if astakos:
  98             self.token = self.token or astakos.token
  99             if not self.base_url:
 100                 pithos_endpoints = astakos.get_service_endpoints(
 101                     self._custom_type('pithos') or 'object-store',
 102                     self._custom_version('pithos') or '')
 103                 self.base_url = pithos_endpoints['publicURL']
 104         else:
 105             raise CLIBaseUrlError(service='astakos')
 106
 107         self._set_account()
 108         self.client = PithosClient(
 109             self.base_url, self.token, self.account, self.container)
 110
 111     def main(self):
 112         self._run()
 113
 114
 115 class _pithos_account(_pithos_init):
 116     """Setup account"""
 117
 118     def __init__(self, arguments={}, auth_base=None, cloud=None):
 119         super(_pithos_account, self).__init__(arguments, auth_base, cloud)
 120         self['account'] = ValueArgument(
 121             'Use (a different) user uuid', ('-A', '--account'))
 122
 123     def print_objects(self, object_list):
 124         for index, obj in enumerate(object_list):
 125             pretty_obj = obj.copy()
 126             index += 1
 127             empty_space = ' ' * (len(str(len(object_list))) - len(str(index)))
 128             if 'subdir' in obj:
 129                 continue
 130             if self._is_dir(obj):
 131                 size = 'D'
 132             else:
 133                 size = format_size(obj['bytes'])
 134                 pretty_obj['bytes'] = '%s (%s)' % (obj['bytes'], size)
 135             oname = obj['name'] if self['more'] else bold(obj['name'])
 136             prfx = ('%s%s. ' % (empty_space, index)) if self['enum'] else ''
 137             if self['detail']:
 138                 self.writeln('%s%s' % (prfx, oname))
 139                 self.print_dict(pretty_obj, exclude=('name'))
 140                 self.writeln()
 141             else:
 142                 oname = '%s%9s %s' % (prfx, size, oname)
 143                 oname += '/' if self._is_dir(obj) else u''
 144                 self.writeln(oname)
 145
 146     @staticmethod
 147     def _is_dir(remote_dict):
 148         return 'application/directory' == remote_dict.get(
 149             'content_type', remote_dict.get('content-type', ''))
 150
 151     def _run(self):
 152         super(_pithos_account, self)._run()
 153         self.client.account = self['account'] or getattr(
 154             self, 'account', getattr(self.client, 'account', None))
 155
 156
 157 class _pithos_container(_pithos_account):
 158     """Setup container"""
 159
 160     def __init__(self, arguments={}, auth_base=None, cloud=None):
 161         super(_pithos_container, self).__init__(arguments, auth_base, cloud)
 162         self['container'] = ValueArgument(
 163             'Use this container (default: pithos)', ('-C', '--container'))
 164
 165     @staticmethod
 166     def _resolve_pithos_url(url):
 167         """Match urls of one of the following formats:
 168         pithos://ACCOUNT/CONTAINER/OBJECT_PATH
 169         /CONTAINER/OBJECT_PATH
 170         return account, container, path
 171         """
 172         account, container, obj_path, prefix = '', '', url, 'pithos://'
 173         if url.startswith(prefix):
 174             account, sep, url = url[len(prefix):].partition('/')
 175             url = '/%s' % url
 176         if url.startswith('/'):
 177             container, sep, obj_path = url[1:].partition('/')
 178         return account, container, obj_path
 179
 180     def _run(self, url=None):
 181         acc, con, self.path = self._resolve_pithos_url(url or '')
 182         self.account = acc or getattr(self, 'account', '')
 183         super(_pithos_container, self)._run()
 184         self.container = con or self['container'] or getattr(
 185             self, 'container', None) or getattr(self.client, 'container', '')
 186         self.client.container = self.container
 187
 188
 189 @command(file_cmds)
 190 class file_info(_pithos_container, _optional_json):
 191     """Get information/details about a file"""
 192
 193     arguments = dict(
 194         object_version=ValueArgument(
 195             'download a file of a specific version', '--object-version'),
 196         hashmap=FlagArgument(
 197             'Get file hashmap instead of details', '--hashmap'),
 198         matching_etag=ValueArgument(
 199             'show output if ETags match', '--if-match'),
 200         non_matching_etag=ValueArgument(
 201             'show output if ETags DO NOT match', '--if-none-match'),
 202         modified_since_date=DateArgument(
 203             'show output modified since then', '--if-modified-since'),
 204         unmodified_since_date=DateArgument(
 205             'show output unmodified since then', '--if-unmodified-since'),
 206         sharing=FlagArgument(
 207             'show object permissions and sharing information', '--sharing'),
 208         metadata=FlagArgument('show only object metadata', '--metadata'),
 209         versions=FlagArgument(
 210             'show the list of versions for the file', '--object-versions')
 211     )
 212
 213     def version_print(self, versions):
 214         return {'/%s/%s' % (self.container, self.path): [
 215             dict(version_id=vitem[0], created=strftime(
 216                 '%d-%m-%Y %H:%M:%S',
 217                 localtime(float(vitem[1])))) for vitem in versions]}
 218
 219     @errors.generic.all
 220     @errors.pithos.connection
 221     @errors.pithos.container
 222     @errors.pithos.object_path
 223     def _run(self):
 224         if self['hashmap']:
 225             r = self.client.get_object_hashmap(
 226                 self.path,
 227                 version=self['object_version'],
 228                 if_match=self['matching_etag'],
 229                 if_none_match=self['non_matching_etag'],
 230                 if_modified_since=self['modified_since_date'],
 231                 if_unmodified_since=self['unmodified_since_date'])
 232         elif self['sharing']:
 233             r = self.client.get_object_sharing(self.path)
 234             r['public url'] = self.client.get_object_info(
 235                 self.path, version=self['object_version']).get(
 236                     'x-object-public', None)
 237         elif self['metadata']:
 238             r, preflen = dict(), len('x-object-meta-')
 239             for k, v in self.client.get_object_meta(self.path).items():
 240                 r[k[preflen:]] = v
 241         elif self['versions']:
 242             r = self.version_print(
 243                 self.client.get_object_versionlist(self.path))
 244         else:
 245             r = self.client.get_object_info(
 246                 self.path, version=self['object_version'])
 247         self._print(r, self.print_dict)
 248
 249     def main(self, path_or_url):
 250         super(self.__class__, self)._run(path_or_url)
 251         self._run()
 252
 253
 254 @command(file_cmds)
 255 class file_list(_pithos_container, _optional_json, _name_filter):
 256     """List all objects in a container or a directory object"""
 257
 258     arguments = dict(
 259         detail=FlagArgument('detailed output', ('-l', '--list')),
 260         limit=IntArgument('limit number of listed items', ('-n', '--number')),
 261         marker=ValueArgument('output greater that marker', '--marker'),
 262         delimiter=ValueArgument('show output up to delimiter', '--delimiter'),
 263         meta=ValueArgument(
 264             'show output with specified meta keys', '--meta',
 265             default=[]),
 266         if_modified_since=ValueArgument(
 267             'show output modified since then', '--if-modified-since'),
 268         if_unmodified_since=ValueArgument(
 269             'show output not modified since then', '--if-unmodified-since'),
 270         until=DateArgument('show metadata until then', '--until'),
 271         format=ValueArgument(
 272             'format to parse until data (default: d/m/Y H:M:S )', '--format'),
 273         shared=FlagArgument('show only shared', '--shared'),
 274         more=FlagArgument('read long results', '--more'),
 275         enum=FlagArgument('Enumerate results', '--enumerate'),
 276         recursive=FlagArgument(
 277             'Recursively list containers and their contents',
 278             ('-R', '--recursive'))
 279     )
 280
 281     @errors.generic.all
 282     @errors.pithos.connection
 283     @errors.pithos.container
 284     @errors.pithos.object_path
 285     def _run(self):
 286         r = self.client.container_get(
 287             limit=False if self['more'] else self['limit'],
 288             marker=self['marker'],
 289             prefix=self['name_pref'],
 290             delimiter=self['delimiter'],
 291             path=self.path or '',
 292             if_modified_since=self['if_modified_since'],
 293             if_unmodified_since=self['if_unmodified_since'],
 294             until=self['until'],
 295             meta=self['meta'],
 296             show_only_shared=self['shared'])
 297         files = self._filter_by_name(r.json)
 298         if self['more']:
 299             outbu, self._out = self._out, StringIO()
 300         try:
 301             if self['json_output'] or self['output_format']:
 302                 self._print(files)
 303             else:
 304                 self.print_objects(files)
 305         finally:
 306             if self['more']:
 307                 pager(self._out.getvalue())
 308                 self._out = outbu
 309
 310     def main(self, path_or_url=''):
 311         super(self.__class__, self)._run(path_or_url)
 312         self._run()
 313
 314
 315 @command(file_cmds)
 316 class file_modify(_pithos_container):
 317     """Modify the attributes of a file or directory object"""
 318
 319     arguments = dict(
 320         publish=FlagArgument(
 321             'Make an object public (returns the public URL)', '--publish'),
 322         unpublish=FlagArgument(
 323             'Make an object unpublic', '--unpublish'),
 324         uuid_for_read_permission=RepeatableArgument(
 325             'Give read access to user/group (can be repeated, accumulative). '
 326             'Format for users: UUID . Format for groups: UUID:GROUP . '
 327             'Use * for all users/groups', '--read-permission'),
 328         uuid_for_write_permission=RepeatableArgument(
 329             'Give write access to user/group (can be repeated, accumulative). '
 330             'Format for users: UUID . Format for groups: UUID:GROUP . '
 331             'Use * for all users/groups', '--write-permission'),
 332         no_permissions=FlagArgument('Remove permissions', '--no-permissions'),
 333         metadata_to_set=KeyValueArgument(
 334             'Add metadata (KEY=VALUE) to an object (can be repeated)',
 335             '--metadata-add'),
 336         metadata_key_to_delete=RepeatableArgument(
 337             'Delete object metadata (can be repeated)', '--metadata-del'),
 338     )
 339     required = [
 340         'publish', 'unpublish', 'uuid_for_read_permission', 'metadata_to_set',
 341         'uuid_for_write_permission', 'no_permissions',
 342         'metadata_key_to_delete']
 343
 344     @errors.generic.all
 345     @errors.pithos.connection
 346     @errors.pithos.container
 347     @errors.pithos.object_path
 348     def _run(self):
 349         if self['publish']:
 350             self.writeln(self.client.publish_object(self.path))
 351         if self['unpublish']:
 352             self.client.unpublish_object(self.path)
 353         if self['uuid_for_read_permission'] or self[
 354                 'uuid_for_write_permission']:
 355             perms = self.client.get_object_sharing(self.path)
 356             read, write = perms.get('read', ''), perms.get('write', '')
 357             read = read.split(',') if read else []
 358             write = write.split(',') if write else []
 359             read += self['uuid_for_read_permission']
 360             write += self['uuid_for_write_permission']
 361             self.client.set_object_sharing(
 362                 self.path, read_permission=read, write_permission=write)
 363             self.print_dict(self.client.get_object_sharing(self.path))
 364         if self['no_permissions']:
 365             self.client.del_object_sharing(self.path)
 366         metadata = self['metadata_to_set'] or dict()
 367         for k in self['metadata_key_to_delete']:
 368             metadata[k] = ''
 369         if metadata:
 370             self.client.set_object_meta(self.path, metadata)
 371             self.print_dict(self.client.get_object_meta(self.path))
 372
 373     def main(self, path_or_url):
 374         super(self.__class__, self)._run(path_or_url)
 375         if self['publish'] and self['unpublish']:
 376             raise CLIInvalidArgument(
 377                 'Arguments %s and %s cannot be used together' % (
 378                     '/'.join(self.arguments['publish'].parsed_name),
 379                     '/'.join(self.arguments['publish'].parsed_name)))
 380         if self['no_permissions'] and (
 381                 self['uuid_for_read_permission'] or self[
 382                     'uuid_for_write_permission']):
 383             raise CLIInvalidArgument(
 384                 '%s cannot be used with other permission arguments' % '/'.join(
 385                     self.arguments['no_permissions'].parsed_name))
 386         self._run()
 387
 388
 389 @command(file_cmds)
 390 class file_create(_pithos_container, _optional_output_cmd):
 391     """Create an empty file"""
 392
 393     arguments = dict(
 394         content_type=ValueArgument(
 395             'Set content type (default: application/octet-stream)',
 396             '--content-type',
 397             default='application/octet-stream')
 398     )
 399
 400     @errors.generic.all
 401     @errors.pithos.connection
 402     @errors.pithos.container
 403     def _run(self):
 404         self._optional_output(
 405             self.client.create_object(self.path, self['content_type']))
 406
 407     def main(self, path_or_url):
 408         super(self.__class__, self)._run(path_or_url)
 409         self._run()
 410
 411
 412 @command(file_cmds)
 413 class file_mkdir(_pithos_container, _optional_output_cmd):
 414     """Create a directory: /file create --content-type='applcation/directory'
 415     """
 416
 417     @errors.generic.all
 418     @errors.pithos.connection
 419     @errors.pithos.container
 420     def _run(self):
 421         self._optional_output(self.client.create_directory(self.path))
 422
 423     def main(self, path_or_url):
 424         super(self.__class__, self)._run(path_or_url)
 425         self._run()
 426
 427
 428 @command(file_cmds)
 429 class file_delete(_pithos_container):
 430     """Delete a file or directory object"""
 431
 432     arguments = dict(
 433         until_date=DateArgument('remove history until then', '--until'),
 434         yes=FlagArgument('Do not prompt for permission', '--yes'),
 435         recursive=FlagArgument(
 436             'If a directory, empty first', ('-r', '--recursive')),
 437         delimiter=ValueArgument(
 438             'delete objects prefixed with <object><delimiter>', '--delimiter')
 439     )
 440
 441     @errors.generic.all
 442     @errors.pithos.connection
 443     @errors.pithos.container
 444     @errors.pithos.object_path
 445     def _run(self):
 446         if self.path:
 447             if self['yes'] or self.ask_user(
 448                     'Delete /%s/%s ?' % (self.container, self.path)):
 449                 self.client.del_object(
 450                     self.path,
 451                     until=self['until_date'],
 452                     delimiter='/' if self['recursive'] else self['delimiter'])
 453             else:
 454                 self.error('Aborted')
 455         else:
 456             if self['yes'] or self.ask_user(
 457                     'Empty container /%s ?' % self.container):
 458                 self.client.container_delete(self.container, delimiter='/')
 459             else:
 460                 self.error('Aborted')
 461
 462     def main(self, path_or_url):
 463         super(self.__class__, self)._run(path_or_url)
 464         self._run()
 465
 466
 467 class _source_destination(_pithos_container, _optional_output_cmd):
 468
 469     sd_arguments = dict(
 470         destination_user_uuid=ValueArgument(
 471             'default: current user uuid', '--to-account'),
 472         destination_container=ValueArgument(
 473             'default: pithos', '--to-container'),
 474         source_prefix=FlagArgument(
 475             'Transfer all files that are prefixed with SOURCE PATH If the '
 476             'destination path is specified, replace SOURCE_PATH with '
 477             'DESTINATION_PATH',
 478             ('-r', '--recursive')),
 479         force=FlagArgument(
 480             'Overwrite destination objects, if needed', ('-f', '--force')),
 481         source_version=ValueArgument(
 482             'The version of the source object', '--source-version')
 483     )
 484
 485     def __init__(self, arguments={}, auth_base=None, cloud=None):
 486         self.arguments.update(arguments)
 487         self.arguments.update(self.sd_arguments)
 488         super(_source_destination, self).__init__(
 489             self.arguments, auth_base, cloud)
 490
 491     def _report_transfer(self, src, dst, transfer_name):
 492         if not dst:
 493             if transfer_name in ('move', ):
 494                 self.error('  delete source directory %s' % src)
 495             return
 496         dst_prf = '' if self.account == self.dst_client.account else (
 497                 'pithos://%s' % self.dst_client.account)
 498         if src:
 499             src_prf = '' if self.account == self.dst_client.account else (
 500                     'pithos://%s' % self.account)
 501             self.error('  %s %s/%s/%s\n  -->  %s/%s/%s' % (
 502                 transfer_name,
 503                 src_prf, self.container, src,
 504                 dst_prf, self.dst_client.container, dst))
 505         else:
 506             self.error('  mkdir %s/%s/%s' % (
 507                 dst_prf, self.dst_client.container, dst))
 508
 509     @errors.generic.all
 510     @errors.pithos.account
 511     def _src_dst(self, version=None):
 512         """Preconditions:
 513         self.account, self.container, self.path
 514         self.dst_acc, self.dst_con, self.dst_path
 515         They should all be configured properly
 516         :returns: [(src_path, dst_path), ...], if src_path is None, create
 517             destination directory
 518         """
 519         src_objects, dst_objects, pairs = dict(), dict(), []
 520         try:
 521             for obj in self.dst_client.list_objects(
 522                     prefix=self.dst_path or self.path or '/'):
 523                 dst_objects[obj['name']] = obj
 524         except ClientError as ce:
 525             if ce.status in (404, ):
 526                 raise CLIError(
 527                     'Destination container pithos://%s/%s not found' % (
 528                         self.dst_client.account, self.dst_client.container))
 529             raise ce
 530         if self['source_prefix']:
 531             #  Copy and replace prefixes
 532             for src_obj in self.client.list_objects(prefix=self.path):
 533                 src_objects[src_obj['name']] = src_obj
 534             for src_path, src_obj in src_objects.items():
 535                 dst_path = '%s%s' % (
 536                     self.dst_path or self.path, src_path[len(self.path):])
 537                 dst_obj = dst_objects.get(dst_path, None)
 538                 if self['force'] or not dst_obj:
 539                     #  Just do it
 540                     pairs.append((
 541                         None if self._is_dir(src_obj) else src_path, dst_path))
 542                     if self._is_dir(src_obj):
 543                         pairs.append((self.path or dst_path, None))
 544                 elif not (self._is_dir(dst_obj) and self._is_dir(src_obj)):
 545                     raise CLIError(
 546                         'Destination object exists', importance=2, details=[
 547                             'Failed while transfering:',
 548                             '    pithos://%s/%s/%s' % (
 549                                     self.account,
 550                                     self.container,
 551                                     src_path),
 552                             '--> pithos://%s/%s/%s' % (
 553                                     self.dst_client.account,
 554                                     self.dst_client.container,
 555                                     dst_path),
 556                             'Use %s to transfer overwrite' % ('/'.join(
 557                                     self.arguments['force'].parsed_name))])
 558         else:
 559             #  One object transfer
 560             try:
 561                 src_version_arg = self.arguments.get('source_version', None)
 562                 src_obj = self.client.get_object_info(
 563                     self.path,
 564                     version=src_version_arg.value if src_version_arg else None)
 565             except ClientError as ce:
 566                 if ce.status in (204, ):
 567                     raise CLIError(
 568                         'Missing specific path container %s' % self.container,
 569                         importance=2, details=[
 570                             'To transfer container contents %s' % (
 571                                 '/'.join(self.arguments[
 572                                     'source_prefix'].parsed_name))])
 573                 raise
 574             dst_path = self.dst_path or self.path
 575             dst_obj = dst_objects.get(dst_path or self.path, None)
 576             if self['force'] or not dst_obj:
 577                 pairs.append(
 578                     (None if self._is_dir(src_obj) else self.path, dst_path))
 579                 if self._is_dir(src_obj):
 580                     pairs.append((self.path or dst_path, None))
 581             elif self._is_dir(src_obj):
 582                 raise CLIError(
 583                     'Cannot transfer an application/directory object',
 584                     importance=2, details=[
 585                         'The object pithos://%s/%s/%s is a directory' % (
 586                             self.account,
 587                             self.container,
 588                             self.path),
 589                         'To recursively copy a directory, use',
 590                         '  %s' % ('/'.join(
 591                             self.arguments['source_prefix'].parsed_name)),
 592                         'To create a file, use',
 593                         '  /file create  (general purpose)',
 594                         '  /file mkdir   (a directory object)'])
 595             else:
 596                 raise CLIError(
 597                     'Destination object exists',
 598                     importance=2, details=[
 599                         'Failed while transfering:',
 600                         '    pithos://%s/%s/%s' % (
 601                                 self.account,
 602                                 self.container,
 603                                 self.path),
 604                         '--> pithos://%s/%s/%s' % (
 605                                 self.dst_client.account,
 606                                 self.dst_client.container,
 607                                 dst_path),
 608                         'Use %s to transfer overwrite' % ('/'.join(
 609                                 self.arguments['force'].parsed_name))])
 610         return pairs
 611
 612     def _run(self, source_path_or_url, destination_path_or_url=''):
 613         super(_source_destination, self)._run(source_path_or_url)
 614         dst_acc, dst_con, dst_path = self._resolve_pithos_url(
 615             destination_path_or_url)
 616         self.dst_client = PithosClient(
 617             base_url=self.client.base_url, token=self.client.token,
 618             container=self[
 619                 'destination_container'] or dst_con or self.client.container,
 620             account=self[
 621                 'destination_user_uuid'] or dst_acc or self.client.account)
 622         self.dst_path = dst_path or self.path
 623
 624
 625 @command(file_cmds)
 626 class file_copy(_source_destination):
 627     """Copy objects, even between different accounts or containers"""
 628
 629     arguments = dict(
 630         public=ValueArgument('publish new object', '--public'),
 631         content_type=ValueArgument(
 632             'change object\'s content type', '--content-type'),
 633         source_version=ValueArgument(
 634             'The version of the source object', '--object-version')
 635     )
 636
 637     @errors.generic.all
 638     @errors.pithos.connection
 639     @errors.pithos.container
 640     @errors.pithos.account
 641     def _run(self):
 642         for src, dst in self._src_dst(self['source_version']):
 643             self._report_transfer(src, dst, 'copy')
 644             if src and dst:
 645                 self.dst_client.copy_object(
 646                     src_container=self.client.container,
 647                     src_object=src,
 648                     dst_container=self.dst_client.container,
 649                     dst_object=dst,
 650                     source_account=self.account,
 651                     source_version=self['source_version'],
 652                     public=self['public'],
 653                     content_type=self['content_type'])
 654             elif dst:
 655                 self.dst_client.create_directory(dst)
 656
 657     def main(self, source_path_or_url, destination_path_or_url=None):
 658         super(file_copy, self)._run(
 659             source_path_or_url, destination_path_or_url or '')
 660         self._run()
 661
 662
 663 @command(file_cmds)
 664 class file_move(_source_destination):
 665     """Move objects, even between different accounts or containers"""
 666
 667     arguments = dict(
 668         public=ValueArgument('publish new object', '--public'),
 669         content_type=ValueArgument(
 670             'change object\'s content type', '--content-type')
 671     )
 672
 673     @errors.generic.all
 674     @errors.pithos.connection
 675     @errors.pithos.container
 676     @errors.pithos.account
 677     def _run(self):
 678         for src, dst in self._src_dst():
 679             self._report_transfer(src, dst, 'move')
 680             if src and dst:
 681                 self.dst_client.move_object(
 682                     src_container=self.client.container,
 683                     src_object=src,
 684                     dst_container=self.dst_client.container,
 685                     dst_object=dst,
 686                     source_account=self.account,
 687                     public=self['public'],
 688                     content_type=self['content_type'])
 689             elif dst:
 690                 self.dst_client.create_directory(dst)
 691             else:
 692                 self.client.del_object(src)
 693
 694     def main(self, source_path_or_url, destination_path_or_url=None):
 695         super(file_move, self)._run(
 696             source_path_or_url, destination_path_or_url or '')
 697         self._run()
 698
 699
 700 @command(file_cmds)
 701 class file_append(_pithos_container, _optional_output_cmd):
 702     """Append local file to (existing) remote object
 703     The remote object should exist.
 704     If the remote object is a directory, it is transformed into a file.
 705     In the later case, objects under the directory remain intact.
 706     """
 707
 708     arguments = dict(
 709         progress_bar=ProgressBarArgument(
 710             'do not show progress bar', ('-N', '--no-progress-bar'),
 711             default=False),
 712         max_threads=IntArgument('default: 1', '--threads'),
 713     )
 714
 715     @errors.generic.all
 716     @errors.pithos.connection
 717     @errors.pithos.container
 718     @errors.pithos.object_path
 719     def _run(self, local_path):
 720         if self['max_threads'] > 0:
 721             self.client.MAX_THREADS = int(self['max_threads'])
 722         (progress_bar, upload_cb) = self._safe_progress_bar('Appending')
 723         try:
 724             with open(local_path, 'rb') as f:
 725                 self._optional_output(
 726                     self.client.append_object(self.path, f, upload_cb))
 727         finally:
 728             self._safe_progress_bar_finish(progress_bar)
 729
 730     def main(self, local_path, remote_path_or_url):
 731         super(self.__class__, self)._run(remote_path_or_url)
 732         self._run(local_path)
 733
 734
 735 @command(file_cmds)
 736 class file_truncate(_pithos_container, _optional_output_cmd):
 737     """Truncate remote file up to size"""
 738
 739     arguments = dict(
 740         size_in_bytes=IntArgument('Length of file after truncation', '--size')
 741     )
 742     required = ('size_in_bytes', )
 743
 744     @errors.generic.all
 745     @errors.pithos.connection
 746     @errors.pithos.container
 747     @errors.pithos.object_path
 748     @errors.pithos.object_size
 749     def _run(self, size):
 750         self._optional_output(self.client.truncate_object(self.path, size))
 751
 752     def main(self, path_or_url):
 753         super(self.__class__, self)._run(path_or_url)
 754         self._run(size=self['size_in_bytes'])
 755
 756
 757 @command(file_cmds)
 758 class file_overwrite(_pithos_container, _optional_output_cmd):
 759     """Overwrite part of a remote file"""
 760
 761     arguments = dict(
 762         progress_bar=ProgressBarArgument(
 763             'do not show progress bar', ('-N', '--no-progress-bar'),
 764             default=False),
 765         start_position=IntArgument('File position in bytes', '--from'),
 766         end_position=IntArgument('File position in bytes', '--to')
 767     )
 768     required = ('start_position', 'end_position')
 769
 770     @errors.generic.all
 771     @errors.pithos.connection
 772     @errors.pithos.container
 773     @errors.pithos.object_path
 774     @errors.pithos.object_size
 775     def _run(self, local_path, start, end):
 776         start, end = int(start), int(end)
 777         (progress_bar, upload_cb) = self._safe_progress_bar(
 778             'Overwrite %s bytes' % (end - start))
 779         try:
 780             with open(path.abspath(local_path), 'rb') as f:
 781                 self._optional_output(self.client.overwrite_object(
 782                     obj=self.path,
 783                     start=start,
 784                     end=end,
 785                     source_file=f,
 786                     upload_cb=upload_cb))
 787         finally:
 788             self._safe_progress_bar_finish(progress_bar)
 789
 790     def main(self, local_path, path_or_url):
 791         super(self.__class__, self)._run(path_or_url)
 792         self.path = self.path or path.basename(local_path)
 793         self._run(
 794             local_path=local_path,
 795             start=self['start_position'],
 796             end=self['end_position'])
 797
 798
 799 @command(file_cmds)
 800 class file_upload(_pithos_container, _optional_output_cmd):
 801     """Upload a file"""
 802
 803     arguments = dict(
 804         max_threads=IntArgument('default: 5', '--threads'),
 805         content_encoding=ValueArgument(
 806             'set MIME content type', '--content-encoding'),
 807         content_disposition=ValueArgument(
 808             'specify objects presentation style', '--content-disposition'),
 809         content_type=ValueArgument('specify content type', '--content-type'),
 810         uuid_for_read_permission=RepeatableArgument(
 811             'Give read access to a user or group (can be repeated) '
 812             'Use * for all users',
 813             '--read-permission'),
 814         uuid_for_write_permission=RepeatableArgument(
 815             'Give write access to a user or group (can be repeated) '
 816             'Use * for all users',
 817             '--write-permission'),
 818         public=FlagArgument('make object publicly accessible', '--public'),
 819         overwrite=FlagArgument('Force (over)write', ('-f', '--force')),
 820         recursive=FlagArgument(
 821             'Recursively upload directory *contents* + subdirectories',
 822             ('-r', '--recursive')),
 823         unchunked=FlagArgument(
 824             'Upload file as one block (not recommended)', '--unchunked'),
 825         md5_checksum=ValueArgument(
 826             'Confirm upload with a custom checksum (MD5)', '--etag'),
 827         use_hashes=FlagArgument(
 828             'Source file contains hashmap not data', '--source-is-hashmap'),
 829     )
 830
 831     def _sharing(self):
 832         sharing = dict()
 833         readlist = self['uuid_for_read_permission']
 834         if readlist:
 835             sharing['read'] = self['uuid_for_read_permission']
 836         writelist = self['uuid_for_write_permission']
 837         if writelist:
 838             sharing['write'] = self['uuid_for_write_permission']
 839         return sharing or None
 840
 841     def _check_container_limit(self, path):
 842         cl_dict = self.client.get_container_limit()
 843         container_limit = int(cl_dict['x-container-policy-quota'])
 844         r = self.client.container_get()
 845         used_bytes = sum(int(o['bytes']) for o in r.json)
 846         path_size = get_path_size(path)
 847         if container_limit and path_size > (container_limit - used_bytes):
 848             raise CLIError(
 849                 'Container %s (limit(%s) - used(%s)) < (size(%s) of %s)' % (
 850                     self.client.container,
 851                     format_size(container_limit),
 852                     format_size(used_bytes),
 853                     format_size(path_size),
 854                     path),
 855                 details=[
 856                     'Check accound limit: /file quota',
 857                     'Check container limit:',
 858                     '\t/file containerlimit get %s' % self.client.container,
 859                     'Increase container limit:',
 860                     '\t/file containerlimit set <new limit> %s' % (
 861                         self.client.container)])
 862
 863     def _src_dst(self, local_path, remote_path, objlist=None):
 864         lpath = path.abspath(local_path)
 865         short_path = path.basename(path.abspath(local_path))
 866         rpath = remote_path or short_path
 867         if path.isdir(lpath):
 868             if not self['recursive']:
 869                 raise CLIError('%s is a directory' % lpath, details=[
 870                     'Use %s to upload directories & contents' % '/'.join(
 871                         self.arguments['recursive'].parsed_name)])
 872             robj = self.client.container_get(path=rpath)
 873             if not self['overwrite']:
 874                 if robj.json:
 875                     raise CLIError(
 876                         'Objects/files prefixed as %s already exist' % rpath,
 877                         details=['Existing objects:'] + ['\t/%s/\t%s' % (
 878                             o['name'],
 879                             o['content_type'][12:]) for o in robj.json] + [
 880                             'Use -f to add, overwrite or resume'])
 881                 else:
 882                     try:
 883                         topobj = self.client.get_object_info(rpath)
 884                         if not self._is_dir(topobj):
 885                             raise CLIError(
 886                                 'Object /%s/%s exists but not a directory' % (
 887                                     self.container, rpath),
 888                                 details=['Use -f to overwrite'])
 889                     except ClientError as ce:
 890                         if ce.status not in (404, ):
 891                             raise
 892             self._check_container_limit(lpath)
 893             prev = ''
 894             for top, subdirs, files in walk(lpath):
 895                 if top != prev:
 896                     prev = top
 897                     try:
 898                         rel_path = rpath + top.split(lpath)[1]
 899                     except IndexError:
 900                         rel_path = rpath
 901                     self.error('mkdir /%s/%s' % (
 902                         self.client.container, rel_path))
 903                     self.client.create_directory(rel_path)
 904                 for f in files:
 905                     fpath = path.join(top, f)
 906                     if path.isfile(fpath):
 907                         rel_path = rel_path.replace(path.sep, '/')
 908                         pathfix = f.replace(path.sep, '/')
 909                         yield open(fpath, 'rb'), '%s/%s' % (rel_path, pathfix)
 910                     else:
 911                         self.error('%s is not a regular file' % fpath)
 912         else:
 913             if not path.isfile(lpath):
 914                 raise CLIError(('%s is not a regular file' % lpath) if (
 915                     path.exists(lpath)) else '%s does not exist' % lpath)
 916             try:
 917                 robj = self.client.get_object_info(rpath)
 918                 if remote_path and self._is_dir(robj):
 919                     rpath += '/%s' % (short_path.replace(path.sep, '/'))
 920                     self.client.get_object_info(rpath)
 921                 if not self['overwrite']:
 922                     raise CLIError(
 923                         'Object /%s/%s already exists' % (
 924                             self.container, rpath),
 925                         details=['use -f to overwrite / resume'])
 926             except ClientError as ce:
 927                 if ce.status not in (404, ):
 928                     raise
 929             self._check_container_limit(lpath)
 930             yield open(lpath, 'rb'), rpath
 931
 932     def _run(self, local_path, remote_path):
 933         if self['max_threads'] > 0:
 934             self.client.MAX_THREADS = int(self['max_threads'])
 935         params = dict(
 936             content_encoding=self['content_encoding'],
 937             content_type=self['content_type'],
 938             content_disposition=self['content_disposition'],
 939             sharing=self._sharing(),
 940             public=self['public'])
 941         uploaded, container_info_cache = list, dict()
 942         rpref = 'pithos://%s' if self['account'] else ''
 943         for f, rpath in self._src_dst(local_path, remote_path):
 944             self.error('%s --> %s/%s/%s' % (
 945                 f.name, rpref, self.client.container, rpath))
 946             if not (self['content_type'] and self['content_encoding']):
 947                 ctype, cenc = guess_mime_type(f.name)
 948                 params['content_type'] = self['content_type'] or ctype
 949                 params['content_encoding'] = self['content_encoding'] or cenc
 950             if self['unchunked']:
 951                 r = self.client.upload_object_unchunked(
 952                     rpath, f,
 953                     etag=self['md5_checksum'], withHashFile=self['use_hashes'],
 954                     **params)
 955                 if self['with_output'] or self['json_output']:
 956                     r['name'] = '/%s/%s' % (self.client.container, rpath)
 957                     uploaded.append(r)
 958             else:
 959                 try:
 960                     (progress_bar, upload_cb) = self._safe_progress_bar(
 961                         'Uploading %s' % f.name.split(path.sep)[-1])
 962                     if progress_bar:
 963                         hash_bar = progress_bar.clone()
 964                         hash_cb = hash_bar.get_generator(
 965                             'Calculating block hashes')
 966                     else:
 967                         hash_cb = None
 968                     r = self.client.upload_object(
 969                         rpath, f,
 970                         hash_cb=hash_cb,
 971                         upload_cb=upload_cb,
 972                         container_info_cache=container_info_cache,
 973                         **params)
 974                     if self['with_output'] or self['json_output']:
 975                         r['name'] = '/%s/%s' % (self.client.container, rpath)
 976                         uploaded.append(r)
 977                 except Exception:
 978                     self._safe_progress_bar_finish(progress_bar)
 979                     raise
 980                 finally:
 981                     self._safe_progress_bar_finish(progress_bar)
 982         self._optional_output(uploaded)
 983         self.error('Upload completed')
 984
 985     def main(self, local_path, remote_path_or_url):
 986         super(self.__class__, self)._run(remote_path_or_url)
 987         remote_path = self.path or path.basename(path.abspath(local_path))
 988         self._run(local_path=local_path, remote_path=remote_path)
 989
 990
 991 class RangeArgument(ValueArgument):
 992     """
 993     :value type: string of the form <start>-<end> where <start> and <end> are
 994         integers
 995     :value returns: the input string, after type checking <start> and <end>
 996     """
 997
 998     @property
 999     def value(self):
1000         return getattr(self, '_value', self.default)
1001
1002     @value.setter
1003     def value(self, newvalues):
1004         if newvalues:
1005             self._value = getattr(self, '_value', self.default)
1006             for newvalue in newvalues.split(','):
1007                 self._value = ('%s,' % self._value) if self._value else ''
1008                 start, sep, end = newvalue.partition('-')
1009                 if sep:
1010                     if start:
1011                         start, end = (int(start), int(end))
1012                         if start > end:
1013                             raise CLIInvalidArgument(
1014                                 'Invalid range %s' % newvalue, details=[
1015                                 'Valid range formats',
1016                                 '  START-END', '  UP_TO', '  -FROM',
1017                                 'where all values are integers'])
1018                         self._value += '%s-%s' % (start, end)
1019                     else:
1020                         self._value += '-%s' % int(end)
1021                 else:
1022                     self._value += '%s' % int(start)
1023
1024
1025 @command(file_cmds)
1026 class file_cat(_pithos_container):
1027     """Fetch remote file contents"""
1028
1029     arguments = dict(
1030         range=RangeArgument('show range of data', '--range'),
1031         if_match=ValueArgument('show output if ETags match', '--if-match'),
1032         if_none_match=ValueArgument(
1033             'show output if ETags match', '--if-none-match'),
1034         if_modified_since=DateArgument(
1035             'show output modified since then', '--if-modified-since'),
1036         if_unmodified_since=DateArgument(
1037             'show output unmodified since then', '--if-unmodified-since'),
1038         object_version=ValueArgument(
1039             'Get contents of the chosen version', '--object-version')
1040     )
1041
1042     @errors.generic.all
1043     @errors.pithos.connection
1044     @errors.pithos.container
1045     @errors.pithos.object_path
1046     def _run(self):
1047         self.client.download_object(
1048             self.path, self._out,
1049             range_str=self['range'],
1050             version=self['object_version'],
1051             if_match=self['if_match'],
1052             if_none_match=self['if_none_match'],
1053             if_modified_since=self['if_modified_since'],
1054             if_unmodified_since=self['if_unmodified_since'])
1055
1056     def main(self, path_or_url):
1057         super(self.__class__, self)._run(path_or_url)
1058         self._run()
1059
1060
1061 @command(file_cmds)
1062 class file_download(_pithos_container):
1063     """Download a remove file or directory object to local file system"""
1064
1065     arguments = dict(
1066         resume=FlagArgument(
1067             'Resume/Overwrite (attempt resume, else overwrite)',
1068             ('-f', '--resume')),
1069         range=RangeArgument('Download only that range of data', '--range'),
1070         matching_etag=ValueArgument('download iff ETag match', '--if-match'),
1071         non_matching_etag=ValueArgument(
1072             'download iff ETags DO NOT match', '--if-none-match'),
1073         modified_since_date=DateArgument(
1074             'download iff remote file is modified since then',
1075             '--if-modified-since'),
1076         unmodified_since_date=DateArgument(
1077             'show output iff remote file is unmodified since then',
1078             '--if-unmodified-since'),
1079         object_version=ValueArgument(
1080             'download a file of a specific version', '--object-version'),
1081         max_threads=IntArgument('default: 5', '--threads'),
1082         progress_bar=ProgressBarArgument(
1083             'do not show progress bar', ('-N', '--no-progress-bar'),
1084             default=False),
1085         recursive=FlagArgument(
1086             'Download a remote directory object and its contents',
1087             ('-r', '--recursive'))
1088         )
1089
1090     def _src_dst(self, local_path):
1091         """Create a list of (src, dst) where src is a remote location and dst
1092         is an open file descriptor. Directories are denoted as (None, dirpath)
1093         and they are pretended to other objects in a very strict order (shorter
1094         to longer path)."""
1095         ret = []
1096         try:
1097             if self.path:
1098                 obj = self.client.get_object_info(
1099                     self.path, version=self['object_version'])
1100                 obj.setdefault('name', self.path.strip('/'))
1101             else:
1102                 obj = None
1103         except ClientError as ce:
1104             if ce.status in (404, ):
1105                 raiseCLIError(ce, details=[
1106                     'To download an object, it must exist either as a file or'
1107                     ' as a directory.',
1108                     'For example, to download everything under prefix/ the '
1109                     'directory "prefix" must exist.',
1110                     'To see if an remote object is actually there:',
1111                     '  /file info [/CONTAINER/]OBJECT',
1112                     'To create a directory object:',
1113                     '  /file mkdir [/CONTAINER/]OBJECT'])
1114             if ce.status in (204, ):
1115                 raise CLIError(
1116                     'No file or directory objects to download',
1117                     details=[
1118                         'To download a container (e.g., %s):' % self.container,
1119                         '  [kamaki] container download %s [LOCAL_PATH]' % (
1120                             self.container)])
1121             raise
1122         rpath = self.path.strip('/')
1123         if local_path and self.path and local_path.endswith('/'):
1124             local_path = local_path[-1:]
1125
1126         if (not obj) or self._is_dir(obj):
1127             if self['recursive']:
1128                 if not (self.path or local_path.endswith('/')):
1129                     #  Download the whole container
1130                     local_path = '' if local_path in ('.', ) else local_path
1131                     local_path = '%s/' % (local_path or self.container)
1132                 obj = obj or dict(
1133                     name='', content_type='application/directory')
1134                 dirs, files = [obj, ], []
1135                 objects = self.client.container_get(
1136                     path=self.path,
1137                     if_modified_since=self['modified_since_date'],
1138                     if_unmodified_since=self['unmodified_since_date'])
1139                 for o in objects.json:
1140                     (dirs if self._is_dir(o) else files).append(o)
1141
1142                 #  Put the directories on top of the list
1143                 for dpath in sorted(['%s%s' % (
1144                         local_path, d['name'][len(rpath):]) for d in dirs]):
1145                     if path.exists(dpath):
1146                         if path.isdir(dpath):
1147                             continue
1148                         raise CLIError(
1149                             'Cannot replace local file %s with a directory '
1150                             'of the same name' % dpath,
1151                             details=[
1152                                 'Either remove the file or specify a'
1153                                 'different target location'])
1154                     ret.append((None, dpath, None))
1155
1156                 #  Append the file objects
1157                 for opath in [o['name'] for o in files]:
1158                     lpath = '%s%s' % (local_path, opath[len(rpath):])
1159                     if self['resume']:
1160                         fxists = path.exists(lpath)
1161                         if fxists and path.isdir(lpath):
1162                             raise CLIError(
1163                                 'Cannot change local dir %s info file' % (
1164                                     lpath),
1165                                 details=[
1166                                     'Either remove the file or specify a'
1167                                     'different target location'])
1168                         ret.append((opath, lpath, fxists))
1169                     elif path.exists(lpath):
1170                         raise CLIError(
1171                             'Cannot overwrite %s' % lpath,
1172                             details=['To overwrite/resume, use  %s' % '/'.join(
1173                                 self.arguments['resume'].parsed_name)])
1174                     else:
1175                         ret.append((opath, lpath, None))
1176             elif self.path:
1177                 raise CLIError(
1178                     'Remote object /%s/%s is a directory' % (
1179                         self.container, local_path),
1180                     details=['Use %s to download directories' % '/'.join(
1181                         self.arguments['recursive'].parsed_name)])
1182             else:
1183                 parsed_name = '/'.join(self.arguments['recursive'].parsed_name)
1184                 raise CLIError(
1185                     'Cannot download container %s' % self.container,
1186                     details=[
1187                         'Use %s to download containers' % parsed_name,
1188                         '  [kamaki] file download %s /%s [LOCAL_PATH]' % (
1189                             parsed_name, self.container)])
1190         else:
1191             #  Remote object is just a file
1192             if path.exists(local_path) and not self['resume']:
1193                 raise CLIError(
1194                     'Cannot overwrite local file %s' % (lpath),
1195                     details=['To overwrite/resume, use  %s' % '/'.join(
1196                         self.arguments['resume'].parsed_name)])
1197             ret.append((rpath, local_path, self['resume']))
1198         for r, l, resume in ret:
1199             if r:
1200                 with open(l, 'rwb+' if resume else 'wb+') as f:
1201                     yield (r, f)
1202             else:
1203                 yield (r, l)
1204
1205     @errors.generic.all
1206     @errors.pithos.connection
1207     @errors.pithos.container
1208     @errors.pithos.object_path
1209     @errors.pithos.local_path
1210     @errors.pithos.local_path_download
1211     def _run(self, local_path):
1212         self.client.MAX_THREADS = self['max_threads'] or 5
1213         progress_bar = None
1214         try:
1215             for rpath, output_file in self._src_dst(local_path):
1216                 if not rpath:
1217                     self.error('Create local directory %s' % output_file)
1218                     makedirs(output_file)
1219                     continue
1220                 self.error('/%s/%s --> %s' % (
1221                     self.container, rpath, output_file.name))
1222                 progress_bar, download_cb = self._safe_progress_bar(
1223                     '  download')
1224                 self.client.download_object(
1225                     rpath, output_file,
1226                     download_cb=download_cb,
1227                     range_str=self['range'],
1228                     version=self['object_version'],
1229                     if_match=self['matching_etag'],
1230                     resume=self['resume'],
1231                     if_none_match=self['non_matching_etag'],
1232                     if_modified_since=self['modified_since_date'],
1233                     if_unmodified_since=self['unmodified_since_date'])
1234         except KeyboardInterrupt:
1235             from threading import activeCount, enumerate as activethreads
1236             timeout = 0.5
1237             while activeCount() > 1:
1238                 self._out.write('\nCancel %s threads: ' % (activeCount() - 1))
1239                 self._out.flush()
1240                 for thread in activethreads():
1241                     try:
1242                         thread.join(timeout)
1243                         self._out.write('.' if thread.isAlive() else '*')
1244                     except RuntimeError:
1245                         continue
1246                     finally:
1247                         self._out.flush()
1248                         timeout += 0.1
1249             self.error('\nDownload canceled by user')
1250             if local_path is not None:
1251                 self.error('to resume, re-run with --resume')
1252         except Exception:
1253             self._safe_progress_bar_finish(progress_bar)
1254             raise
1255         finally:
1256             self._safe_progress_bar_finish(progress_bar)
1257
1258     def main(self, remote_path_or_url, local_path=None):
1259         super(self.__class__, self)._run(remote_path_or_url)
1260         local_path = local_path or self.path or '.'
1261         self._run(local_path=local_path)
1262
1263
1264 @command(container_cmds)
1265 class container_info(_pithos_account, _optional_json):
1266     """Get information about a container"""
1267
1268     arguments = dict(
1269         until_date=DateArgument('show metadata until then', '--until'),
1270         metadata=FlagArgument('Show only container metadata', '--metadata'),
1271         sizelimit=FlagArgument(
1272             'Show the maximum size limit for container', '--size-limit'),
1273         in_bytes=FlagArgument('Show size limit in bytes', ('-b', '--bytes'))
1274     )
1275
1276     @errors.generic.all
1277     @errors.pithos.connection
1278     @errors.pithos.container
1279     @errors.pithos.object_path
1280     def _run(self):
1281         if self['metadata']:
1282             r, preflen = dict(), len('x-container-meta-')
1283             for k, v in self.client.get_container_meta(
1284                     until=self['until_date']).items():
1285                 r[k[preflen:]] = v
1286         elif self['sizelimit']:
1287             r = self.client.get_container_limit(
1288                 self.container)['x-container-policy-quota']
1289             r = {'size limit': 'unlimited' if r in ('0', ) else (
1290                 int(r) if self['in_bytes'] else format_size(r))}
1291         else:
1292             r = self.client.get_container_info(self.container)
1293         self._print(r, self.print_dict)
1294
1295     def main(self, container):
1296         super(self.__class__, self)._run()
1297         self.container, self.client.container = container, container
1298         self._run()
1299
1300
1301 class VersioningArgument(ValueArgument):
1302
1303     schemes = ('auto', 'none')
1304
1305     @property
1306     def value(self):
1307         return getattr(self, '_value', None)
1308
1309     @value.setter
1310     def value(self, new_scheme):
1311         if new_scheme:
1312             new_scheme = new_scheme.lower()
1313             if new_scheme not in self.schemes:
1314                 raise CLIInvalidArgument('Invalid versioning value', details=[
1315                     'Valid versioning values are %s' % ', '.join(
1316                         self.schemes)])
1317             self._value = new_scheme
1318
1319
1320 @command(container_cmds)
1321 class container_modify(_pithos_account, _optional_json):
1322     """Modify the properties of a container"""
1323
1324     arguments = dict(
1325         metadata_to_add=KeyValueArgument(
1326             'Add metadata in the form KEY=VALUE (can be repeated)',
1327             '--metadata-add'),
1328         metadata_to_delete=RepeatableArgument(
1329             'Delete metadata by KEY (can be repeated)', '--metadata-del'),
1330         sizelimit=DataSizeArgument(
1331             'Set max size limit (0 for unlimited, '
1332             'use units B, KiB, KB, etc.)', '--size-limit'),
1333         versioning=VersioningArgument(
1334             'Set a versioning scheme (%s)' % ', '.join(
1335                 VersioningArgument.schemes), '--versioning')
1336     )
1337     required = ['metadata_to_add', 'metadata_to_delete', 'sizelimit']
1338
1339     @errors.generic.all
1340     @errors.pithos.connection
1341     @errors.pithos.container
1342     def _run(self, container):
1343         metadata = self['metadata_to_add']
1344         for k in self['metadata_to_delete']:
1345             metadata[k] = ''
1346         if metadata:
1347             self.client.set_container_meta(metadata)
1348             self._print(self.client.get_container_meta(), self.print_dict)
1349         if self['sizelimit'] is not None:
1350             self.client.set_container_limit(self['sizelimit'])
1351             r = self.client.get_container_limit()['x-container-policy-quota']
1352             r = 'unlimited' if r in ('0', ) else format_size(r)
1353             self.writeln('new size limit: %s' % r)
1354         if self['versioning']:
1355             self.client.set_container_versioning(self['versioning'])
1356             self.writeln('new versioning scheme: %s' % (
1357                 self.client.get_container_versioning(self.container)[
1358                     'x-container-policy-versioning']))
1359
1360     def main(self, container):
1361         super(self.__class__, self)._run()
1362         self.client.container, self.container = container, container
1363         self._run(container=container)
1364
1365
1366 @command(container_cmds)
1367 class container_list(_pithos_account, _optional_json, _name_filter):
1368     """List all containers, or their contents"""
1369
1370     arguments = dict(
1371         detail=FlagArgument('Containers with details', ('-l', '--list')),
1372         limit=IntArgument('limit number of listed items', ('-n', '--number')),
1373         marker=ValueArgument('output greater that marker', '--marker'),
1374         modified_since_date=ValueArgument(
1375             'show output modified since then', '--if-modified-since'),
1376         unmodified_since_date=ValueArgument(
1377             'show output not modified since then', '--if-unmodified-since'),
1378         until_date=DateArgument('show metadata until then', '--until'),
1379         shared=FlagArgument('show only shared', '--shared'),
1380         more=FlagArgument('read long results', '--more'),
1381         enum=FlagArgument('Enumerate results', '--enumerate'),
1382         recursive=FlagArgument(
1383             'Recursively list containers and their contents',
1384             ('-r', '--recursive'))
1385     )
1386
1387     def print_containers(self, container_list):
1388         for index, container in enumerate(container_list):
1389             if 'bytes' in container:
1390                 size = format_size(container['bytes'])
1391             prfx = ('%s. ' % (index + 1)) if self['enum'] else ''
1392             _cname = container['name'] if (
1393                 self['more']) else bold(container['name'])
1394             cname = u'%s%s' % (prfx, _cname)
1395             if self['detail']:
1396                 self.writeln(cname)
1397                 pretty_c = container.copy()
1398                 if 'bytes' in container:
1399                     pretty_c['bytes'] = '%s (%s)' % (container['bytes'], size)
1400                 self.print_dict(pretty_c, exclude=('name'))
1401                 self.writeln()
1402             else:
1403                 if 'count' in container and 'bytes' in container:
1404                     self.writeln('%s (%s, %s objects)' % (
1405                         cname, size, container['count']))
1406                 else:
1407                     self.writeln(cname)
1408             objects = container.get('objects', [])
1409             if objects:
1410                 self.print_objects(objects)
1411                 self.writeln('')
1412
1413     def _create_object_forest(self, container_list):
1414         try:
1415             for container in container_list:
1416                 self.client.container = container['name']
1417                 objects = self.client.container_get(
1418                     limit=False if self['more'] else self['limit'],
1419                     if_modified_since=self['modified_since_date'],
1420                     if_unmodified_since=self['unmodified_since_date'],
1421                     until=self['until_date'],
1422                     show_only_shared=self['shared'])
1423                 container['objects'] = objects.json
1424         finally:
1425             self.client.container = None
1426
1427     @errors.generic.all
1428     @errors.pithos.connection
1429     @errors.pithos.object_path
1430     @errors.pithos.container
1431     def _run(self, container):
1432         if container:
1433             r = self.client.container_get(
1434                 limit=False if self['more'] else self['limit'],
1435                 marker=self['marker'],
1436                 if_modified_since=self['modified_since_date'],
1437                 if_unmodified_since=self['unmodified_since_date'],
1438                 until=self['until_date'],
1439                 show_only_shared=self['shared'])
1440         else:
1441             r = self.client.account_get(
1442                 limit=False if self['more'] else self['limit'],
1443                 marker=self['marker'],
1444                 if_modified_since=self['modified_since_date'],
1445                 if_unmodified_since=self['unmodified_since_date'],
1446                 until=self['until_date'],
1447                 show_only_shared=self['shared'])
1448         files = self._filter_by_name(r.json)
1449         if self['recursive'] and not container:
1450             self._create_object_forest(files)
1451         if self['more']:
1452             outbu, self._out = self._out, StringIO()
1453         try:
1454             if self['json_output'] or self['output_format']:
1455                 self._print(files)
1456             else:
1457                 (self.print_objects if container else self.print_containers)(
1458                     files)
1459         finally:
1460             if self['more']:
1461                 pager(self._out.getvalue())
1462                 self._out = outbu
1463
1464     def main(self, container=None):
1465         super(self.__class__, self)._run()
1466         self.client.container, self.container = container, container
1467         self._run(container)
1468
1469
1470 @command(container_cmds)
1471 class container_create(_pithos_account):
1472     """Create a new container"""
1473
1474     arguments = dict(
1475         versioning=ValueArgument(
1476             'set container versioning (auto/none)', '--versioning'),
1477         limit=IntArgument('set default container limit', '--limit'),
1478         meta=KeyValueArgument(
1479             'set container metadata (can be repeated)', '--meta')
1480     )
1481
1482     @errors.generic.all
1483     @errors.pithos.connection
1484     @errors.pithos.container
1485     def _run(self, container):
1486         try:
1487             self.client.create_container(
1488                 container=container,
1489                 sizelimit=self['limit'],
1490                 versioning=self['versioning'],
1491                 metadata=self['meta'],
1492                 success=(201, ))
1493         except ClientError as ce:
1494             if ce.status in (202, ):
1495                 raise CLIError(
1496                     'Container %s alread exists' % container, details=[
1497                     'Either delete %s or choose another name' % (container)])
1498             raise
1499
1500     def main(self, new_container):
1501         super(self.__class__, self)._run()
1502         self._run(container=new_container)
1503
1504
1505 @command(container_cmds)
1506 class container_delete(_pithos_account):
1507     """Delete a container"""
1508
1509     arguments = dict(
1510         yes=FlagArgument('Do not prompt for permission', '--yes'),
1511         recursive=FlagArgument(
1512             'delete container even if not empty', ('-r', '--recursive'))
1513     )
1514
1515     @errors.generic.all
1516     @errors.pithos.connection
1517     @errors.pithos.container
1518     def _run(self, container):
1519         num_of_contents = int(self.client.get_container_info(container)[
1520             'x-container-object-count'])
1521         delimiter, msg = None, 'Delete container %s ?' % container
1522         if self['recursive']:
1523             delimiter, msg = '/', 'Empty and d%s' % msg[1:]
1524         elif num_of_contents:
1525             raise CLIError('Container %s is not empty' % container, details=[
1526                 'Use %s to delete non-empty containers' % '/'.join(
1527                     self.arguments['recursive'].parsed_name)])
1528         if self['yes'] or self.ask_user(msg):
1529             if num_of_contents:
1530                 self.client.del_container(delimiter=delimiter)
1531             self.client.purge_container()
1532
1533     def main(self, container):
1534         super(self.__class__, self)._run()
1535         self.container, self.client.container = container, container
1536         self._run(container)
1537
1538
1539 @command(container_cmds)
1540 class container_empty(_pithos_account):
1541     """Empty a container"""
1542
1543     arguments = dict(yes=FlagArgument('Do not prompt for permission', '--yes'))
1544
1545     @errors.generic.all
1546     @errors.pithos.connection
1547     @errors.pithos.container
1548     def _run(self, container):
1549         if self['yes'] or self.ask_user('Empty container %s ?' % container):
1550             self.client.del_container(delimiter='/')
1551
1552     def main(self, container):
1553         super(self.__class__, self)._run()
1554         self.container, self.client.container = container, container
1555         self._run(container)