code.grnet.gr Git - kamaki/blob - kamaki/cli/commands/pithos.py

   1 # Copyright 2011-2013 GRNET S.A. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or
   4 # without modification, are permitted provided that the following
   5 # conditions are met:
   6 #
   7 #   1. Redistributions of source code must retain the above
   8 #      copyright notice, this list of conditions and the following
   9 #      disclaimer.
  10 #
  11 #   2. Redistributions in binary form must reproduce the above
  12 #      copyright notice, this list of conditions and the following
  13 #      disclaimer in the documentation and/or other materials
  14 #      provided with the distribution.
  15 #
  16 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  17 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  23 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  24 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  26 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27 # POSSIBILITY OF SUCH DAMAGE.
  28 #
  29 # The views and conclusions contained in the software and
  30 # documentation are those of the authors and should not be
  31 # interpreted as representing official policies, either expressed
  32 # or implied, of GRNET S.A.command
  33
  34 from time import localtime, strftime
  35 from io import StringIO
  36 from pydoc import pager
  37 from os import path, walk, makedirs
  38
  39 from kamaki.clients.pithos import PithosClient, ClientError
  40
  41 from kamaki.cli import command
  42 from kamaki.cli.command_tree import CommandTree
  43 from kamaki.cli.commands import (
  44     _command_init, errors, addLogSettings, DontRaiseKeyError, _optional_json,
  45     _name_filter, _optional_output_cmd)
  46 from kamaki.cli.errors import (
  47     CLIBaseUrlError, CLIError, CLIInvalidArgument, raiseCLIError,
  48     CLISyntaxError)
  49 from kamaki.cli.argument import (
  50     FlagArgument, IntArgument, ValueArgument, DateArgument, KeyValueArgument,
  51     ProgressBarArgument, RepeatableArgument, DataSizeArgument)
  52 from kamaki.cli.utils import (
  53     format_size, bold, get_path_size, guess_mime_type)
  54
  55 file_cmds = CommandTree('file', 'Pithos+/Storage object level API commands')
  56 container_cmds = CommandTree(
  57     'container', 'Pithos+/Storage container level API commands')
  58 sharer_cmds = CommandTree('sharer', 'Pithos+/Storage sharers')
  59 group_cmds = CommandTree('group', 'Pithos+/Storage user groups')
  60 _commands = [file_cmds, container_cmds, sharer_cmds, group_cmds]
  61
  62
  63 class _pithos_init(_command_init):
  64     """Initilize a pithos+ client
  65     There is always a default account (current user uuid)
  66     There is always a default container (pithos)
  67     """
  68
  69     @DontRaiseKeyError
  70     def _custom_container(self):
  71         return self.config.get_cloud(self.cloud, 'pithos_container')
  72
  73     @DontRaiseKeyError
  74     def _custom_uuid(self):
  75         return self.config.get_cloud(self.cloud, 'pithos_uuid')
  76
  77     def _set_account(self):
  78         self.account = self._custom_uuid()
  79         if self.account:
  80             return
  81         astakos = getattr(self, 'auth_base', None)
  82         if astakos:
  83             self.account = astakos.user_term('id', self.token)
  84         else:
  85             raise CLIBaseUrlError(service='astakos')
  86
  87     @errors.generic.all
  88     @addLogSettings
  89     def _run(self):
  90         cloud = getattr(self, 'cloud', None)
  91         if cloud:
  92             self.base_url = self._custom_url('pithos')
  93         else:
  94             self.cloud = 'default'
  95         self.token = self._custom_token('pithos')
  96         self.container = self._custom_container() or 'pithos'
  97
  98         astakos = getattr(self, 'auth_base', None)
  99         if astakos:
 100             self.token = self.token or astakos.token
 101             if not self.base_url:
 102                 pithos_endpoints = astakos.get_service_endpoints(
 103                     self._custom_type('pithos') or 'object-store',
 104                     self._custom_version('pithos') or '')
 105                 self.base_url = pithos_endpoints['publicURL']
 106         else:
 107             raise CLIBaseUrlError(service='astakos')
 108
 109         self._set_account()
 110         self.client = PithosClient(
 111             self.base_url, self.token, self.account, self.container)
 112
 113     def main(self):
 114         self._run()
 115
 116
 117 class _pithos_account(_pithos_init):
 118     """Setup account"""
 119
 120     def __init__(self, arguments={}, auth_base=None, cloud=None):
 121         super(_pithos_account, self).__init__(arguments, auth_base, cloud)
 122         self['account'] = ValueArgument(
 123             'Use (a different) user uuid', ('-A', '--account'))
 124
 125     def print_objects(self, object_list):
 126         for index, obj in enumerate(object_list):
 127             pretty_obj = obj.copy()
 128             index += 1
 129             empty_space = ' ' * (len(str(len(object_list))) - len(str(index)))
 130             if 'subdir' in obj:
 131                 continue
 132             if self._is_dir(obj):
 133                 size = 'D'
 134             else:
 135                 size = format_size(obj['bytes'])
 136                 pretty_obj['bytes'] = '%s (%s)' % (obj['bytes'], size)
 137             oname = obj['name'] if self['more'] else bold(obj['name'])
 138             prfx = ('%s%s. ' % (empty_space, index)) if self['enum'] else ''
 139             if self['detail']:
 140                 self.writeln('%s%s' % (prfx, oname))
 141                 self.print_dict(pretty_obj, exclude=('name'))
 142                 self.writeln()
 143             else:
 144                 oname = '%s%9s %s' % (prfx, size, oname)
 145                 oname += '/' if self._is_dir(obj) else u''
 146                 self.writeln(oname)
 147
 148     @staticmethod
 149     def _is_dir(remote_dict):
 150         return 'application/directory' == remote_dict.get(
 151             'content_type', remote_dict.get('content-type', ''))
 152
 153     def _run(self):
 154         super(_pithos_account, self)._run()
 155         self.client.account = self['account'] or getattr(
 156             self, 'account', getattr(self.client, 'account', None))
 157
 158
 159 class _pithos_container(_pithos_account):
 160     """Setup container"""
 161
 162     def __init__(self, arguments={}, auth_base=None, cloud=None):
 163         super(_pithos_container, self).__init__(arguments, auth_base, cloud)
 164         self['container'] = ValueArgument(
 165             'Use this container (default: pithos)', ('-C', '--container'))
 166
 167     @staticmethod
 168     def _resolve_pithos_url(url):
 169         """Match urls of one of the following formats:
 170         pithos://ACCOUNT/CONTAINER/OBJECT_PATH
 171         /CONTAINER/OBJECT_PATH
 172         return account, container, path
 173         """
 174         account, container, obj_path, prefix = '', '', url, 'pithos://'
 175         if url.startswith(prefix):
 176             account, sep, url = url[len(prefix):].partition('/')
 177             url = '/%s' % url
 178         if url.startswith('/'):
 179             container, sep, obj_path = url[1:].partition('/')
 180         return account, container, obj_path
 181
 182     def _run(self, url=None):
 183         acc, con, self.path = self._resolve_pithos_url(url or '')
 184         #  self.account = acc or getattr(self, 'account', '')
 185         super(_pithos_container, self)._run()
 186         self.container = con or self['container'] or getattr(
 187             self, 'container', None) or getattr(self.client, 'container', '')
 188         self.client.account = acc or self.client.account
 189         self.client.container = self.container
 190
 191
 192 @command(file_cmds)
 193 class file_info(_pithos_container, _optional_json):
 194     """Get information/details about a file"""
 195
 196     arguments = dict(
 197         object_version=ValueArgument(
 198             'download a file of a specific version', '--object-version'),
 199         hashmap=FlagArgument(
 200             'Get file hashmap instead of details', '--hashmap'),
 201         matching_etag=ValueArgument(
 202             'show output if ETags match', '--if-match'),
 203         non_matching_etag=ValueArgument(
 204             'show output if ETags DO NOT match', '--if-none-match'),
 205         modified_since_date=DateArgument(
 206             'show output modified since then', '--if-modified-since'),
 207         unmodified_since_date=DateArgument(
 208             'show output unmodified since then', '--if-unmodified-since'),
 209         sharing=FlagArgument(
 210             'show object permissions and sharing information', '--sharing'),
 211         metadata=FlagArgument('show only object metadata', '--metadata'),
 212         versions=FlagArgument(
 213             'show the list of versions for the file', '--object-versions')
 214     )
 215
 216     def version_print(self, versions):
 217         return {'/%s/%s' % (self.container, self.path): [
 218             dict(version_id=vitem[0], created=strftime(
 219                 '%d-%m-%Y %H:%M:%S',
 220                 localtime(float(vitem[1])))) for vitem in versions]}
 221
 222     @errors.generic.all
 223     @errors.pithos.connection
 224     @errors.pithos.container
 225     @errors.pithos.object_path
 226     def _run(self):
 227         if self['hashmap']:
 228             r = self.client.get_object_hashmap(
 229                 self.path,
 230                 version=self['object_version'],
 231                 if_match=self['matching_etag'],
 232                 if_none_match=self['non_matching_etag'],
 233                 if_modified_since=self['modified_since_date'],
 234                 if_unmodified_since=self['unmodified_since_date'])
 235         elif self['sharing']:
 236             r = self.client.get_object_sharing(self.path)
 237             r['public url'] = self.client.get_object_info(
 238                 self.path, version=self['object_version']).get(
 239                     'x-object-public', None)
 240         elif self['metadata']:
 241             r, preflen = dict(), len('x-object-meta-')
 242             for k, v in self.client.get_object_meta(self.path).items():
 243                 r[k[preflen:]] = v
 244         elif self['versions']:
 245             r = self.version_print(
 246                 self.client.get_object_versionlist(self.path))
 247         else:
 248             r = self.client.get_object_info(
 249                 self.path, version=self['object_version'])
 250         self._print(r, self.print_dict)
 251
 252     def main(self, path_or_url):
 253         super(self.__class__, self)._run(path_or_url)
 254         self._run()
 255
 256
 257 @command(file_cmds)
 258 class file_list(_pithos_container, _optional_json, _name_filter):
 259     """List all objects in a container or a directory object"""
 260
 261     arguments = dict(
 262         detail=FlagArgument('detailed output', ('-l', '--list')),
 263         limit=IntArgument('limit number of listed items', ('-n', '--number')),
 264         marker=ValueArgument('output greater that marker', '--marker'),
 265         delimiter=ValueArgument('show output up to delimiter', '--delimiter'),
 266         meta=ValueArgument(
 267             'show output with specified meta keys', '--meta',
 268             default=[]),
 269         if_modified_since=ValueArgument(
 270             'show output modified since then', '--if-modified-since'),
 271         if_unmodified_since=ValueArgument(
 272             'show output not modified since then', '--if-unmodified-since'),
 273         until=DateArgument('show metadata until then', '--until'),
 274         format=ValueArgument(
 275             'format to parse until data (default: d/m/Y H:M:S )', '--format'),
 276         shared_by_me=FlagArgument(
 277             'show only files shared to other users', '--shared-by-me'),
 278         public=FlagArgument('show only published objects', '--public'),
 279         more=FlagArgument('read long results', '--more'),
 280         enum=FlagArgument('Enumerate results', '--enumerate'),
 281         recursive=FlagArgument(
 282             'Recursively list containers and their contents',
 283             ('-R', '--recursive'))
 284     )
 285
 286     @errors.generic.all
 287     @errors.pithos.connection
 288     @errors.pithos.container
 289     @errors.pithos.object_path
 290     def _run(self):
 291         r = self.client.container_get(
 292             limit=False if self['more'] else self['limit'],
 293             marker=self['marker'],
 294             prefix=self['name_pref'],
 295             delimiter=self['delimiter'],
 296             path=self.path or '',
 297             show_only_shared=self['shared_by_me'],
 298             public=self['public'],
 299             if_modified_since=self['if_modified_since'],
 300             if_unmodified_since=self['if_unmodified_since'],
 301             until=self['until'],
 302             meta=self['meta'])
 303
 304         #  REMOVE THIS if version >> 0.12
 305         if not r.json:
 306             self.error('  NOTE: Since v0.12, use / for containers e.g.,')
 307             self.error('    [kamaki] file list /pithos')
 308
 309         files = self._filter_by_name(r.json)
 310         if self['more']:
 311             outbu, self._out = self._out, StringIO()
 312         try:
 313             if self['json_output'] or self['output_format']:
 314                 self._print(files)
 315             else:
 316                 self.print_objects(files)
 317         finally:
 318             if self['more']:
 319                 pager(self._out.getvalue())
 320                 self._out = outbu
 321
 322     def main(self, path_or_url=''):
 323         super(self.__class__, self)._run(path_or_url)
 324         self._run()
 325
 326
 327 @command(file_cmds)
 328 class file_modify(_pithos_container):
 329     """Modify the attributes of a file or directory object"""
 330
 331     arguments = dict(
 332         publish=FlagArgument(
 333             'Make an object public (returns the public URL)', '--publish'),
 334         unpublish=FlagArgument(
 335             'Make an object unpublic', '--unpublish'),
 336         uuid_for_read_permission=RepeatableArgument(
 337             'Give read access to user/group (can be repeated, accumulative). '
 338             'Format for users: UUID . Format for groups: UUID:GROUP . '
 339             'Use * for all users/groups', '--read-permission'),
 340         uuid_for_write_permission=RepeatableArgument(
 341             'Give write access to user/group (can be repeated, accumulative). '
 342             'Format for users: UUID . Format for groups: UUID:GROUP . '
 343             'Use * for all users/groups', '--write-permission'),
 344         no_permissions=FlagArgument('Remove permissions', '--no-permissions'),
 345         metadata_to_set=KeyValueArgument(
 346             'Add metadata (KEY=VALUE) to an object (can be repeated)',
 347             '--metadata-add'),
 348         metadata_key_to_delete=RepeatableArgument(
 349             'Delete object metadata (can be repeated)', '--metadata-del'),
 350     )
 351     required = [
 352         'publish', 'unpublish', 'uuid_for_read_permission', 'metadata_to_set',
 353         'uuid_for_write_permission', 'no_permissions',
 354         'metadata_key_to_delete']
 355
 356     @errors.generic.all
 357     @errors.pithos.connection
 358     @errors.pithos.container
 359     @errors.pithos.object_path
 360     def _run(self):
 361         if self['publish']:
 362             self.writeln(self.client.publish_object(self.path))
 363         if self['unpublish']:
 364             self.client.unpublish_object(self.path)
 365         if self['uuid_for_read_permission'] or self[
 366                 'uuid_for_write_permission']:
 367             perms = self.client.get_object_sharing(self.path)
 368             read, write = perms.get('read', ''), perms.get('write', '')
 369             read = read.split(',') if read else []
 370             write = write.split(',') if write else []
 371             read += (self['uuid_for_read_permission'] or [])
 372             write += (self['uuid_for_write_permission'] or [])
 373             self.client.set_object_sharing(
 374                 self.path, read_permission=read, write_permission=write)
 375             self.print_dict(self.client.get_object_sharing(self.path))
 376         if self['no_permissions']:
 377             self.client.del_object_sharing(self.path)
 378         metadata = self['metadata_to_set'] or dict()
 379         for k in (self['metadata_key_to_delete'] or []):
 380             metadata[k] = ''
 381         if metadata:
 382             self.client.set_object_meta(self.path, metadata)
 383             self.print_dict(self.client.get_object_meta(self.path))
 384
 385     def main(self, path_or_url):
 386         super(self.__class__, self)._run(path_or_url)
 387         if self['publish'] and self['unpublish']:
 388             raise CLIInvalidArgument(
 389                 'Arguments %s and %s cannot be used together' % (
 390                     self.arguments['publish'].lvalue,
 391                     self.arguments['publish'].lvalue))
 392         if self['no_permissions'] and (
 393                 self['uuid_for_read_permission'] or self[
 394                     'uuid_for_write_permission']):
 395             raise CLIInvalidArgument(
 396                 '%s cannot be used with other permission arguments' % (
 397                     self.arguments['no_permissions'].lvalue))
 398         self._run()
 399
 400
 401 @command(file_cmds)
 402 class file_create(_pithos_container, _optional_output_cmd):
 403     """Create an empty file"""
 404
 405     arguments = dict(
 406         content_type=ValueArgument(
 407             'Set content type (default: application/octet-stream)',
 408             '--content-type',
 409             default='application/octet-stream')
 410     )
 411
 412     @errors.generic.all
 413     @errors.pithos.connection
 414     @errors.pithos.container
 415     def _run(self):
 416         self._optional_output(
 417             self.client.create_object(self.path, self['content_type']))
 418
 419     def main(self, path_or_url):
 420         super(self.__class__, self)._run(path_or_url)
 421         self._run()
 422
 423
 424 @command(file_cmds)
 425 class file_mkdir(_pithos_container, _optional_output_cmd):
 426     """Create a directory: /file create --content-type='applcation/directory'
 427     """
 428
 429     @errors.generic.all
 430     @errors.pithos.connection
 431     @errors.pithos.container
 432     def _run(self):
 433         self._optional_output(self.client.create_directory(self.path))
 434
 435     def main(self, path_or_url):
 436         super(self.__class__, self)._run(path_or_url)
 437         self._run()
 438
 439
 440 @command(file_cmds)
 441 class file_delete(_pithos_container):
 442     """Delete a file or directory object"""
 443
 444     arguments = dict(
 445         until_date=DateArgument('remove history until then', '--until'),
 446         yes=FlagArgument('Do not prompt for permission', '--yes'),
 447         recursive=FlagArgument(
 448             'If a directory, empty first', ('-r', '--recursive')),
 449         delimiter=ValueArgument(
 450             'delete objects prefixed with <object><delimiter>', '--delimiter')
 451     )
 452
 453     @errors.generic.all
 454     @errors.pithos.connection
 455     @errors.pithos.container
 456     @errors.pithos.object_path
 457     def _run(self):
 458         if self.path:
 459             if self['yes'] or self.ask_user(
 460                     'Delete /%s/%s ?' % (self.container, self.path)):
 461                 self.client.del_object(
 462                     self.path,
 463                     until=self['until_date'],
 464                     delimiter='/' if self['recursive'] else self['delimiter'])
 465             else:
 466                 self.error('Aborted')
 467         else:
 468             if self['yes'] or self.ask_user(
 469                     'Empty container /%s ?' % self.container):
 470                 self.client.container_delete(self.container, delimiter='/')
 471             else:
 472                 self.error('Aborted')
 473
 474     def main(self, path_or_url):
 475         super(self.__class__, self)._run(path_or_url)
 476         self._run()
 477
 478
 479 class _source_destination(_pithos_container, _optional_output_cmd):
 480
 481     sd_arguments = dict(
 482         destination_user_uuid=ValueArgument(
 483             'default: current user uuid', '--to-account'),
 484         destination_container=ValueArgument(
 485             'default: pithos', '--to-container'),
 486         source_prefix=FlagArgument(
 487             'Transfer all files that are prefixed with SOURCE PATH If the '
 488             'destination path is specified, replace SOURCE_PATH with '
 489             'DESTINATION_PATH',
 490             ('-r', '--recursive')),
 491         force=FlagArgument(
 492             'Overwrite destination objects, if needed', ('-f', '--force')),
 493         source_version=ValueArgument(
 494             'The version of the source object', '--source-version')
 495     )
 496
 497     def __init__(self, arguments={}, auth_base=None, cloud=None):
 498         self.arguments.update(arguments)
 499         self.arguments.update(self.sd_arguments)
 500         super(_source_destination, self).__init__(
 501             self.arguments, auth_base, cloud)
 502
 503     def _report_transfer(self, src, dst, transfer_name):
 504         if not dst:
 505             if transfer_name in ('move', ):
 506                 self.error('  delete source directory %s' % src)
 507             return
 508         dst_prf = '' if self.account == self.dst_client.account else (
 509                 'pithos://%s' % self.dst_client.account)
 510         if src:
 511             src_prf = '' if self.account == self.dst_client.account else (
 512                     'pithos://%s' % self.account)
 513             self.error('  %s %s/%s/%s\n  -->  %s/%s/%s' % (
 514                 transfer_name,
 515                 src_prf, self.container, src,
 516                 dst_prf, self.dst_client.container, dst))
 517         else:
 518             self.error('  mkdir %s/%s/%s' % (
 519                 dst_prf, self.dst_client.container, dst))
 520
 521     @errors.generic.all
 522     @errors.pithos.account
 523     def _src_dst(self, version=None):
 524         """Preconditions:
 525         self.account, self.container, self.path
 526         self.dst_acc, self.dst_con, self.dst_path
 527         They should all be configured properly
 528         :returns: [(src_path, dst_path), ...], if src_path is None, create
 529             destination directory
 530         """
 531         src_objects, dst_objects, pairs = dict(), dict(), []
 532         try:
 533             for obj in self.dst_client.list_objects(
 534                     prefix=self.dst_path or self.path or '/'):
 535                 dst_objects[obj['name']] = obj
 536         except ClientError as ce:
 537             if ce.status in (404, ):
 538                 raise CLIError(
 539                     'Destination container pithos://%s/%s not found' % (
 540                         self.dst_client.account, self.dst_client.container))
 541             raise ce
 542         if self['source_prefix']:
 543             #  Copy and replace prefixes
 544             for src_obj in self.client.list_objects(prefix=self.path):
 545                 src_objects[src_obj['name']] = src_obj
 546             for src_path, src_obj in src_objects.items():
 547                 dst_path = '%s%s' % (
 548                     self.dst_path or self.path, src_path[len(self.path):])
 549                 dst_obj = dst_objects.get(dst_path, None)
 550                 if self['force'] or not dst_obj:
 551                     #  Just do it
 552                     pairs.append((
 553                         None if self._is_dir(src_obj) else src_path, dst_path))
 554                     if self._is_dir(src_obj):
 555                         pairs.append((self.path or dst_path, None))
 556                 elif not (self._is_dir(dst_obj) and self._is_dir(src_obj)):
 557                     raise CLIError(
 558                         'Destination object exists', importance=2, details=[
 559                             'Failed while transfering:',
 560                             '    pithos://%s/%s/%s' % (
 561                                     self.account,
 562                                     self.container,
 563                                     src_path),
 564                             '--> pithos://%s/%s/%s' % (
 565                                     self.dst_client.account,
 566                                     self.dst_client.container,
 567                                     dst_path),
 568                             'Use %s to transfer overwrite' % (
 569                                     self.arguments['force'].lvalue)])
 570         else:
 571             #  One object transfer
 572             try:
 573                 src_version_arg = self.arguments.get('source_version', None)
 574                 src_obj = self.client.get_object_info(
 575                     self.path,
 576                     version=src_version_arg.value if src_version_arg else None)
 577             except ClientError as ce:
 578                 if ce.status in (204, ):
 579                     raise CLIError(
 580                         'Missing specific path container %s' % self.container,
 581                         importance=2, details=[
 582                             'To transfer container contents %s' % (
 583                                 self.arguments['source_prefix'].lvalue)])
 584                 raise
 585             dst_path = self.dst_path or self.path
 586             dst_obj = dst_objects.get(dst_path or self.path, None)
 587             if self['force'] or not dst_obj:
 588                 pairs.append(
 589                     (None if self._is_dir(src_obj) else self.path, dst_path))
 590                 if self._is_dir(src_obj):
 591                     pairs.append((self.path or dst_path, None))
 592             elif self._is_dir(src_obj):
 593                 raise CLIError(
 594                     'Cannot transfer an application/directory object',
 595                     importance=2, details=[
 596                         'The object pithos://%s/%s/%s is a directory' % (
 597                             self.account,
 598                             self.container,
 599                             self.path),
 600                         'To recursively copy a directory, use',
 601                         '  %s' % self.arguments['source_prefix'].lvalue,
 602                         'To create a file, use',
 603                         '  /file create  (general purpose)',
 604                         '  /file mkdir   (a directory object)'])
 605             else:
 606                 raise CLIError(
 607                     'Destination object exists',
 608                     importance=2, details=[
 609                         'Failed while transfering:',
 610                         '    pithos://%s/%s/%s' % (
 611                                 self.account,
 612                                 self.container,
 613                                 self.path),
 614                         '--> pithos://%s/%s/%s' % (
 615                                 self.dst_client.account,
 616                                 self.dst_client.container,
 617                                 dst_path),
 618                         'Use %s to transfer overwrite' % (
 619                                 self.arguments['force'].lvalue)])
 620         return pairs
 621
 622     def _run(self, source_path_or_url, destination_path_or_url=''):
 623         super(_source_destination, self)._run(source_path_or_url)
 624         dst_acc, dst_con, dst_path = self._resolve_pithos_url(
 625             destination_path_or_url)
 626         self.dst_client = PithosClient(
 627             base_url=self.client.base_url, token=self.client.token,
 628             container=self[
 629                 'destination_container'] or dst_con or self.client.container,
 630             account=self[
 631                 'destination_user_uuid'] or dst_acc or self.account)
 632         self.dst_path = dst_path or self.path
 633
 634
 635 @command(file_cmds)
 636 class file_copy(_source_destination):
 637     """Copy objects, even between different accounts or containers"""
 638
 639     arguments = dict(
 640         public=ValueArgument('publish new object', '--public'),
 641         content_type=ValueArgument(
 642             'change object\'s content type', '--content-type'),
 643         source_version=ValueArgument(
 644             'The version of the source object', '--object-version')
 645     )
 646
 647     @errors.generic.all
 648     @errors.pithos.connection
 649     @errors.pithos.container
 650     @errors.pithos.account
 651     def _run(self):
 652         for src, dst in self._src_dst(self['source_version']):
 653             self._report_transfer(src, dst, 'copy')
 654             if src and dst:
 655                 self.dst_client.copy_object(
 656                     src_container=self.client.container,
 657                     src_object=src,
 658                     dst_container=self.dst_client.container,
 659                     dst_object=dst,
 660                     source_account=self.client.account,
 661                     source_version=self['source_version'],
 662                     public=self['public'],
 663                     content_type=self['content_type'])
 664             elif dst:
 665                 self.dst_client.create_directory(dst)
 666
 667     def main(self, source_path_or_url, destination_path_or_url=None):
 668         super(file_copy, self)._run(
 669             source_path_or_url, destination_path_or_url or '')
 670         self._run()
 671
 672
 673 @command(file_cmds)
 674 class file_move(_source_destination):
 675     """Move objects, even between different accounts or containers"""
 676
 677     arguments = dict(
 678         public=ValueArgument('publish new object', '--public'),
 679         content_type=ValueArgument(
 680             'change object\'s content type', '--content-type')
 681     )
 682
 683     @errors.generic.all
 684     @errors.pithos.connection
 685     @errors.pithos.container
 686     @errors.pithos.account
 687     def _run(self):
 688         for src, dst in self._src_dst():
 689             self._report_transfer(src, dst, 'move')
 690             if src and dst:
 691                 self.dst_client.move_object(
 692                     src_container=self.client.container,
 693                     src_object=src,
 694                     dst_container=self.dst_client.container,
 695                     dst_object=dst,
 696                     source_account=self.account,
 697                     public=self['public'],
 698                     content_type=self['content_type'])
 699             elif dst:
 700                 self.dst_client.create_directory(dst)
 701             else:
 702                 self.client.del_object(src)
 703
 704     def main(self, source_path_or_url, destination_path_or_url=None):
 705         super(file_move, self)._run(
 706             source_path_or_url, destination_path_or_url or '')
 707         self._run()
 708
 709
 710 @command(file_cmds)
 711 class file_append(_pithos_container, _optional_output_cmd):
 712     """Append local file to (existing) remote object
 713     The remote object should exist.
 714     If the remote object is a directory, it is transformed into a file.
 715     In the later case, objects under the directory remain intact.
 716     """
 717
 718     arguments = dict(
 719         progress_bar=ProgressBarArgument(
 720             'do not show progress bar', ('-N', '--no-progress-bar'),
 721             default=False),
 722         max_threads=IntArgument('default: 1', '--threads'),
 723     )
 724
 725     @errors.generic.all
 726     @errors.pithos.connection
 727     @errors.pithos.container
 728     @errors.pithos.object_path
 729     def _run(self, local_path):
 730         if self['max_threads'] > 0:
 731             self.client.MAX_THREADS = int(self['max_threads'])
 732         (progress_bar, upload_cb) = self._safe_progress_bar('Appending')
 733         try:
 734             with open(local_path, 'rb') as f:
 735                 self._optional_output(
 736                     self.client.append_object(self.path, f, upload_cb))
 737         finally:
 738             self._safe_progress_bar_finish(progress_bar)
 739
 740     def main(self, local_path, remote_path_or_url):
 741         super(self.__class__, self)._run(remote_path_or_url)
 742         self._run(local_path)
 743
 744
 745 @command(file_cmds)
 746 class file_truncate(_pithos_container, _optional_output_cmd):
 747     """Truncate remote file up to size"""
 748
 749     arguments = dict(
 750         size_in_bytes=IntArgument('Length of file after truncation', '--size')
 751     )
 752     required = ('size_in_bytes', )
 753
 754     @errors.generic.all
 755     @errors.pithos.connection
 756     @errors.pithos.container
 757     @errors.pithos.object_path
 758     @errors.pithos.object_size
 759     def _run(self, size):
 760         self._optional_output(self.client.truncate_object(self.path, size))
 761
 762     def main(self, path_or_url):
 763         super(self.__class__, self)._run(path_or_url)
 764         self._run(size=self['size_in_bytes'])
 765
 766
 767 @command(file_cmds)
 768 class file_overwrite(_pithos_container, _optional_output_cmd):
 769     """Overwrite part of a remote file"""
 770
 771     arguments = dict(
 772         progress_bar=ProgressBarArgument(
 773             'do not show progress bar', ('-N', '--no-progress-bar'),
 774             default=False),
 775         start_position=IntArgument('File position in bytes', '--from'),
 776         end_position=IntArgument('File position in bytes', '--to'),
 777     )
 778     required = ('start_position', 'end_position')
 779
 780     @errors.generic.all
 781     @errors.pithos.connection
 782     @errors.pithos.container
 783     @errors.pithos.object_path
 784     @errors.pithos.object_size
 785     def _run(self, local_path, start, end):
 786         start, end = int(start), int(end)
 787         (progress_bar, upload_cb) = self._safe_progress_bar(
 788             'Overwrite %s bytes' % (end - start))
 789         try:
 790             with open(path.abspath(local_path), 'rb') as f:
 791                 self._optional_output(self.client.overwrite_object(
 792                     obj=self.path,
 793                     start=start,
 794                     end=end,
 795                     source_file=f,
 796                     upload_cb=upload_cb))
 797         finally:
 798             self._safe_progress_bar_finish(progress_bar)
 799
 800     def main(self, local_path, path_or_url):
 801         super(self.__class__, self)._run(path_or_url)
 802         self.path = self.path or path.basename(local_path)
 803         self._run(
 804             local_path=local_path,
 805             start=self['start_position'],
 806             end=self['end_position'])
 807
 808
 809 @command(file_cmds)
 810 class file_upload(_pithos_container, _optional_output_cmd):
 811     """Upload a file"""
 812
 813     arguments = dict(
 814         max_threads=IntArgument('default: 5', '--threads'),
 815         content_encoding=ValueArgument(
 816             'set MIME content type', '--content-encoding'),
 817         content_disposition=ValueArgument(
 818             'specify objects presentation style', '--content-disposition'),
 819         content_type=ValueArgument('specify content type', '--content-type'),
 820         uuid_for_read_permission=RepeatableArgument(
 821             'Give read access to a user or group (can be repeated) '
 822             'Use * for all users',
 823             '--read-permission'),
 824         uuid_for_write_permission=RepeatableArgument(
 825             'Give write access to a user or group (can be repeated) '
 826             'Use * for all users',
 827             '--write-permission'),
 828         public=FlagArgument('make object publicly accessible', '--public'),
 829         progress_bar=ProgressBarArgument(
 830             'do not show progress bar',
 831             ('-N', '--no-progress-bar'),
 832             default=False),
 833         overwrite=FlagArgument('Force (over)write', ('-f', '--force')),
 834         recursive=FlagArgument(
 835             'Recursively upload directory *contents* + subdirectories',
 836             ('-r', '--recursive')),
 837         unchunked=FlagArgument(
 838             'Upload file as one block (not recommended)', '--unchunked'),
 839         md5_checksum=ValueArgument(
 840             'Confirm upload with a custom checksum (MD5)', '--etag'),
 841         use_hashes=FlagArgument(
 842             'Source file contains hashmap not data', '--source-is-hashmap'),
 843     )
 844
 845     def _sharing(self):
 846         sharing = dict()
 847         readlist = self['uuid_for_read_permission']
 848         if readlist:
 849             sharing['read'] = self['uuid_for_read_permission']
 850         writelist = self['uuid_for_write_permission']
 851         if writelist:
 852             sharing['write'] = self['uuid_for_write_permission']
 853         return sharing or None
 854
 855     def _check_container_limit(self, path):
 856         cl_dict = self.client.get_container_limit()
 857         container_limit = int(cl_dict['x-container-policy-quota'])
 858         r = self.client.container_get()
 859         used_bytes = sum(int(o['bytes']) for o in r.json)
 860         path_size = get_path_size(path)
 861         if container_limit and path_size > (container_limit - used_bytes):
 862             raise CLIError(
 863                 'Container %s (limit(%s) - used(%s)) < (size(%s) of %s)' % (
 864                     self.client.container,
 865                     format_size(container_limit),
 866                     format_size(used_bytes),
 867                     format_size(path_size),
 868                     path),
 869                 details=[
 870                     'Check accound limit: /file quota',
 871                     'Check container limit:',
 872                     '\t/file containerlimit get %s' % self.client.container,
 873                     'Increase container limit:',
 874                     '\t/file containerlimit set <new limit> %s' % (
 875                         self.client.container)])
 876
 877     def _src_dst(self, local_path, remote_path, objlist=None):
 878         lpath = path.abspath(local_path)
 879         short_path = path.basename(path.abspath(local_path))
 880         rpath = remote_path or short_path
 881         if path.isdir(lpath):
 882             if not self['recursive']:
 883                 raise CLIError('%s is a directory' % lpath, details=[
 884                     'Use %s to upload directories & contents' % (
 885                         self.arguments['recursive'].lvalue)])
 886             robj = self.client.container_get(path=rpath)
 887             if not self['overwrite']:
 888                 if robj.json:
 889                     raise CLIError(
 890                         'Objects/files prefixed as %s already exist' % rpath,
 891                         details=['Existing objects:'] + ['\t/%s/\t%s' % (
 892                             o['name'],
 893                             o['content_type'][12:]) for o in robj.json] + [
 894                             'Use -f to add, overwrite or resume'])
 895                 else:
 896                     try:
 897                         topobj = self.client.get_object_info(rpath)
 898                         if not self._is_dir(topobj):
 899                             raise CLIError(
 900                                 'Object /%s/%s exists but not a directory' % (
 901                                     self.container, rpath),
 902                                 details=['Use -f to overwrite'])
 903                     except ClientError as ce:
 904                         if ce.status not in (404, ):
 905                             raise
 906             self._check_container_limit(lpath)
 907             prev = ''
 908             for top, subdirs, files in walk(lpath):
 909                 if top != prev:
 910                     prev = top
 911                     try:
 912                         rel_path = rpath + top.split(lpath)[1]
 913                     except IndexError:
 914                         rel_path = rpath
 915                     self.error('mkdir /%s/%s' % (
 916                         self.client.container, rel_path))
 917                     self.client.create_directory(rel_path)
 918                 for f in files:
 919                     fpath = path.join(top, f)
 920                     if path.isfile(fpath):
 921                         rel_path = rel_path.replace(path.sep, '/')
 922                         pathfix = f.replace(path.sep, '/')
 923                         yield open(fpath, 'rb'), '%s/%s' % (rel_path, pathfix)
 924                     else:
 925                         self.error('%s is not a regular file' % fpath)
 926         else:
 927             if not path.isfile(lpath):
 928                 raise CLIError(('%s is not a regular file' % lpath) if (
 929                     path.exists(lpath)) else '%s does not exist' % lpath)
 930             try:
 931                 robj = self.client.get_object_info(rpath)
 932                 if remote_path and self._is_dir(robj):
 933                     rpath += '/%s' % (short_path.replace(path.sep, '/'))
 934                     self.client.get_object_info(rpath)
 935                 if not self['overwrite']:
 936                     raise CLIError(
 937                         'Object /%s/%s already exists' % (
 938                             self.container, rpath),
 939                         details=['use -f to overwrite / resume'])
 940             except ClientError as ce:
 941                 if ce.status not in (404, ):
 942                     raise
 943             self._check_container_limit(lpath)
 944             yield open(lpath, 'rb'), rpath
 945
 946     def _run(self, local_path, remote_path):
 947         self.client.MAX_THREADS = int(self['max_threads'] or 5)
 948         params = dict(
 949             content_encoding=self['content_encoding'],
 950             content_type=self['content_type'],
 951             content_disposition=self['content_disposition'],
 952             sharing=self._sharing(),
 953             public=self['public'])
 954         uploaded, container_info_cache = list, dict()
 955         rpref = 'pithos://%s' if self['account'] else ''
 956         for f, rpath in self._src_dst(local_path, remote_path):
 957             self.error('%s --> %s/%s/%s' % (
 958                 f.name, rpref, self.client.container, rpath))
 959             if not (self['content_type'] and self['content_encoding']):
 960                 ctype, cenc = guess_mime_type(f.name)
 961                 params['content_type'] = self['content_type'] or ctype
 962                 params['content_encoding'] = self['content_encoding'] or cenc
 963             if self['unchunked']:
 964                 r = self.client.upload_object_unchunked(
 965                     rpath, f,
 966                     etag=self['md5_checksum'], withHashFile=self['use_hashes'],
 967                     **params)
 968                 if self['with_output'] or self['json_output']:
 969                     r['name'] = '/%s/%s' % (self.client.container, rpath)
 970                     uploaded.append(r)
 971             else:
 972                 try:
 973                     (progress_bar, upload_cb) = self._safe_progress_bar(
 974                         'Uploading %s' % f.name.split(path.sep)[-1])
 975                     if progress_bar:
 976                         hash_bar = progress_bar.clone()
 977                         hash_cb = hash_bar.get_generator(
 978                             'Calculating block hashes')
 979                     else:
 980                         hash_cb = None
 981                     r = self.client.upload_object(
 982                         rpath, f,
 983                         hash_cb=hash_cb,
 984                         upload_cb=upload_cb,
 985                         container_info_cache=container_info_cache,
 986                         **params)
 987                     if self['with_output'] or self['json_output']:
 988                         r['name'] = '/%s/%s' % (self.client.container, rpath)
 989                         uploaded.append(r)
 990                 except Exception:
 991                     self._safe_progress_bar_finish(progress_bar)
 992                     raise
 993                 finally:
 994                     self._safe_progress_bar_finish(progress_bar)
 995         self._optional_output(uploaded)
 996         self.error('Upload completed')
 997
 998     def main(self, local_path, remote_path_or_url):
 999         super(self.__class__, self)._run(remote_path_or_url)
1000         remote_path = self.path or path.basename(path.abspath(local_path))
1001         self._run(local_path=local_path, remote_path=remote_path)
1002
1003
1004 class RangeArgument(ValueArgument):
1005     """
1006     :value type: string of the form <start>-<end> where <start> and <end> are
1007         integers
1008     :value returns: the input string, after type checking <start> and <end>
1009     """
1010
1011     @property
1012     def value(self):
1013         return getattr(self, '_value', self.default)
1014
1015     @value.setter
1016     def value(self, newvalues):
1017         if newvalues:
1018             self._value = getattr(self, '_value', self.default)
1019             for newvalue in newvalues.split(','):
1020                 self._value = ('%s,' % self._value) if self._value else ''
1021                 start, sep, end = newvalue.partition('-')
1022                 if sep:
1023                     if start:
1024                         start, end = (int(start), int(end))
1025                         if start > end:
1026                             raise CLIInvalidArgument(
1027                                 'Invalid range %s' % newvalue, details=[
1028                                 'Valid range formats',
1029                                 '  START-END', '  UP_TO', '  -FROM',
1030                                 'where all values are integers',
1031                                 'OR a compination (csv), e.g.,',
1032                                 '  %s=5,10-20,-5' % self.lvalue])
1033                         self._value += '%s-%s' % (start, end)
1034                     else:
1035                         self._value += '-%s' % int(end)
1036                 else:
1037                     self._value += '%s' % int(start)
1038
1039
1040 @command(file_cmds)
1041 class file_cat(_pithos_container):
1042     """Fetch remote file contents"""
1043
1044     arguments = dict(
1045         range=RangeArgument('show range of data e.g., 5,10-20,-5', '--range'),
1046         if_match=ValueArgument('show output if ETags match', '--if-match'),
1047         if_none_match=ValueArgument(
1048             'show output if ETags match', '--if-none-match'),
1049         if_modified_since=DateArgument(
1050             'show output modified since then', '--if-modified-since'),
1051         if_unmodified_since=DateArgument(
1052             'show output unmodified since then', '--if-unmodified-since'),
1053         object_version=ValueArgument(
1054             'Get contents of the chosen version', '--object-version')
1055     )
1056
1057     @errors.generic.all
1058     @errors.pithos.connection
1059     @errors.pithos.container
1060     @errors.pithos.object_path
1061     def _run(self):
1062         self.client.download_object(
1063             self.path, self._out,
1064             range_str=self['range'],
1065             version=self['object_version'],
1066             if_match=self['if_match'],
1067             if_none_match=self['if_none_match'],
1068             if_modified_since=self['if_modified_since'],
1069             if_unmodified_since=self['if_unmodified_since'])
1070         self._out.flush()
1071
1072     def main(self, path_or_url):
1073         super(self.__class__, self)._run(path_or_url)
1074         self._run()
1075
1076
1077 @command(file_cmds)
1078 class file_download(_pithos_container):
1079     """Download a remove file or directory object to local file system"""
1080
1081     arguments = dict(
1082         resume=FlagArgument(
1083             'Resume/Overwrite (attempt resume, else overwrite)',
1084             ('-f', '--resume')),
1085         range=RangeArgument(
1086             'Download only that range of data e.g., 5,10-20,-5', '--range'),
1087         matching_etag=ValueArgument('download iff ETag match', '--if-match'),
1088         non_matching_etag=ValueArgument(
1089             'download iff ETags DO NOT match', '--if-none-match'),
1090         modified_since_date=DateArgument(
1091             'download iff remote file is modified since then',
1092             '--if-modified-since'),
1093         unmodified_since_date=DateArgument(
1094             'show output iff remote file is unmodified since then',
1095             '--if-unmodified-since'),
1096         object_version=ValueArgument(
1097             'download a file of a specific version', '--object-version'),
1098         max_threads=IntArgument('default: 5', '--threads'),
1099         progress_bar=ProgressBarArgument(
1100             'do not show progress bar', ('-N', '--no-progress-bar'),
1101             default=False),
1102         recursive=FlagArgument(
1103             'Download a remote directory object and its contents',
1104             ('-r', '--recursive'))
1105         )
1106
1107     def _src_dst(self, local_path):
1108         """Create a list of (src, dst) where src is a remote location and dst
1109         is an open file descriptor. Directories are denoted as (None, dirpath)
1110         and they are pretended to other objects in a very strict order (shorter
1111         to longer path)."""
1112         ret = []
1113         try:
1114             if self.path:
1115                 obj = self.client.get_object_info(
1116                     self.path, version=self['object_version'])
1117                 obj.setdefault('name', self.path.strip('/'))
1118             else:
1119                 obj = None
1120         except ClientError as ce:
1121             if ce.status in (404, ):
1122                 raiseCLIError(ce, details=[
1123                     'To download an object, it must exist either as a file or'
1124                     ' as a directory.',
1125                     'For example, to download everything under prefix/ the '
1126                     'directory "prefix" must exist.',
1127                     'To see if an remote object is actually there:',
1128                     '  /file info [/CONTAINER/]OBJECT',
1129                     'To create a directory object:',
1130                     '  /file mkdir [/CONTAINER/]OBJECT'])
1131             if ce.status in (204, ):
1132                 raise CLIError(
1133                     'No file or directory objects to download',
1134                     details=[
1135                         'To download a container (e.g., %s):' % self.container,
1136                         '  [kamaki] container download %s [LOCAL_PATH]' % (
1137                             self.container)])
1138             raise
1139         rpath = self.path.strip('/')
1140         if local_path and self.path and local_path.endswith('/'):
1141             local_path = local_path[-1:]
1142
1143         if (not obj) or self._is_dir(obj):
1144             if self['recursive']:
1145                 if not (self.path or local_path.endswith('/')):
1146                     #  Download the whole container
1147                     local_path = '' if local_path in ('.', ) else local_path
1148                     local_path = '%s/' % (local_path or self.container)
1149                 obj = obj or dict(
1150                     name='', content_type='application/directory')
1151                 dirs, files = [obj, ], []
1152                 objects = self.client.container_get(
1153                     path=self.path,
1154                     if_modified_since=self['modified_since_date'],
1155                     if_unmodified_since=self['unmodified_since_date'])
1156                 for o in objects.json:
1157                     (dirs if self._is_dir(o) else files).append(o)
1158
1159                 #  Put the directories on top of the list
1160                 for dpath in sorted(['%s%s' % (
1161                         local_path, d['name'][len(rpath):]) for d in dirs]):
1162                     if path.exists(dpath):
1163                         if path.isdir(dpath):
1164                             continue
1165                         raise CLIError(
1166                             'Cannot replace local file %s with a directory '
1167                             'of the same name' % dpath,
1168                             details=[
1169                                 'Either remove the file or specify a'
1170                                 'different target location'])
1171                     ret.append((None, dpath, None))
1172
1173                 #  Append the file objects
1174                 for opath in [o['name'] for o in files]:
1175                     lpath = '%s%s' % (local_path, opath[len(rpath):])
1176                     if self['resume']:
1177                         fxists = path.exists(lpath)
1178                         if fxists and path.isdir(lpath):
1179                             raise CLIError(
1180                                 'Cannot change local dir %s info file' % (
1181                                     lpath),
1182                                 details=[
1183                                     'Either remove the file or specify a'
1184                                     'different target location'])
1185                         ret.append((opath, lpath, fxists))
1186                     elif path.exists(lpath):
1187                         raise CLIError(
1188                             'Cannot overwrite %s' % lpath,
1189                             details=['To overwrite/resume, use  %s' % (
1190                                 self.arguments['resume'].lvalue)])
1191                     else:
1192                         ret.append((opath, lpath, None))
1193             elif self.path:
1194                 raise CLIError(
1195                     'Remote object /%s/%s is a directory' % (
1196                         self.container, local_path),
1197                     details=['Use %s to download directories' % (
1198                         self.arguments['recursive'].lvalue)])
1199             else:
1200                 parsed_name = self.arguments['recursive'].lvalue
1201                 raise CLIError(
1202                     'Cannot download container %s' % self.container,
1203                     details=[
1204                         'Use %s to download containers' % parsed_name,
1205                         '  [kamaki] file download %s /%s [LOCAL_PATH]' % (
1206                             parsed_name, self.container)])
1207         else:
1208             #  Remote object is just a file
1209             if path.exists(local_path):
1210                 if not self['resume']:
1211                     raise CLIError(
1212                         'Cannot overwrite local file %s' % (local_path),
1213                         details=['To overwrite/resume, use  %s' % (
1214                             self.arguments['resume'].lvalue)])
1215             elif '/' in local_path[1:-1]:
1216                 dirs = [p for p in local_path.split('/') if p]
1217                 pref = '/' if local_path.startswith('/') else ''
1218                 for d in dirs[:-1]:
1219                     pref += d
1220                     if not path.exists(pref):
1221                         ret.append((None, d, None))
1222                     elif not path.isdir(pref):
1223                         raise CLIError(
1224                             'Failed to use %s as a destination' % local_path,
1225                             importance=3,
1226                             details=[
1227                                 'Local file %s is not a directory' % pref,
1228                                 'Destination prefix must consist of '
1229                                 'directories or non-existing names',
1230                                 'Either remove the file, or choose another '
1231                                 'destination'])
1232             ret.append((rpath, local_path, self['resume']))
1233         for r, l, resume in ret:
1234             if r:
1235                 with open(l, 'rwb+' if resume else 'wb+') as f:
1236                     yield (r, f)
1237             else:
1238                 yield (r, l)
1239
1240     @errors.generic.all
1241     @errors.pithos.connection
1242     @errors.pithos.container
1243     @errors.pithos.object_path
1244     @errors.pithos.local_path
1245     @errors.pithos.local_path_download
1246     def _run(self, local_path):
1247         self.client.MAX_THREADS = int(self['max_threads'] or 5)
1248         progress_bar = None
1249         try:
1250             for rpath, output_file in self._src_dst(local_path):
1251                 if not rpath:
1252                     self.error('Create local directory %s' % output_file)
1253                     makedirs(output_file)
1254                     continue
1255                 self.error('/%s/%s --> %s' % (
1256                     self.container, rpath, output_file.name))
1257                 progress_bar, download_cb = self._safe_progress_bar(
1258                     '  download')
1259                 self.client.download_object(
1260                     rpath, output_file,
1261                     download_cb=download_cb,
1262                     range_str=self['range'],
1263                     version=self['object_version'],
1264                     if_match=self['matching_etag'],
1265                     resume=self['resume'],
1266                     if_none_match=self['non_matching_etag'],
1267                     if_modified_since=self['modified_since_date'],
1268                     if_unmodified_since=self['unmodified_since_date'])
1269         except KeyboardInterrupt:
1270             from threading import activeCount, enumerate as activethreads
1271             timeout = 0.5
1272             while activeCount() > 1:
1273                 self._out.write('\nCancel %s threads: ' % (activeCount() - 1))
1274                 self._out.flush()
1275                 for thread in activethreads():
1276                     try:
1277                         thread.join(timeout)
1278                         self._out.write('.' if thread.isAlive() else '*')
1279                     except RuntimeError:
1280                         continue
1281                     finally:
1282                         self._out.flush()
1283                         timeout += 0.1
1284             self.error('\nDownload canceled by user')
1285             if local_path is not None:
1286                 self.error('to resume, re-run with --resume')
1287         finally:
1288             self._safe_progress_bar_finish(progress_bar)
1289
1290     def main(self, remote_path_or_url, local_path=None):
1291         super(self.__class__, self)._run(remote_path_or_url)
1292         local_path = local_path or self.path or '.'
1293         self._run(local_path=local_path)
1294
1295
1296 @command(container_cmds)
1297 class container_info(_pithos_account, _optional_json):
1298     """Get information about a container"""
1299
1300     arguments = dict(
1301         until_date=DateArgument('show metadata until then', '--until'),
1302         metadata=FlagArgument('Show only container metadata', '--metadata'),
1303         sizelimit=FlagArgument(
1304             'Show the maximum size limit for container', '--size-limit'),
1305         in_bytes=FlagArgument('Show size limit in bytes', ('-b', '--bytes'))
1306     )
1307
1308     @errors.generic.all
1309     @errors.pithos.connection
1310     @errors.pithos.container
1311     @errors.pithos.object_path
1312     def _run(self):
1313         if self['metadata']:
1314             r, preflen = dict(), len('x-container-meta-')
1315             for k, v in self.client.get_container_meta(
1316                     until=self['until_date']).items():
1317                 r[k[preflen:]] = v
1318         elif self['sizelimit']:
1319             r = self.client.get_container_limit(
1320                 self.container)['x-container-policy-quota']
1321             r = {'size limit': 'unlimited' if r in ('0', ) else (
1322                 int(r) if self['in_bytes'] else format_size(r))}
1323         else:
1324             r = self.client.get_container_info(self.container)
1325         self._print(r, self.print_dict)
1326
1327     def main(self, container):
1328         super(self.__class__, self)._run()
1329         self.container, self.client.container = container, container
1330         self._run()
1331
1332
1333 class VersioningArgument(ValueArgument):
1334
1335     schemes = ('auto', 'none')
1336
1337     @property
1338     def value(self):
1339         return getattr(self, '_value', None)
1340
1341     @value.setter
1342     def value(self, new_scheme):
1343         if new_scheme:
1344             new_scheme = new_scheme.lower()
1345             if new_scheme not in self.schemes:
1346                 raise CLIInvalidArgument('Invalid versioning value', details=[
1347                     'Valid versioning values are %s' % ', '.join(
1348                         self.schemes)])
1349             self._value = new_scheme
1350
1351
1352 @command(container_cmds)
1353 class container_modify(_pithos_account, _optional_json):
1354     """Modify the properties of a container"""
1355
1356     arguments = dict(
1357         metadata_to_add=KeyValueArgument(
1358             'Add metadata in the form KEY=VALUE (can be repeated)',
1359             '--metadata-add'),
1360         metadata_to_delete=RepeatableArgument(
1361             'Delete metadata by KEY (can be repeated)', '--metadata-del'),
1362         sizelimit=DataSizeArgument(
1363             'Set max size limit (0 for unlimited, '
1364             'use units B, KiB, KB, etc.)', '--size-limit'),
1365         versioning=VersioningArgument(
1366             'Set a versioning scheme (%s)' % ', '.join(
1367                 VersioningArgument.schemes), '--versioning')
1368     )
1369     required = [
1370         'metadata_to_add', 'metadata_to_delete', 'sizelimit', 'versioning']
1371
1372     @errors.generic.all
1373     @errors.pithos.connection
1374     @errors.pithos.container
1375     def _run(self, container):
1376         metadata = self['metadata_to_add']
1377         for k in (self['metadata_to_delete'] or []):
1378             metadata[k] = ''
1379         if metadata:
1380             self.client.set_container_meta(metadata)
1381             self._print(self.client.get_container_meta(), self.print_dict)
1382         if self['sizelimit'] is not None:
1383             self.client.set_container_limit(self['sizelimit'])
1384             r = self.client.get_container_limit()['x-container-policy-quota']
1385             r = 'unlimited' if r in ('0', ) else format_size(r)
1386             self.writeln('new size limit: %s' % r)
1387         if self['versioning']:
1388             self.client.set_container_versioning(self['versioning'])
1389             self.writeln('new versioning scheme: %s' % (
1390                 self.client.get_container_versioning(self.container)[
1391                     'x-container-policy-versioning']))
1392
1393     def main(self, container):
1394         super(self.__class__, self)._run()
1395         self.client.container, self.container = container, container
1396         self._run(container=container)
1397
1398
1399 @command(container_cmds)
1400 class container_list(_pithos_account, _optional_json, _name_filter):
1401     """List all containers, or their contents"""
1402
1403     arguments = dict(
1404         detail=FlagArgument('Containers with details', ('-l', '--list')),
1405         limit=IntArgument('limit number of listed items', ('-n', '--number')),
1406         marker=ValueArgument('output greater that marker', '--marker'),
1407         modified_since_date=ValueArgument(
1408             'show output modified since then', '--if-modified-since'),
1409         unmodified_since_date=ValueArgument(
1410             'show output not modified since then', '--if-unmodified-since'),
1411         until_date=DateArgument('show metadata until then', '--until'),
1412         shared=FlagArgument('show only shared', '--shared'),
1413         more=FlagArgument('read long results', '--more'),
1414         enum=FlagArgument('Enumerate results', '--enumerate'),
1415         recursive=FlagArgument(
1416             'Recursively list containers and their contents',
1417             ('-r', '--recursive')),
1418         shared_by_me=FlagArgument(
1419             'show only files shared to other users', '--shared-by-me'),
1420         public=FlagArgument('show only published objects', '--public'),
1421     )
1422
1423     def print_containers(self, container_list):
1424         for index, container in enumerate(container_list):
1425             if 'bytes' in container:
1426                 size = format_size(container['bytes'])
1427             prfx = ('%s. ' % (index + 1)) if self['enum'] else ''
1428             _cname = container['name'] if (
1429                 self['more']) else bold(container['name'])
1430             cname = u'%s%s' % (prfx, _cname)
1431             if self['detail']:
1432                 self.writeln(cname)
1433                 pretty_c = container.copy()
1434                 if 'bytes' in container:
1435                     pretty_c['bytes'] = '%s (%s)' % (container['bytes'], size)
1436                 self.print_dict(pretty_c, exclude=('name'))
1437                 self.writeln()
1438             else:
1439                 if 'count' in container and 'bytes' in container:
1440                     self.writeln('%s (%s, %s objects)' % (
1441                         cname, size, container['count']))
1442                 else:
1443                     self.writeln(cname)
1444             objects = container.get('objects', [])
1445             if objects:
1446                 self.print_objects(objects)
1447                 self.writeln('')
1448
1449     def _create_object_forest(self, container_list):
1450         try:
1451             for container in container_list:
1452                 self.client.container = container['name']
1453                 objects = self.client.container_get(
1454                     limit=False if self['more'] else self['limit'],
1455                     if_modified_since=self['modified_since_date'],
1456                     if_unmodified_since=self['unmodified_since_date'],
1457                     until=self['until_date'],
1458                     show_only_shared=self['shared_by_me'],
1459                     public=self['public'])
1460                 container['objects'] = objects.json
1461         finally:
1462             self.client.container = None
1463
1464     @errors.generic.all
1465     @errors.pithos.connection
1466     @errors.pithos.object_path
1467     @errors.pithos.container
1468     def _run(self, container):
1469         if container:
1470             r = self.client.container_get(
1471                 limit=False if self['more'] else self['limit'],
1472                 marker=self['marker'],
1473                 if_modified_since=self['modified_since_date'],
1474                 if_unmodified_since=self['unmodified_since_date'],
1475                 until=self['until_date'],
1476                 show_only_shared=self['shared_by_me'],
1477                 public=self['public'])
1478         else:
1479             r = self.client.account_get(
1480                 limit=False if self['more'] else self['limit'],
1481                 marker=self['marker'],
1482                 if_modified_since=self['modified_since_date'],
1483                 if_unmodified_since=self['unmodified_since_date'],
1484                 until=self['until_date'],
1485                 show_only_shared=self['shared_by_me'],
1486                 public=self['public'])
1487         files = self._filter_by_name(r.json)
1488         if self['recursive'] and not container:
1489             self._create_object_forest(files)
1490         if self['more']:
1491             outbu, self._out = self._out, StringIO()
1492         try:
1493             if self['json_output'] or self['output_format']:
1494                 self._print(files)
1495             else:
1496                 (self.print_objects if container else self.print_containers)(
1497                     files)
1498         finally:
1499             if self['more']:
1500                 pager(self._out.getvalue())
1501                 self._out = outbu
1502
1503     def main(self, container=None):
1504         super(self.__class__, self)._run()
1505         self.client.container, self.container = container, container
1506         self._run(container)
1507
1508
1509 @command(container_cmds)
1510 class container_create(_pithos_account):
1511     """Create a new container"""
1512
1513     arguments = dict(
1514         versioning=ValueArgument(
1515             'set container versioning (auto/none)', '--versioning'),
1516         limit=IntArgument('set default container limit', '--limit'),
1517         meta=KeyValueArgument(
1518             'set container metadata (can be repeated)', '--meta')
1519     )
1520
1521     @errors.generic.all
1522     @errors.pithos.connection
1523     @errors.pithos.container
1524     def _run(self, container):
1525         try:
1526             self.client.create_container(
1527                 container=container,
1528                 sizelimit=self['limit'],
1529                 versioning=self['versioning'],
1530                 metadata=self['meta'],
1531                 success=(201, ))
1532         except ClientError as ce:
1533             if ce.status in (202, ):
1534                 raise CLIError(
1535                     'Container %s alread exists' % container, details=[
1536                     'Either delete %s or choose another name' % (container)])
1537             raise
1538
1539     def main(self, new_container):
1540         super(self.__class__, self)._run()
1541         self._run(container=new_container)
1542
1543
1544 @command(container_cmds)
1545 class container_delete(_pithos_account):
1546     """Delete a container"""
1547
1548     arguments = dict(
1549         yes=FlagArgument('Do not prompt for permission', '--yes'),
1550         recursive=FlagArgument(
1551             'delete container even if not empty', ('-r', '--recursive'))
1552     )
1553
1554     @errors.generic.all
1555     @errors.pithos.connection
1556     @errors.pithos.container
1557     def _run(self, container):
1558         num_of_contents = int(self.client.get_container_info(container)[
1559             'x-container-object-count'])
1560         delimiter, msg = None, 'Delete container %s ?' % container
1561         if self['recursive']:
1562             delimiter, msg = '/', 'Empty and d%s' % msg[1:]
1563         elif num_of_contents:
1564             raise CLIError('Container %s is not empty' % container, details=[
1565                 'Use %s to delete non-empty containers' % (
1566                     self.arguments['recursive'].lvalue)])
1567         if self['yes'] or self.ask_user(msg):
1568             if num_of_contents:
1569                 self.client.del_container(delimiter=delimiter)
1570             self.client.purge_container()
1571
1572     def main(self, container):
1573         super(self.__class__, self)._run()
1574         self.container, self.client.container = container, container
1575         self._run(container)
1576
1577
1578 @command(container_cmds)
1579 class container_empty(_pithos_account):
1580     """Empty a container"""
1581
1582     arguments = dict(yes=FlagArgument('Do not prompt for permission', '--yes'))
1583
1584     @errors.generic.all
1585     @errors.pithos.connection
1586     @errors.pithos.container
1587     def _run(self, container):
1588         if self['yes'] or self.ask_user('Empty container %s ?' % container):
1589             self.client.del_container(delimiter='/')
1590
1591     def main(self, container):
1592         super(self.__class__, self)._run()
1593         self.container, self.client.container = container, container
1594         self._run(container)
1595
1596
1597 @command(sharer_cmds)
1598 class sharer_list(_pithos_account, _optional_json):
1599     """List accounts who share file objects with current user"""
1600
1601     arguments = dict(
1602         detail=FlagArgument('show detailed output', ('-l', '--details')),
1603         marker=ValueArgument('show output greater then marker', '--marker')
1604     )
1605
1606     @errors.generic.all
1607     @errors.pithos.connection
1608     def _run(self):
1609         accounts = self.client.get_sharing_accounts(marker=self['marker'])
1610         if not (self['json_output'] or self['output_format']):
1611             usernames = self._uuids2usernames(
1612                 [acc['name'] for acc in accounts])
1613             for item in accounts:
1614                 uuid = item['name']
1615                 item['id'], item['name'] = uuid, usernames[uuid]
1616                 if not self['detail']:
1617                     item.pop('last_modified')
1618         self._print(accounts)
1619
1620     def main(self):
1621         super(self.__class__, self)._run()
1622         self._run()
1623
1624
1625 @command(sharer_cmds)
1626 class sharer_info(_pithos_account, _optional_json):
1627     """Details on a Pithos+ sharer account (default: current account)"""
1628
1629     @errors.generic.all
1630     @errors.pithos.connection
1631     def _run(self):
1632         self._print(self.client.get_account_info(), self.print_dict)
1633
1634     def main(self, account_uuid=None):
1635         super(self.__class__, self)._run()
1636         if account_uuid:
1637             self.client.account, self.account = account_uuid, account_uuid
1638         self._run()
1639
1640
1641 class _pithos_group(_pithos_account):
1642     prefix = 'x-account-group-'
1643     preflen = len(prefix)
1644
1645     def _groups(self):
1646         groups = dict()
1647         for k, v in self.client.get_account_group().items():
1648             groups[k[self.preflen:]] = v
1649         return groups
1650
1651
1652 @command(group_cmds)
1653 class group_list(_pithos_group, _optional_json):
1654     """list all groups and group members"""
1655
1656     @errors.generic.all
1657     @errors.pithos.connection
1658     def _run(self):
1659         self._print(self._groups(), self.print_dict)
1660
1661     def main(self):
1662         super(self.__class__, self)._run()
1663         self._run()
1664
1665
1666 @command(group_cmds)
1667 class group_create(_pithos_group, _optional_json):
1668     """Create a group of users"""
1669
1670     arguments = dict(
1671         user_uuid=RepeatableArgument('Add a user to the group', '--uuid'),
1672         username=RepeatableArgument('Add a user to the group', '--username')
1673     )
1674     required = ['user_uuid', 'username']
1675
1676     @errors.generic.all
1677     @errors.pithos.connection
1678     def _run(self, groupname, *users):
1679         if groupname in self._groups() and not self.ask_user(
1680                 'Group %s already exists, overwrite?' % groupname):
1681             self.error('Aborted')
1682             return
1683         self.client.set_account_group(groupname, users)
1684         self._print(self._groups(), self.print_dict)
1685
1686     def main(self, groupname):
1687         super(self.__class__, self)._run()
1688         users = (self['user_uuid'] or []) + self._usernames2uuids(
1689             self['username'] or []).values()
1690         if users:
1691             self._run(groupname, *users)
1692         else:
1693             raise CLISyntaxError(
1694                 'No valid users specified, use %s or %s' % (
1695                     self.arguments['user_uuid'].lvalue,
1696                     self.arguments['username'].lvalue),
1697                 details=[
1698                     'Check if a username or uuid is valid with',
1699                     '  user uuid2username', 'OR', '  user username2uuid'])
1700
1701
1702 @command(group_cmds)
1703 class group_delete(_pithos_group, _optional_json):
1704     """Delete a user group"""
1705
1706     @errors.generic.all
1707     @errors.pithos.connection
1708     def _run(self, groupname):
1709         self.client.del_account_group(groupname)
1710         self._print(self._groups(), self.print_dict)
1711
1712     def main(self, groupname):
1713         super(self.__class__, self)._run()
1714         self._run(groupname)
1715
1716
1717 #  Deprecated commands
1718
1719 @command(file_cmds)
1720 class file_publish(_pithos_init):
1721     """DEPRECATED, replaced by [kamaki] file modify OBJECT --publish"""
1722
1723     def main(self, *args):
1724         raise CLISyntaxError('DEPRECATED', details=[
1725             'This command is replaced by:',
1726             '  [kamaki] file modify OBJECT --publish'])
1727
1728
1729 @command(file_cmds)
1730 class file_unpublish(_pithos_init):
1731     """DEPRECATED, replaced by [kamaki] file modify OBJECT --unpublish"""
1732
1733     def main(self, *args):
1734         raise CLISyntaxError('DEPRECATED', details=[
1735             'This command is replaced by:',
1736             '  [kamaki] file modify OBJECT --unpublish'])