code.grnet.gr Git - kamaki/blob - kamaki/cli/commands/pithos.py

   1 # Copyright 2011-2013 GRNET S.A. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or
   4 # without modification, are permitted provided that the following
   5 # conditions are met:
   6 #
   7 #   1. Redistributions of source code must retain the above
   8 #      copyright notice, this list of conditions and the following
   9 #      disclaimer.
  10 #
  11 #   2. Redistributions in binary form must reproduce the above
  12 #      copyright notice, this list of conditions and the following
  13 #      disclaimer in the documentation and/or other materials
  14 #      provided with the distribution.
  15 #
  16 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  17 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  23 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  24 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  26 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27 # POSSIBILITY OF SUCH DAMAGE.
  28 #
  29 # The views and conclusions contained in the software and
  30 # documentation are those of the authors and should not be
  31 # interpreted as representing official policies, either expressed
  32 # or implied, of GRNET S.A.command
  33
  34 from time import localtime, strftime
  35 from io import StringIO
  36 from pydoc import pager
  37 from os import path, walk, makedirs
  38
  39 from kamaki.clients.pithos import PithosClient, ClientError
  40
  41 from kamaki.cli import command
  42 from kamaki.cli.command_tree import CommandTree
  43 from kamaki.cli.commands import (
  44     _command_init, errors, addLogSettings, DontRaiseKeyError, _optional_json,
  45     _name_filter, _optional_output_cmd)
  46 from kamaki.cli.errors import (
  47     CLIBaseUrlError, CLIError, CLIInvalidArgument, raiseCLIError,
  48     CLISyntaxError)
  49 from kamaki.cli.argument import (
  50     FlagArgument, IntArgument, ValueArgument, DateArgument, KeyValueArgument,
  51     ProgressBarArgument, RepeatableArgument, DataSizeArgument)
  52 from kamaki.cli.utils import (
  53     format_size, bold, get_path_size, guess_mime_type)
  54
  55 file_cmds = CommandTree('file', 'Pithos+/Storage object level API commands')
  56 container_cmds = CommandTree(
  57     'container', 'Pithos+/Storage container level API commands')
  58 sharer_cmds = CommandTree('sharer', 'Pithos+/Storage sharers')
  59 group_cmds = CommandTree('group', 'Pithos+/Storage user groups')
  60 _commands = [file_cmds, container_cmds, sharer_cmds, group_cmds]
  61
  62
  63 class _pithos_init(_command_init):
  64     """Initilize a pithos+ client
  65     There is always a default account (current user uuid)
  66     There is always a default container (pithos)
  67     """
  68
  69     @DontRaiseKeyError
  70     def _custom_container(self):
  71         return self.config.get_cloud(self.cloud, 'pithos_container')
  72
  73     @DontRaiseKeyError
  74     def _custom_uuid(self):
  75         return self.config.get_cloud(self.cloud, 'pithos_uuid')
  76
  77     def _set_account(self):
  78         self.account = self._custom_uuid()
  79         if self.account:
  80             return
  81         astakos = getattr(self, 'auth_base', None)
  82         if astakos:
  83             self.account = astakos.user_term('id', self.token)
  84         else:
  85             raise CLIBaseUrlError(service='astakos')
  86
  87     @errors.generic.all
  88     @addLogSettings
  89     def _run(self):
  90         cloud = getattr(self, 'cloud', None)
  91         if cloud:
  92             self.base_url = self._custom_url('pithos')
  93         else:
  94             self.cloud = 'default'
  95         self.token = self._custom_token('pithos')
  96         self.container = self._custom_container() or 'pithos'
  97
  98         astakos = getattr(self, 'auth_base', None)
  99         if astakos:
 100             self.token = self.token or astakos.token
 101             if not self.base_url:
 102                 pithos_endpoints = astakos.get_service_endpoints(
 103                     self._custom_type('pithos') or 'object-store',
 104                     self._custom_version('pithos') or '')
 105                 self.base_url = pithos_endpoints['publicURL']
 106         else:
 107             raise CLIBaseUrlError(service='astakos')
 108
 109         self._set_account()
 110         self.client = PithosClient(
 111             self.base_url, self.token, self.account, self.container)
 112
 113     def main(self):
 114         self._run()
 115
 116
 117 class _pithos_account(_pithos_init):
 118     """Setup account"""
 119
 120     def __init__(self, arguments={}, auth_base=None, cloud=None):
 121         super(_pithos_account, self).__init__(arguments, auth_base, cloud)
 122         self['account'] = ValueArgument(
 123             'Use (a different) user uuid', ('-A', '--account'))
 124
 125     def print_objects(self, object_list):
 126         for index, obj in enumerate(object_list):
 127             pretty_obj = obj.copy()
 128             index += 1
 129             empty_space = ' ' * (len(str(len(object_list))) - len(str(index)))
 130             if 'subdir' in obj:
 131                 continue
 132             if self._is_dir(obj):
 133                 size = 'D'
 134             else:
 135                 size = format_size(obj['bytes'])
 136                 pretty_obj['bytes'] = '%s (%s)' % (obj['bytes'], size)
 137             oname = obj['name'] if self['more'] else bold(obj['name'])
 138             prfx = ('%s%s. ' % (empty_space, index)) if self['enum'] else ''
 139             if self['detail']:
 140                 self.writeln('%s%s' % (prfx, oname))
 141                 self.print_dict(pretty_obj, exclude=('name'))
 142                 self.writeln()
 143             else:
 144                 oname = '%s%9s %s' % (prfx, size, oname)
 145                 oname += '/' if self._is_dir(obj) else u''
 146                 self.writeln(oname)
 147
 148     @staticmethod
 149     def _is_dir(remote_dict):
 150         return 'application/directory' == remote_dict.get(
 151             'content_type', remote_dict.get('content-type', ''))
 152
 153     def _run(self):
 154         super(_pithos_account, self)._run()
 155         self.client.account = self['account'] or getattr(
 156             self, 'account', getattr(self.client, 'account', None))
 157
 158
 159 class _pithos_container(_pithos_account):
 160     """Setup container"""
 161
 162     def __init__(self, arguments={}, auth_base=None, cloud=None):
 163         super(_pithos_container, self).__init__(arguments, auth_base, cloud)
 164         self['container'] = ValueArgument(
 165             'Use this container (default: pithos)', ('-C', '--container'))
 166
 167     @staticmethod
 168     def _resolve_pithos_url(url):
 169         """Match urls of one of the following formats:
 170         pithos://ACCOUNT/CONTAINER/OBJECT_PATH
 171         /CONTAINER/OBJECT_PATH
 172         return account, container, path
 173         """
 174         account, container, obj_path, prefix = '', '', url, 'pithos://'
 175         if url.startswith(prefix):
 176             account, sep, url = url[len(prefix):].partition('/')
 177             url = '/%s' % url
 178         if url.startswith('/'):
 179             container, sep, obj_path = url[1:].partition('/')
 180         return account, container, obj_path
 181
 182     def _run(self, url=None):
 183         acc, con, self.path = self._resolve_pithos_url(url or '')
 184         #  self.account = acc or getattr(self, 'account', '')
 185         super(_pithos_container, self)._run()
 186         self.container = con or self['container'] or getattr(
 187             self, 'container', None) or getattr(self.client, 'container', '')
 188         self.client.account = acc or self.client.account
 189         self.client.container = self.container
 190
 191
 192 @command(file_cmds)
 193 class file_info(_pithos_container, _optional_json):
 194     """Get information/details about a file"""
 195
 196     arguments = dict(
 197         object_version=ValueArgument(
 198             'download a file of a specific version', '--object-version'),
 199         hashmap=FlagArgument(
 200             'Get file hashmap instead of details', '--hashmap'),
 201         matching_etag=ValueArgument(
 202             'show output if ETags match', '--if-match'),
 203         non_matching_etag=ValueArgument(
 204             'show output if ETags DO NOT match', '--if-none-match'),
 205         modified_since_date=DateArgument(
 206             'show output modified since then', '--if-modified-since'),
 207         unmodified_since_date=DateArgument(
 208             'show output unmodified since then', '--if-unmodified-since'),
 209         sharing=FlagArgument(
 210             'show object permissions and sharing information', '--sharing'),
 211         metadata=FlagArgument('show only object metadata', '--metadata'),
 212         versions=FlagArgument(
 213             'show the list of versions for the file', '--object-versions')
 214     )
 215
 216     def version_print(self, versions):
 217         return {'/%s/%s' % (self.container, self.path): [
 218             dict(version_id=vitem[0], created=strftime(
 219                 '%d-%m-%Y %H:%M:%S',
 220                 localtime(float(vitem[1])))) for vitem in versions]}
 221
 222     @errors.generic.all
 223     @errors.pithos.connection
 224     @errors.pithos.container
 225     @errors.pithos.object_path
 226     def _run(self):
 227         if self['hashmap']:
 228             r = self.client.get_object_hashmap(
 229                 self.path,
 230                 version=self['object_version'],
 231                 if_match=self['matching_etag'],
 232                 if_none_match=self['non_matching_etag'],
 233                 if_modified_since=self['modified_since_date'],
 234                 if_unmodified_since=self['unmodified_since_date'])
 235         elif self['sharing']:
 236             r = self.client.get_object_sharing(self.path)
 237             r['public url'] = self.client.get_object_info(
 238                 self.path, version=self['object_version']).get(
 239                     'x-object-public', None)
 240         elif self['metadata']:
 241             r, preflen = dict(), len('x-object-meta-')
 242             for k, v in self.client.get_object_meta(self.path).items():
 243                 r[k[preflen:]] = v
 244         elif self['versions']:
 245             r = self.version_print(
 246                 self.client.get_object_versionlist(self.path))
 247         else:
 248             r = self.client.get_object_info(
 249                 self.path, version=self['object_version'])
 250         self._print(r, self.print_dict)
 251
 252     def main(self, path_or_url):
 253         super(self.__class__, self)._run(path_or_url)
 254         self._run()
 255
 256
 257 @command(file_cmds)
 258 class file_list(_pithos_container, _optional_json, _name_filter):
 259     """List all objects in a container or a directory object"""
 260
 261     arguments = dict(
 262         detail=FlagArgument('detailed output', ('-l', '--list')),
 263         limit=IntArgument('limit number of listed items', ('-n', '--number')),
 264         marker=ValueArgument('output greater that marker', '--marker'),
 265         delimiter=ValueArgument('show output up to delimiter', '--delimiter'),
 266         meta=ValueArgument(
 267             'show output with specified meta keys', '--meta',
 268             default=[]),
 269         if_modified_since=ValueArgument(
 270             'show output modified since then', '--if-modified-since'),
 271         if_unmodified_since=ValueArgument(
 272             'show output not modified since then', '--if-unmodified-since'),
 273         until=DateArgument('show metadata until then', '--until'),
 274         format=ValueArgument(
 275             'format to parse until data (default: d/m/Y H:M:S )', '--format'),
 276         shared_by_me=FlagArgument(
 277             'show only files shared to other users', '--shared-by-me'),
 278         public=FlagArgument('show only published objects', '--public'),
 279         more=FlagArgument('read long results', '--more'),
 280         enum=FlagArgument('Enumerate results', '--enumerate'),
 281         recursive=FlagArgument(
 282             'Recursively list containers and their contents',
 283             ('-R', '--recursive'))
 284     )
 285
 286     @errors.generic.all
 287     @errors.pithos.connection
 288     @errors.pithos.container
 289     @errors.pithos.object_path
 290     def _run(self):
 291         r = self.client.container_get(
 292             limit=False if self['more'] else self['limit'],
 293             marker=self['marker'],
 294             prefix=self['name_pref'],
 295             delimiter=self['delimiter'],
 296             path=self.path or '',
 297             show_only_shared=self['shared_by_me'],
 298             public=self['public'],
 299             if_modified_since=self['if_modified_since'],
 300             if_unmodified_since=self['if_unmodified_since'],
 301             until=self['until'],
 302             meta=self['meta'])
 303
 304         #  REMOVE THIS if version >> 0.12
 305         if not r.json:
 306             self.error('  NOTE: Since v0.12, use / for containers e.g.,')
 307             self.error('    [kamaki] file list /pithos')
 308
 309         files = self._filter_by_name(r.json)
 310         if self['more']:
 311             outbu, self._out = self._out, StringIO()
 312         try:
 313             if self['json_output'] or self['output_format']:
 314                 self._print(files)
 315             else:
 316                 self.print_objects(files)
 317         finally:
 318             if self['more']:
 319                 pager(self._out.getvalue())
 320                 self._out = outbu
 321
 322     def main(self, path_or_url=''):
 323         super(self.__class__, self)._run(path_or_url)
 324         self._run()
 325
 326
 327 @command(file_cmds)
 328 class file_modify(_pithos_container):
 329     """Modify the attributes of a file or directory object"""
 330
 331     arguments = dict(
 332         publish=FlagArgument(
 333             'Make an object public (returns the public URL)', '--publish'),
 334         unpublish=FlagArgument(
 335             'Make an object unpublic', '--unpublish'),
 336         uuid_for_read_permission=RepeatableArgument(
 337             'Give read access to user/group (can be repeated, accumulative). '
 338             'Format for users: UUID . Format for groups: UUID:GROUP . '
 339             'Use * for all users/groups', '--read-permission'),
 340         uuid_for_write_permission=RepeatableArgument(
 341             'Give write access to user/group (can be repeated, accumulative). '
 342             'Format for users: UUID . Format for groups: UUID:GROUP . '
 343             'Use * for all users/groups', '--write-permission'),
 344         no_permissions=FlagArgument('Remove permissions', '--no-permissions'),
 345         metadata_to_set=KeyValueArgument(
 346             'Add metadata (KEY=VALUE) to an object (can be repeated)',
 347             '--metadata-add'),
 348         metadata_key_to_delete=RepeatableArgument(
 349             'Delete object metadata (can be repeated)', '--metadata-del'),
 350     )
 351     required = [
 352         'publish', 'unpublish', 'uuid_for_read_permission', 'metadata_to_set',
 353         'uuid_for_write_permission', 'no_permissions',
 354         'metadata_key_to_delete']
 355
 356     @errors.generic.all
 357     @errors.pithos.connection
 358     @errors.pithos.container
 359     @errors.pithos.object_path
 360     def _run(self):
 361         if self['publish']:
 362             self.writeln(self.client.publish_object(self.path))
 363         if self['unpublish']:
 364             self.client.unpublish_object(self.path)
 365         if self['uuid_for_read_permission'] or self[
 366                 'uuid_for_write_permission']:
 367             perms = self.client.get_object_sharing(self.path)
 368             read, write = perms.get('read', ''), perms.get('write', '')
 369             read = read.split(',') if read else []
 370             write = write.split(',') if write else []
 371             read += (self['uuid_for_read_permission'] or [])
 372             write += (self['uuid_for_write_permission'] or [])
 373             self.client.set_object_sharing(
 374                 self.path, read_permission=read, write_permission=write)
 375             self.print_dict(self.client.get_object_sharing(self.path))
 376         if self['no_permissions']:
 377             self.client.del_object_sharing(self.path)
 378         metadata = self['metadata_to_set'] or dict()
 379         for k in (self['metadata_key_to_delete'] or []):
 380             metadata[k] = ''
 381         if metadata:
 382             self.client.set_object_meta(self.path, metadata)
 383             self.print_dict(self.client.get_object_meta(self.path))
 384
 385     def main(self, path_or_url):
 386         super(self.__class__, self)._run(path_or_url)
 387         if self['publish'] and self['unpublish']:
 388             raise CLIInvalidArgument(
 389                 'Arguments %s and %s cannot be used together' % (
 390                     self.arguments['publish'].lvalue,
 391                     self.arguments['publish'].lvalue))
 392         if self['no_permissions'] and (
 393                 self['uuid_for_read_permission'] or self[
 394                     'uuid_for_write_permission']):
 395             raise CLIInvalidArgument(
 396                 '%s cannot be used with other permission arguments' % (
 397                     self.arguments['no_permissions'].lvalue))
 398         self._run()
 399
 400
 401 @command(file_cmds)
 402 class file_create(_pithos_container, _optional_output_cmd):
 403     """Create an empty file"""
 404
 405     arguments = dict(
 406         content_type=ValueArgument(
 407             'Set content type (default: application/octet-stream)',
 408             '--content-type',
 409             default='application/octet-stream')
 410     )
 411
 412     @errors.generic.all
 413     @errors.pithos.connection
 414     @errors.pithos.container
 415     def _run(self):
 416         self._optional_output(
 417             self.client.create_object(self.path, self['content_type']))
 418
 419     def main(self, path_or_url):
 420         super(self.__class__, self)._run(path_or_url)
 421         self._run()
 422
 423
 424 @command(file_cmds)
 425 class file_mkdir(_pithos_container, _optional_output_cmd):
 426     """Create a directory: /file create --content-type='applcation/directory'
 427     """
 428
 429     @errors.generic.all
 430     @errors.pithos.connection
 431     @errors.pithos.container
 432     def _run(self):
 433         self._optional_output(self.client.create_directory(self.path))
 434
 435     def main(self, path_or_url):
 436         super(self.__class__, self)._run(path_or_url)
 437         self._run()
 438
 439
 440 @command(file_cmds)
 441 class file_delete(_pithos_container):
 442     """Delete a file or directory object"""
 443
 444     arguments = dict(
 445         until_date=DateArgument('remove history until then', '--until'),
 446         yes=FlagArgument('Do not prompt for permission', '--yes'),
 447         recursive=FlagArgument(
 448             'If a directory, empty first', ('-r', '--recursive')),
 449         delimiter=ValueArgument(
 450             'delete objects prefixed with <object><delimiter>', '--delimiter')
 451     )
 452
 453     @errors.generic.all
 454     @errors.pithos.connection
 455     @errors.pithos.container
 456     @errors.pithos.object_path
 457     def _run(self):
 458         if self.path:
 459             if self['yes'] or self.ask_user(
 460                     'Delete /%s/%s ?' % (self.container, self.path)):
 461                 self.client.del_object(
 462                     self.path,
 463                     until=self['until_date'],
 464                     delimiter='/' if self['recursive'] else self['delimiter'])
 465             else:
 466                 self.error('Aborted')
 467         else:
 468             if self['yes'] or self.ask_user(
 469                     'Empty container /%s ?' % self.container):
 470                 self.client.container_delete(self.container, delimiter='/')
 471             else:
 472                 self.error('Aborted')
 473
 474     def main(self, path_or_url):
 475         super(self.__class__, self)._run(path_or_url)
 476         self._run()
 477
 478
 479 class _source_destination(_pithos_container, _optional_output_cmd):
 480
 481     sd_arguments = dict(
 482         destination_user_uuid=ValueArgument(
 483             'default: current user uuid', '--to-account'),
 484         destination_container=ValueArgument(
 485             'default: pithos', '--to-container'),
 486         source_prefix=FlagArgument(
 487             'Transfer all files that are prefixed with SOURCE PATH If the '
 488             'destination path is specified, replace SOURCE_PATH with '
 489             'DESTINATION_PATH',
 490             ('-r', '--recursive')),
 491         force=FlagArgument(
 492             'Overwrite destination objects, if needed', ('-f', '--force')),
 493         source_version=ValueArgument(
 494             'The version of the source object', '--source-version')
 495     )
 496
 497     def __init__(self, arguments={}, auth_base=None, cloud=None):
 498         self.arguments.update(arguments)
 499         self.arguments.update(self.sd_arguments)
 500         super(_source_destination, self).__init__(
 501             self.arguments, auth_base, cloud)
 502
 503     def _report_transfer(self, src, dst, transfer_name):
 504         if not dst:
 505             if transfer_name in ('move', ):
 506                 self.error('  delete source directory %s' % src)
 507             return
 508         dst_prf = '' if self.account == self.dst_client.account else (
 509                 'pithos://%s' % self.dst_client.account)
 510         if src:
 511             src_prf = '' if self.account == self.dst_client.account else (
 512                     'pithos://%s' % self.account)
 513             self.error('  %s %s/%s/%s\n  -->  %s/%s/%s' % (
 514                 transfer_name,
 515                 src_prf, self.container, src,
 516                 dst_prf, self.dst_client.container, dst))
 517         else:
 518             self.error('  mkdir %s/%s/%s' % (
 519                 dst_prf, self.dst_client.container, dst))
 520
 521     @errors.generic.all
 522     @errors.pithos.account
 523     def _src_dst(self, version=None):
 524         """Preconditions:
 525         self.account, self.container, self.path
 526         self.dst_acc, self.dst_con, self.dst_path
 527         They should all be configured properly
 528         :returns: [(src_path, dst_path), ...], if src_path is None, create
 529             destination directory
 530         """
 531         src_objects, dst_objects, pairs = dict(), dict(), []
 532         try:
 533             for obj in self.dst_client.list_objects(
 534                     prefix=self.dst_path or self.path or '/'):
 535                 dst_objects[obj['name']] = obj
 536         except ClientError as ce:
 537             if ce.status in (404, ):
 538                 raise CLIError(
 539                     'Destination container pithos://%s/%s not found' % (
 540                         self.dst_client.account, self.dst_client.container))
 541             raise ce
 542         if self['source_prefix']:
 543             #  Copy and replace prefixes
 544             for src_obj in self.client.list_objects(prefix=self.path):
 545                 src_objects[src_obj['name']] = src_obj
 546             for src_path, src_obj in src_objects.items():
 547                 dst_path = '%s%s' % (
 548                     self.dst_path or self.path, src_path[len(self.path):])
 549                 dst_obj = dst_objects.get(dst_path, None)
 550                 if self['force'] or not dst_obj:
 551                     #  Just do it
 552                     pairs.append((
 553                         None if self._is_dir(src_obj) else src_path, dst_path))
 554                     if self._is_dir(src_obj):
 555                         pairs.append((self.path or dst_path, None))
 556                 elif not (self._is_dir(dst_obj) and self._is_dir(src_obj)):
 557                     raise CLIError(
 558                         'Destination object exists', importance=2, details=[
 559                             'Failed while transfering:',
 560                             '    pithos://%s/%s/%s' % (
 561                                     self.account,
 562                                     self.container,
 563                                     src_path),
 564                             '--> pithos://%s/%s/%s' % (
 565                                     self.dst_client.account,
 566                                     self.dst_client.container,
 567                                     dst_path),
 568                             'Use %s to transfer overwrite' % (
 569                                     self.arguments['force'].lvalue)])
 570         else:
 571             #  One object transfer
 572             try:
 573                 src_version_arg = self.arguments.get('source_version', None)
 574                 src_obj = self.client.get_object_info(
 575                     self.path,
 576                     version=src_version_arg.value if src_version_arg else None)
 577             except ClientError as ce:
 578                 if ce.status in (204, ):
 579                     raise CLIError(
 580                         'Missing specific path container %s' % self.container,
 581                         importance=2, details=[
 582                             'To transfer container contents %s' % (
 583                                 self.arguments['source_prefix'].lvalue)])
 584                 raise
 585             dst_path = self.dst_path or self.path
 586             dst_obj = dst_objects.get(dst_path or self.path, None)
 587             if self['force'] or not dst_obj:
 588                 pairs.append(
 589                     (None if self._is_dir(src_obj) else self.path, dst_path))
 590                 if self._is_dir(src_obj):
 591                     pairs.append((self.path or dst_path, None))
 592             elif self._is_dir(src_obj):
 593                 raise CLIError(
 594                     'Cannot transfer an application/directory object',
 595                     importance=2, details=[
 596                         'The object pithos://%s/%s/%s is a directory' % (
 597                             self.account,
 598                             self.container,
 599                             self.path),
 600                         'To recursively copy a directory, use',
 601                         '  %s' % self.arguments['source_prefix'].lvalue,
 602                         'To create a file, use',
 603                         '  /file create  (general purpose)',
 604                         '  /file mkdir   (a directory object)'])
 605             else:
 606                 raise CLIError(
 607                     'Destination object exists',
 608                     importance=2, details=[
 609                         'Failed while transfering:',
 610                         '    pithos://%s/%s/%s' % (
 611                                 self.account,
 612                                 self.container,
 613                                 self.path),
 614                         '--> pithos://%s/%s/%s' % (
 615                                 self.dst_client.account,
 616                                 self.dst_client.container,
 617                                 dst_path),
 618                         'Use %s to transfer overwrite' % (
 619                                 self.arguments['force'].lvalue)])
 620         return pairs
 621
 622     def _run(self, source_path_or_url, destination_path_or_url=''):
 623         super(_source_destination, self)._run(source_path_or_url)
 624         dst_acc, dst_con, dst_path = self._resolve_pithos_url(
 625             destination_path_or_url)
 626         self.dst_client = PithosClient(
 627             base_url=self.client.base_url, token=self.client.token,
 628             container=self[
 629                 'destination_container'] or dst_con or self.client.container,
 630             account=self[
 631                 'destination_user_uuid'] or dst_acc or self.account)
 632         self.dst_path = dst_path or self.path
 633
 634
 635 @command(file_cmds)
 636 class file_copy(_source_destination):
 637     """Copy objects, even between different accounts or containers"""
 638
 639     arguments = dict(
 640         public=ValueArgument('publish new object', '--public'),
 641         content_type=ValueArgument(
 642             'change object\'s content type', '--content-type'),
 643         source_version=ValueArgument(
 644             'The version of the source object', '--object-version')
 645     )
 646
 647     @errors.generic.all
 648     @errors.pithos.connection
 649     @errors.pithos.container
 650     @errors.pithos.account
 651     def _run(self):
 652         for src, dst in self._src_dst(self['source_version']):
 653             self._report_transfer(src, dst, 'copy')
 654             if src and dst:
 655                 self.dst_client.copy_object(
 656                     src_container=self.client.container,
 657                     src_object=src,
 658                     dst_container=self.dst_client.container,
 659                     dst_object=dst,
 660                     source_account=self.client.account,
 661                     source_version=self['source_version'],
 662                     public=self['public'],
 663                     content_type=self['content_type'])
 664             elif dst:
 665                 self.dst_client.create_directory(dst)
 666
 667     def main(self, source_path_or_url, destination_path_or_url=None):
 668         super(file_copy, self)._run(
 669             source_path_or_url, destination_path_or_url or '')
 670         self._run()
 671
 672
 673 @command(file_cmds)
 674 class file_move(_source_destination):
 675     """Move objects, even between different accounts or containers"""
 676
 677     arguments = dict(
 678         public=ValueArgument('publish new object', '--public'),
 679         content_type=ValueArgument(
 680             'change object\'s content type', '--content-type')
 681     )
 682
 683     @errors.generic.all
 684     @errors.pithos.connection
 685     @errors.pithos.container
 686     @errors.pithos.account
 687     def _run(self):
 688         for src, dst in self._src_dst():
 689             self._report_transfer(src, dst, 'move')
 690             if src and dst:
 691                 self.dst_client.move_object(
 692                     src_container=self.client.container,
 693                     src_object=src,
 694                     dst_container=self.dst_client.container,
 695                     dst_object=dst,
 696                     source_account=self.account,
 697                     public=self['public'],
 698                     content_type=self['content_type'])
 699             elif dst:
 700                 self.dst_client.create_directory(dst)
 701             else:
 702                 self.client.del_object(src)
 703
 704     def main(self, source_path_or_url, destination_path_or_url=None):
 705         super(file_move, self)._run(
 706             source_path_or_url, destination_path_or_url or '')
 707         self._run()
 708
 709
 710 @command(file_cmds)
 711 class file_append(_pithos_container, _optional_output_cmd):
 712     """Append local file to (existing) remote object
 713     The remote object should exist.
 714     If the remote object is a directory, it is transformed into a file.
 715     In the later case, objects under the directory remain intact.
 716     """
 717
 718     arguments = dict(
 719         progress_bar=ProgressBarArgument(
 720             'do not show progress bar', ('-N', '--no-progress-bar'),
 721             default=False),
 722         max_threads=IntArgument('default: 1', '--threads'),
 723     )
 724
 725     @errors.generic.all
 726     @errors.pithos.connection
 727     @errors.pithos.container
 728     @errors.pithos.object_path
 729     def _run(self, local_path):
 730         if self['max_threads'] > 0:
 731             self.client.MAX_THREADS = int(self['max_threads'])
 732         (progress_bar, upload_cb) = self._safe_progress_bar('Appending')
 733         try:
 734             with open(local_path, 'rb') as f:
 735                 self._optional_output(
 736                     self.client.append_object(self.path, f, upload_cb))
 737         finally:
 738             self._safe_progress_bar_finish(progress_bar)
 739
 740     def main(self, local_path, remote_path_or_url):
 741         super(self.__class__, self)._run(remote_path_or_url)
 742         self._run(local_path)
 743
 744
 745 @command(file_cmds)
 746 class file_truncate(_pithos_container, _optional_output_cmd):
 747     """Truncate remote file up to size"""
 748
 749     arguments = dict(
 750         size_in_bytes=IntArgument('Length of file after truncation', '--size')
 751     )
 752     required = ('size_in_bytes', )
 753
 754     @errors.generic.all
 755     @errors.pithos.connection
 756     @errors.pithos.container
 757     @errors.pithos.object_path
 758     @errors.pithos.object_size
 759     def _run(self, size):
 760         self._optional_output(self.client.truncate_object(self.path, size))
 761
 762     def main(self, path_or_url):
 763         super(self.__class__, self)._run(path_or_url)
 764         self._run(size=self['size_in_bytes'])
 765
 766
 767 @command(file_cmds)
 768 class file_overwrite(_pithos_container, _optional_output_cmd):
 769     """Overwrite part of a remote file"""
 770
 771     arguments = dict(
 772         progress_bar=ProgressBarArgument(
 773             'do not show progress bar', ('-N', '--no-progress-bar'),
 774             default=False),
 775         start_position=IntArgument('File position in bytes', '--from'),
 776         end_position=IntArgument('File position in bytes', '--to')
 777     )
 778     required = ('start_position', 'end_position')
 779
 780     @errors.generic.all
 781     @errors.pithos.connection
 782     @errors.pithos.container
 783     @errors.pithos.object_path
 784     @errors.pithos.object_size
 785     def _run(self, local_path, start, end):
 786         start, end = int(start), int(end)
 787         (progress_bar, upload_cb) = self._safe_progress_bar(
 788             'Overwrite %s bytes' % (end - start))
 789         try:
 790             with open(path.abspath(local_path), 'rb') as f:
 791                 self._optional_output(self.client.overwrite_object(
 792                     obj=self.path,
 793                     start=start,
 794                     end=end,
 795                     source_file=f,
 796                     upload_cb=upload_cb))
 797         finally:
 798             self._safe_progress_bar_finish(progress_bar)
 799
 800     def main(self, local_path, path_or_url):
 801         super(self.__class__, self)._run(path_or_url)
 802         self.path = self.path or path.basename(local_path)
 803         self._run(
 804             local_path=local_path,
 805             start=self['start_position'],
 806             end=self['end_position'])
 807
 808
 809 @command(file_cmds)
 810 class file_upload(_pithos_container, _optional_output_cmd):
 811     """Upload a file"""
 812
 813     arguments = dict(
 814         max_threads=IntArgument('default: 5', '--threads'),
 815         content_encoding=ValueArgument(
 816             'set MIME content type', '--content-encoding'),
 817         content_disposition=ValueArgument(
 818             'specify objects presentation style', '--content-disposition'),
 819         content_type=ValueArgument('specify content type', '--content-type'),
 820         uuid_for_read_permission=RepeatableArgument(
 821             'Give read access to a user or group (can be repeated) '
 822             'Use * for all users',
 823             '--read-permission'),
 824         uuid_for_write_permission=RepeatableArgument(
 825             'Give write access to a user or group (can be repeated) '
 826             'Use * for all users',
 827             '--write-permission'),
 828         public=FlagArgument('make object publicly accessible', '--public'),
 829         progress_bar=ProgressBarArgument(
 830             'do not show progress bar',
 831             ('-N', '--no-progress-bar'),
 832             default=False),
 833         overwrite=FlagArgument('Force (over)write', ('-f', '--force')),
 834         recursive=FlagArgument(
 835             'Recursively upload directory *contents* + subdirectories',
 836             ('-r', '--recursive')),
 837         unchunked=FlagArgument(
 838             'Upload file as one block (not recommended)', '--unchunked'),
 839         md5_checksum=ValueArgument(
 840             'Confirm upload with a custom checksum (MD5)', '--etag'),
 841         use_hashes=FlagArgument(
 842             'Source file contains hashmap not data', '--source-is-hashmap'),
 843     )
 844
 845     def _sharing(self):
 846         sharing = dict()
 847         readlist = self['uuid_for_read_permission']
 848         if readlist:
 849             sharing['read'] = self['uuid_for_read_permission']
 850         writelist = self['uuid_for_write_permission']
 851         if writelist:
 852             sharing['write'] = self['uuid_for_write_permission']
 853         return sharing or None
 854
 855     def _check_container_limit(self, path):
 856         cl_dict = self.client.get_container_limit()
 857         container_limit = int(cl_dict['x-container-policy-quota'])
 858         r = self.client.container_get()
 859         used_bytes = sum(int(o['bytes']) for o in r.json)
 860         path_size = get_path_size(path)
 861         if container_limit and path_size > (container_limit - used_bytes):
 862             raise CLIError(
 863                 'Container %s (limit(%s) - used(%s)) < (size(%s) of %s)' % (
 864                     self.client.container,
 865                     format_size(container_limit),
 866                     format_size(used_bytes),
 867                     format_size(path_size),
 868                     path),
 869                 details=[
 870                     'Check accound limit: /file quota',
 871                     'Check container limit:',
 872                     '\t/file containerlimit get %s' % self.client.container,
 873                     'Increase container limit:',
 874                     '\t/file containerlimit set <new limit> %s' % (
 875                         self.client.container)])
 876
 877     def _src_dst(self, local_path, remote_path, objlist=None):
 878         lpath = path.abspath(local_path)
 879         short_path = path.basename(path.abspath(local_path))
 880         rpath = remote_path or short_path
 881         if path.isdir(lpath):
 882             if not self['recursive']:
 883                 raise CLIError('%s is a directory' % lpath, details=[
 884                     'Use %s to upload directories & contents' % (
 885                         self.arguments['recursive'].lvalue)])
 886             robj = self.client.container_get(path=rpath)
 887             if not self['overwrite']:
 888                 if robj.json:
 889                     raise CLIError(
 890                         'Objects/files prefixed as %s already exist' % rpath,
 891                         details=['Existing objects:'] + ['\t/%s/\t%s' % (
 892                             o['name'],
 893                             o['content_type'][12:]) for o in robj.json] + [
 894                             'Use -f to add, overwrite or resume'])
 895                 else:
 896                     try:
 897                         topobj = self.client.get_object_info(rpath)
 898                         if not self._is_dir(topobj):
 899                             raise CLIError(
 900                                 'Object /%s/%s exists but not a directory' % (
 901                                     self.container, rpath),
 902                                 details=['Use -f to overwrite'])
 903                     except ClientError as ce:
 904                         if ce.status not in (404, ):
 905                             raise
 906             self._check_container_limit(lpath)
 907             prev = ''
 908             for top, subdirs, files in walk(lpath):
 909                 if top != prev:
 910                     prev = top
 911                     try:
 912                         rel_path = rpath + top.split(lpath)[1]
 913                     except IndexError:
 914                         rel_path = rpath
 915                     self.error('mkdir /%s/%s' % (
 916                         self.client.container, rel_path))
 917                     self.client.create_directory(rel_path)
 918                 for f in files:
 919                     fpath = path.join(top, f)
 920                     if path.isfile(fpath):
 921                         rel_path = rel_path.replace(path.sep, '/')
 922                         pathfix = f.replace(path.sep, '/')
 923                         yield open(fpath, 'rb'), '%s/%s' % (rel_path, pathfix)
 924                     else:
 925                         self.error('%s is not a regular file' % fpath)
 926         else:
 927             if not path.isfile(lpath):
 928                 raise CLIError(('%s is not a regular file' % lpath) if (
 929                     path.exists(lpath)) else '%s does not exist' % lpath)
 930             try:
 931                 robj = self.client.get_object_info(rpath)
 932                 if remote_path and self._is_dir(robj):
 933                     rpath += '/%s' % (short_path.replace(path.sep, '/'))
 934                     self.client.get_object_info(rpath)
 935                 if not self['overwrite']:
 936                     raise CLIError(
 937                         'Object /%s/%s already exists' % (
 938                             self.container, rpath),
 939                         details=['use -f to overwrite / resume'])
 940             except ClientError as ce:
 941                 if ce.status not in (404, ):
 942                     raise
 943             self._check_container_limit(lpath)
 944             yield open(lpath, 'rb'), rpath
 945
 946     def _run(self, local_path, remote_path):
 947         self.client.MAX_THREADS = int(self['max_threads'] or 5)
 948         params = dict(
 949             content_encoding=self['content_encoding'],
 950             content_type=self['content_type'],
 951             content_disposition=self['content_disposition'],
 952             sharing=self._sharing(),
 953             public=self['public'])
 954         uploaded, container_info_cache = list, dict()
 955         rpref = 'pithos://%s' if self['account'] else ''
 956         for f, rpath in self._src_dst(local_path, remote_path):
 957             self.error('%s --> %s/%s/%s' % (
 958                 f.name, rpref, self.client.container, rpath))
 959             if not (self['content_type'] and self['content_encoding']):
 960                 ctype, cenc = guess_mime_type(f.name)
 961                 params['content_type'] = self['content_type'] or ctype
 962                 params['content_encoding'] = self['content_encoding'] or cenc
 963             if self['unchunked']:
 964                 r = self.client.upload_object_unchunked(
 965                     rpath, f,
 966                     etag=self['md5_checksum'], withHashFile=self['use_hashes'],
 967                     **params)
 968                 if self['with_output'] or self['json_output']:
 969                     r['name'] = '/%s/%s' % (self.client.container, rpath)
 970                     uploaded.append(r)
 971             else:
 972                 try:
 973                     (progress_bar, upload_cb) = self._safe_progress_bar(
 974                         'Uploading %s' % f.name.split(path.sep)[-1])
 975                     if progress_bar:
 976                         hash_bar = progress_bar.clone()
 977                         hash_cb = hash_bar.get_generator(
 978                             'Calculating block hashes')
 979                     else:
 980                         hash_cb = None
 981                     r = self.client.upload_object(
 982                         rpath, f,
 983                         hash_cb=hash_cb,
 984                         upload_cb=upload_cb,
 985                         container_info_cache=container_info_cache,
 986                         **params)
 987                     if self['with_output'] or self['json_output']:
 988                         r['name'] = '/%s/%s' % (self.client.container, rpath)
 989                         uploaded.append(r)
 990                 except Exception:
 991                     self._safe_progress_bar_finish(progress_bar)
 992                     raise
 993                 finally:
 994                     self._safe_progress_bar_finish(progress_bar)
 995         self._optional_output(uploaded)
 996         self.error('Upload completed')
 997
 998     def main(self, local_path, remote_path_or_url):
 999         super(self.__class__, self)._run(remote_path_or_url)
1000         remote_path = self.path or path.basename(path.abspath(local_path))
1001         self._run(local_path=local_path, remote_path=remote_path)
1002
1003
1004 class RangeArgument(ValueArgument):
1005     """
1006     :value type: string of the form <start>-<end> where <start> and <end> are
1007         integers
1008     :value returns: the input string, after type checking <start> and <end>
1009     """
1010
1011     @property
1012     def value(self):
1013         return getattr(self, '_value', self.default)
1014
1015     @value.setter
1016     def value(self, newvalues):
1017         if newvalues:
1018             self._value = getattr(self, '_value', self.default)
1019             for newvalue in newvalues.split(','):
1020                 self._value = ('%s,' % self._value) if self._value else ''
1021                 start, sep, end = newvalue.partition('-')
1022                 if sep:
1023                     if start:
1024                         start, end = (int(start), int(end))
1025                         if start > end:
1026                             raise CLIInvalidArgument(
1027                                 'Invalid range %s' % newvalue, details=[
1028                                 'Valid range formats',
1029                                 '  START-END', '  UP_TO', '  -FROM',
1030                                 'where all values are integers',
1031                                 'OR a compination (csv), e.g.,',
1032                                 '  %s=5,10-20,-5' % self.lvalue])
1033                         self._value += '%s-%s' % (start, end)
1034                     else:
1035                         self._value += '-%s' % int(end)
1036                 else:
1037                     self._value += '%s' % int(start)
1038
1039
1040 @command(file_cmds)
1041 class file_cat(_pithos_container):
1042     """Fetch remote file contents"""
1043
1044     arguments = dict(
1045         range=RangeArgument('show range of data e.g., 5,10-20,-5', '--range'),
1046         if_match=ValueArgument('show output if ETags match', '--if-match'),
1047         if_none_match=ValueArgument(
1048             'show output if ETags match', '--if-none-match'),
1049         if_modified_since=DateArgument(
1050             'show output modified since then', '--if-modified-since'),
1051         if_unmodified_since=DateArgument(
1052             'show output unmodified since then', '--if-unmodified-since'),
1053         object_version=ValueArgument(
1054             'Get contents of the chosen version', '--object-version')
1055     )
1056
1057     @errors.generic.all
1058     @errors.pithos.connection
1059     @errors.pithos.container
1060     @errors.pithos.object_path
1061     def _run(self):
1062         r = self.client.download_object(
1063             self.path, self._out,
1064             range_str=self['range'],
1065             version=self['object_version'],
1066             if_match=self['if_match'],
1067             if_none_match=self['if_none_match'],
1068             if_modified_since=self['if_modified_since'],
1069             if_unmodified_since=self['if_unmodified_since'])
1070         print r
1071
1072     def main(self, path_or_url):
1073         super(self.__class__, self)._run(path_or_url)
1074         self._run()
1075
1076
1077 @command(file_cmds)
1078 class file_download(_pithos_container):
1079     """Download a remove file or directory object to local file system"""
1080
1081     arguments = dict(
1082         resume=FlagArgument(
1083             'Resume/Overwrite (attempt resume, else overwrite)',
1084             ('-f', '--resume')),
1085         range=RangeArgument(
1086             'Download only that range of data e.g., 5,10-20,-5', '--range'),
1087         matching_etag=ValueArgument('download iff ETag match', '--if-match'),
1088         non_matching_etag=ValueArgument(
1089             'download iff ETags DO NOT match', '--if-none-match'),
1090         modified_since_date=DateArgument(
1091             'download iff remote file is modified since then',
1092             '--if-modified-since'),
1093         unmodified_since_date=DateArgument(
1094             'show output iff remote file is unmodified since then',
1095             '--if-unmodified-since'),
1096         object_version=ValueArgument(
1097             'download a file of a specific version', '--object-version'),
1098         max_threads=IntArgument('default: 5', '--threads'),
1099         progress_bar=ProgressBarArgument(
1100             'do not show progress bar', ('-N', '--no-progress-bar'),
1101             default=False),
1102         recursive=FlagArgument(
1103             'Download a remote directory object and its contents',
1104             ('-r', '--recursive'))
1105         )
1106
1107     def _src_dst(self, local_path):
1108         """Create a list of (src, dst) where src is a remote location and dst
1109         is an open file descriptor. Directories are denoted as (None, dirpath)
1110         and they are pretended to other objects in a very strict order (shorter
1111         to longer path)."""
1112         ret = []
1113         try:
1114             if self.path:
1115                 obj = self.client.get_object_info(
1116                     self.path, version=self['object_version'])
1117                 obj.setdefault('name', self.path.strip('/'))
1118             else:
1119                 obj = None
1120         except ClientError as ce:
1121             if ce.status in (404, ):
1122                 raiseCLIError(ce, details=[
1123                     'To download an object, it must exist either as a file or'
1124                     ' as a directory.',
1125                     'For example, to download everything under prefix/ the '
1126                     'directory "prefix" must exist.',
1127                     'To see if an remote object is actually there:',
1128                     '  /file info [/CONTAINER/]OBJECT',
1129                     'To create a directory object:',
1130                     '  /file mkdir [/CONTAINER/]OBJECT'])
1131             if ce.status in (204, ):
1132                 raise CLIError(
1133                     'No file or directory objects to download',
1134                     details=[
1135                         'To download a container (e.g., %s):' % self.container,
1136                         '  [kamaki] container download %s [LOCAL_PATH]' % (
1137                             self.container)])
1138             raise
1139         rpath = self.path.strip('/')
1140         if local_path and self.path and local_path.endswith('/'):
1141             local_path = local_path[-1:]
1142
1143         if (not obj) or self._is_dir(obj):
1144             if self['recursive']:
1145                 if not (self.path or local_path.endswith('/')):
1146                     #  Download the whole container
1147                     local_path = '' if local_path in ('.', ) else local_path
1148                     local_path = '%s/' % (local_path or self.container)
1149                 obj = obj or dict(
1150                     name='', content_type='application/directory')
1151                 dirs, files = [obj, ], []
1152                 objects = self.client.container_get(
1153                     path=self.path,
1154                     if_modified_since=self['modified_since_date'],
1155                     if_unmodified_since=self['unmodified_since_date'])
1156                 for o in objects.json:
1157                     (dirs if self._is_dir(o) else files).append(o)
1158
1159                 #  Put the directories on top of the list
1160                 for dpath in sorted(['%s%s' % (
1161                         local_path, d['name'][len(rpath):]) for d in dirs]):
1162                     if path.exists(dpath):
1163                         if path.isdir(dpath):
1164                             continue
1165                         raise CLIError(
1166                             'Cannot replace local file %s with a directory '
1167                             'of the same name' % dpath,
1168                             details=[
1169                                 'Either remove the file or specify a'
1170                                 'different target location'])
1171                     ret.append((None, dpath, None))
1172
1173                 #  Append the file objects
1174                 for opath in [o['name'] for o in files]:
1175                     lpath = '%s%s' % (local_path, opath[len(rpath):])
1176                     if self['resume']:
1177                         fxists = path.exists(lpath)
1178                         if fxists and path.isdir(lpath):
1179                             raise CLIError(
1180                                 'Cannot change local dir %s info file' % (
1181                                     lpath),
1182                                 details=[
1183                                     'Either remove the file or specify a'
1184                                     'different target location'])
1185                         ret.append((opath, lpath, fxists))
1186                     elif path.exists(lpath):
1187                         raise CLIError(
1188                             'Cannot overwrite %s' % lpath,
1189                             details=['To overwrite/resume, use  %s' % (
1190                                 self.arguments['resume'].lvalue)])
1191                     else:
1192                         ret.append((opath, lpath, None))
1193             elif self.path:
1194                 raise CLIError(
1195                     'Remote object /%s/%s is a directory' % (
1196                         self.container, local_path),
1197                     details=['Use %s to download directories' % (
1198                         self.arguments['recursive'].lvalue)])
1199             else:
1200                 parsed_name = self.arguments['recursive'].lvalue
1201                 raise CLIError(
1202                     'Cannot download container %s' % self.container,
1203                     details=[
1204                         'Use %s to download containers' % parsed_name,
1205                         '  [kamaki] file download %s /%s [LOCAL_PATH]' % (
1206                             parsed_name, self.container)])
1207         else:
1208             #  Remote object is just a file
1209             if path.exists(local_path):
1210                 if not self['resume']:
1211                     raise CLIError(
1212                         'Cannot overwrite local file %s' % (local_path),
1213                         details=['To overwrite/resume, use  %s' % (
1214                             self.arguments['resume'].lvalue)])
1215             elif '/' in local_path[1:-1]:
1216                 dirs = [p for p in local_path.split('/') if p]
1217                 pref = '/' if local_path.startswith('/') else ''
1218                 for d in dirs[:-1]:
1219                     pref += d
1220                     if not path.exists(pref):
1221                         ret.append((None, d, None))
1222                     elif not path.isdir(pref):
1223                         raise CLIError(
1224                             'Failed to use %s as a destination' % local_path,
1225                             importance=3,
1226                             details=[
1227                                 'Local file %s is not a directory' % pref,
1228                                 'Destination prefix must consist of '
1229                                 'directories or non-existing names',
1230                                 'Either remove the file, or choose another '
1231                                 'destination'])
1232             ret.append((rpath, local_path, self['resume']))
1233         for r, l, resume in ret:
1234             if r:
1235                 with open(l, 'rwb+' if resume else 'wb+') as f:
1236                     yield (r, f)
1237             else:
1238                 yield (r, l)
1239
1240     @errors.generic.all
1241     @errors.pithos.connection
1242     @errors.pithos.container
1243     @errors.pithos.object_path
1244     @errors.pithos.local_path
1245     @errors.pithos.local_path_download
1246     def _run(self, local_path):
1247         self.client.MAX_THREADS = int(self['max_threads'] or 5)
1248         progress_bar = None
1249         try:
1250             for rpath, output_file in self._src_dst(local_path):
1251                 if not rpath:
1252                     self.error('Create local directory %s' % output_file)
1253                     makedirs(output_file)
1254                     continue
1255                 self.error('/%s/%s --> %s' % (
1256                     self.container, rpath, output_file.name))
1257                 progress_bar, download_cb = self._safe_progress_bar(
1258                     '  download')
1259                 self.client.download_object(
1260                     rpath, output_file,
1261                     download_cb=download_cb,
1262                     range_str=self['range'],
1263                     version=self['object_version'],
1264                     if_match=self['matching_etag'],
1265                     resume=self['resume'],
1266                     if_none_match=self['non_matching_etag'],
1267                     if_modified_since=self['modified_since_date'],
1268                     if_unmodified_since=self['unmodified_since_date'])
1269         except KeyboardInterrupt:
1270             from threading import activeCount, enumerate as activethreads
1271             timeout = 0.5
1272             while activeCount() > 1:
1273                 self._out.write('\nCancel %s threads: ' % (activeCount() - 1))
1274                 self._out.flush()
1275                 for thread in activethreads():
1276                     try:
1277                         thread.join(timeout)
1278                         self._out.write('.' if thread.isAlive() else '*')
1279                     except RuntimeError:
1280                         continue
1281                     finally:
1282                         self._out.flush()
1283                         timeout += 0.1
1284             self.error('\nDownload canceled by user')
1285             if local_path is not None:
1286                 self.error('to resume, re-run with --resume')
1287         except Exception:
1288             self._safe_progress_bar_finish(progress_bar)
1289             raise
1290         finally:
1291             self._safe_progress_bar_finish(progress_bar)
1292
1293     def main(self, remote_path_or_url, local_path=None):
1294         super(self.__class__, self)._run(remote_path_or_url)
1295         local_path = local_path or self.path or '.'
1296         self._run(local_path=local_path)
1297
1298
1299 @command(container_cmds)
1300 class container_info(_pithos_account, _optional_json):
1301     """Get information about a container"""
1302
1303     arguments = dict(
1304         until_date=DateArgument('show metadata until then', '--until'),
1305         metadata=FlagArgument('Show only container metadata', '--metadata'),
1306         sizelimit=FlagArgument(
1307             'Show the maximum size limit for container', '--size-limit'),
1308         in_bytes=FlagArgument('Show size limit in bytes', ('-b', '--bytes'))
1309     )
1310
1311     @errors.generic.all
1312     @errors.pithos.connection
1313     @errors.pithos.container
1314     @errors.pithos.object_path
1315     def _run(self):
1316         if self['metadata']:
1317             r, preflen = dict(), len('x-container-meta-')
1318             for k, v in self.client.get_container_meta(
1319                     until=self['until_date']).items():
1320                 r[k[preflen:]] = v
1321         elif self['sizelimit']:
1322             r = self.client.get_container_limit(
1323                 self.container)['x-container-policy-quota']
1324             r = {'size limit': 'unlimited' if r in ('0', ) else (
1325                 int(r) if self['in_bytes'] else format_size(r))}
1326         else:
1327             r = self.client.get_container_info(self.container)
1328         self._print(r, self.print_dict)
1329
1330     def main(self, container):
1331         super(self.__class__, self)._run()
1332         self.container, self.client.container = container, container
1333         self._run()
1334
1335
1336 class VersioningArgument(ValueArgument):
1337
1338     schemes = ('auto', 'none')
1339
1340     @property
1341     def value(self):
1342         return getattr(self, '_value', None)
1343
1344     @value.setter
1345     def value(self, new_scheme):
1346         if new_scheme:
1347             new_scheme = new_scheme.lower()
1348             if new_scheme not in self.schemes:
1349                 raise CLIInvalidArgument('Invalid versioning value', details=[
1350                     'Valid versioning values are %s' % ', '.join(
1351                         self.schemes)])
1352             self._value = new_scheme
1353
1354
1355 @command(container_cmds)
1356 class container_modify(_pithos_account, _optional_json):
1357     """Modify the properties of a container"""
1358
1359     arguments = dict(
1360         metadata_to_add=KeyValueArgument(
1361             'Add metadata in the form KEY=VALUE (can be repeated)',
1362             '--metadata-add'),
1363         metadata_to_delete=RepeatableArgument(
1364             'Delete metadata by KEY (can be repeated)', '--metadata-del'),
1365         sizelimit=DataSizeArgument(
1366             'Set max size limit (0 for unlimited, '
1367             'use units B, KiB, KB, etc.)', '--size-limit'),
1368         versioning=VersioningArgument(
1369             'Set a versioning scheme (%s)' % ', '.join(
1370                 VersioningArgument.schemes), '--versioning')
1371     )
1372     required = [
1373         'metadata_to_add', 'metadata_to_delete', 'sizelimit', 'versioning']
1374
1375     @errors.generic.all
1376     @errors.pithos.connection
1377     @errors.pithos.container
1378     def _run(self, container):
1379         metadata = self['metadata_to_add']
1380         for k in (self['metadata_to_delete'] or []):
1381             metadata[k] = ''
1382         if metadata:
1383             self.client.set_container_meta(metadata)
1384             self._print(self.client.get_container_meta(), self.print_dict)
1385         if self['sizelimit'] is not None:
1386             self.client.set_container_limit(self['sizelimit'])
1387             r = self.client.get_container_limit()['x-container-policy-quota']
1388             r = 'unlimited' if r in ('0', ) else format_size(r)
1389             self.writeln('new size limit: %s' % r)
1390         if self['versioning']:
1391             self.client.set_container_versioning(self['versioning'])
1392             self.writeln('new versioning scheme: %s' % (
1393                 self.client.get_container_versioning(self.container)[
1394                     'x-container-policy-versioning']))
1395
1396     def main(self, container):
1397         super(self.__class__, self)._run()
1398         self.client.container, self.container = container, container
1399         self._run(container=container)
1400
1401
1402 @command(container_cmds)
1403 class container_list(_pithos_account, _optional_json, _name_filter):
1404     """List all containers, or their contents"""
1405
1406     arguments = dict(
1407         detail=FlagArgument('Containers with details', ('-l', '--list')),
1408         limit=IntArgument('limit number of listed items', ('-n', '--number')),
1409         marker=ValueArgument('output greater that marker', '--marker'),
1410         modified_since_date=ValueArgument(
1411             'show output modified since then', '--if-modified-since'),
1412         unmodified_since_date=ValueArgument(
1413             'show output not modified since then', '--if-unmodified-since'),
1414         until_date=DateArgument('show metadata until then', '--until'),
1415         shared=FlagArgument('show only shared', '--shared'),
1416         more=FlagArgument('read long results', '--more'),
1417         enum=FlagArgument('Enumerate results', '--enumerate'),
1418         recursive=FlagArgument(
1419             'Recursively list containers and their contents',
1420             ('-r', '--recursive')),
1421         shared_by_me=FlagArgument(
1422             'show only files shared to other users', '--shared-by-me'),
1423         public=FlagArgument('show only published objects', '--public'),
1424     )
1425
1426     def print_containers(self, container_list):
1427         for index, container in enumerate(container_list):
1428             if 'bytes' in container:
1429                 size = format_size(container['bytes'])
1430             prfx = ('%s. ' % (index + 1)) if self['enum'] else ''
1431             _cname = container['name'] if (
1432                 self['more']) else bold(container['name'])
1433             cname = u'%s%s' % (prfx, _cname)
1434             if self['detail']:
1435                 self.writeln(cname)
1436                 pretty_c = container.copy()
1437                 if 'bytes' in container:
1438                     pretty_c['bytes'] = '%s (%s)' % (container['bytes'], size)
1439                 self.print_dict(pretty_c, exclude=('name'))
1440                 self.writeln()
1441             else:
1442                 if 'count' in container and 'bytes' in container:
1443                     self.writeln('%s (%s, %s objects)' % (
1444                         cname, size, container['count']))
1445                 else:
1446                     self.writeln(cname)
1447             objects = container.get('objects', [])
1448             if objects:
1449                 self.print_objects(objects)
1450                 self.writeln('')
1451
1452     def _create_object_forest(self, container_list):
1453         try:
1454             for container in container_list:
1455                 self.client.container = container['name']
1456                 objects = self.client.container_get(
1457                     limit=False if self['more'] else self['limit'],
1458                     if_modified_since=self['modified_since_date'],
1459                     if_unmodified_since=self['unmodified_since_date'],
1460                     until=self['until_date'],
1461                     show_only_shared=self['shared_by_me'],
1462                     public=self['public'])
1463                 container['objects'] = objects.json
1464         finally:
1465             self.client.container = None
1466
1467     @errors.generic.all
1468     @errors.pithos.connection
1469     @errors.pithos.object_path
1470     @errors.pithos.container
1471     def _run(self, container):
1472         if container:
1473             r = self.client.container_get(
1474                 limit=False if self['more'] else self['limit'],
1475                 marker=self['marker'],
1476                 if_modified_since=self['modified_since_date'],
1477                 if_unmodified_since=self['unmodified_since_date'],
1478                 until=self['until_date'],
1479                 show_only_shared=self['shared_by_me'],
1480                 public=self['public'])
1481         else:
1482             r = self.client.account_get(
1483                 limit=False if self['more'] else self['limit'],
1484                 marker=self['marker'],
1485                 if_modified_since=self['modified_since_date'],
1486                 if_unmodified_since=self['unmodified_since_date'],
1487                 until=self['until_date'],
1488                 show_only_shared=self['shared_by_me'],
1489                 public=self['public'])
1490         files = self._filter_by_name(r.json)
1491         if self['recursive'] and not container:
1492             self._create_object_forest(files)
1493         if self['more']:
1494             outbu, self._out = self._out, StringIO()
1495         try:
1496             if self['json_output'] or self['output_format']:
1497                 self._print(files)
1498             else:
1499                 (self.print_objects if container else self.print_containers)(
1500                     files)
1501         finally:
1502             if self['more']:
1503                 pager(self._out.getvalue())
1504                 self._out = outbu
1505
1506     def main(self, container=None):
1507         super(self.__class__, self)._run()
1508         self.client.container, self.container = container, container
1509         self._run(container)
1510
1511
1512 @command(container_cmds)
1513 class container_create(_pithos_account):
1514     """Create a new container"""
1515
1516     arguments = dict(
1517         versioning=ValueArgument(
1518             'set container versioning (auto/none)', '--versioning'),
1519         limit=IntArgument('set default container limit', '--limit'),
1520         meta=KeyValueArgument(
1521             'set container metadata (can be repeated)', '--meta')
1522     )
1523
1524     @errors.generic.all
1525     @errors.pithos.connection
1526     @errors.pithos.container
1527     def _run(self, container):
1528         try:
1529             self.client.create_container(
1530                 container=container,
1531                 sizelimit=self['limit'],
1532                 versioning=self['versioning'],
1533                 metadata=self['meta'],
1534                 success=(201, ))
1535         except ClientError as ce:
1536             if ce.status in (202, ):
1537                 raise CLIError(
1538                     'Container %s alread exists' % container, details=[
1539                     'Either delete %s or choose another name' % (container)])
1540             raise
1541
1542     def main(self, new_container):
1543         super(self.__class__, self)._run()
1544         self._run(container=new_container)
1545
1546
1547 @command(container_cmds)
1548 class container_delete(_pithos_account):
1549     """Delete a container"""
1550
1551     arguments = dict(
1552         yes=FlagArgument('Do not prompt for permission', '--yes'),
1553         recursive=FlagArgument(
1554             'delete container even if not empty', ('-r', '--recursive'))
1555     )
1556
1557     @errors.generic.all
1558     @errors.pithos.connection
1559     @errors.pithos.container
1560     def _run(self, container):
1561         num_of_contents = int(self.client.get_container_info(container)[
1562             'x-container-object-count'])
1563         delimiter, msg = None, 'Delete container %s ?' % container
1564         if self['recursive']:
1565             delimiter, msg = '/', 'Empty and d%s' % msg[1:]
1566         elif num_of_contents:
1567             raise CLIError('Container %s is not empty' % container, details=[
1568                 'Use %s to delete non-empty containers' % (
1569                     self.arguments['recursive'].lvalue)])
1570         if self['yes'] or self.ask_user(msg):
1571             if num_of_contents:
1572                 self.client.del_container(delimiter=delimiter)
1573             self.client.purge_container()
1574
1575     def main(self, container):
1576         super(self.__class__, self)._run()
1577         self.container, self.client.container = container, container
1578         self._run(container)
1579
1580
1581 @command(container_cmds)
1582 class container_empty(_pithos_account):
1583     """Empty a container"""
1584
1585     arguments = dict(yes=FlagArgument('Do not prompt for permission', '--yes'))
1586
1587     @errors.generic.all
1588     @errors.pithos.connection
1589     @errors.pithos.container
1590     def _run(self, container):
1591         if self['yes'] or self.ask_user('Empty container %s ?' % container):
1592             self.client.del_container(delimiter='/')
1593
1594     def main(self, container):
1595         super(self.__class__, self)._run()
1596         self.container, self.client.container = container, container
1597         self._run(container)
1598
1599
1600 @command(sharer_cmds)
1601 class sharer_list(_pithos_account, _optional_json):
1602     """List accounts who share file objects with current user"""
1603
1604     arguments = dict(
1605         detail=FlagArgument('show detailed output', ('-l', '--details')),
1606         marker=ValueArgument('show output greater then marker', '--marker')
1607     )
1608
1609     @errors.generic.all
1610     @errors.pithos.connection
1611     def _run(self):
1612         accounts = self.client.get_sharing_accounts(marker=self['marker'])
1613         if not (self['json_output'] or self['output_format']):
1614             usernames = self._uuids2usernames(
1615                 [acc['name'] for acc in accounts])
1616             for item in accounts:
1617                 uuid = item['name']
1618                 item['id'], item['name'] = uuid, usernames[uuid]
1619                 if not self['detail']:
1620                     item.pop('last_modified')
1621         self._print(accounts)
1622
1623     def main(self):
1624         super(self.__class__, self)._run()
1625         self._run()
1626
1627
1628 @command(sharer_cmds)
1629 class sharer_info(_pithos_account, _optional_json):
1630     """Details on a Pithos+ sharer account (default: current account)"""
1631
1632     @errors.generic.all
1633     @errors.pithos.connection
1634     def _run(self):
1635         self._print(self.client.get_account_info(), self.print_dict)
1636
1637     def main(self, account_uuid=None):
1638         super(self.__class__, self)._run()
1639         if account_uuid:
1640             self.client.account, self.account = account_uuid, account_uuid
1641         self._run()
1642
1643
1644 class _pithos_group(_pithos_account):
1645     prefix = 'x-account-group-'
1646     preflen = len(prefix)
1647
1648     def _groups(self):
1649         groups = dict()
1650         for k, v in self.client.get_account_group().items():
1651             groups[k[self.preflen:]] = v
1652         return groups
1653
1654
1655 @command(group_cmds)
1656 class group_list(_pithos_group, _optional_json):
1657     """list all groups and group members"""
1658
1659     @errors.generic.all
1660     @errors.pithos.connection
1661     def _run(self):
1662         self._print(self._groups(), self.print_dict)
1663
1664     def main(self):
1665         super(self.__class__, self)._run()
1666         self._run()
1667
1668
1669 @command(group_cmds)
1670 class group_create(_pithos_group, _optional_json):
1671     """Create a group of users"""
1672
1673     arguments = dict(
1674         user_uuid=RepeatableArgument('Add a user to the group', '--uuid'),
1675         username=RepeatableArgument('Add a user to the group', '--username')
1676     )
1677     required = ['user_uuid', 'username']
1678
1679     @errors.generic.all
1680     @errors.pithos.connection
1681     def _run(self, groupname, *users):
1682         if groupname in self._groups() and not self.ask_user(
1683                 'Group %s already exists, overwrite?' % groupname):
1684             self.error('Aborted')
1685             return
1686         self.client.set_account_group(groupname, users)
1687         self._print(self._groups(), self.print_dict)
1688
1689     def main(self, groupname):
1690         super(self.__class__, self)._run()
1691         users = (self['user_uuid'] or []) + self._usernames2uuids(
1692             self['username'] or []).values()
1693         if users:
1694             self._run(groupname, *users)
1695         else:
1696             raise CLISyntaxError(
1697                 'No valid users specified, use %s or %s' % (
1698                     self.arguments['user_uuid'].lvalue,
1699                     self.arguments['username'].lvalue),
1700                 details=[
1701                     'Check if a username or uuid is valid with',
1702                     '  user uuid2username', 'OR', '  user username2uuid'])
1703
1704
1705 @command(group_cmds)
1706 class group_delete(_pithos_group, _optional_json):
1707     """Delete a user group"""
1708
1709     @errors.generic.all
1710     @errors.pithos.connection
1711     def _run(self, groupname):
1712         self.client.del_account_group(groupname)
1713         self._print(self._groups(), self.print_dict)
1714
1715     def main(self, groupname):
1716         super(self.__class__, self)._run()
1717         self._run(groupname)
1718
1719
1720 #  Deprecated commands
1721
1722 @command(file_cmds)
1723 class file_publish(_pithos_init):
1724     """DEPRECATED, replaced by [kamaki] file modify OBJECT --publish"""
1725
1726     def main(self, *args):
1727         raise CLISyntaxError('DEPRECATED', details=[
1728             'This command is replaced by:',
1729             '  [kamaki] file modify OBJECT --publish'])
1730
1731
1732 @command(file_cmds)
1733 class file_unpublish(_pithos_init):
1734     """DEPRECATED, replaced by [kamaki] file modify OBJECT --unpublish"""
1735
1736     def main(self, *args):
1737         raise CLISyntaxError('DEPRECATED', details=[
1738             'This command is replaced by:',
1739             '  [kamaki] file modify OBJECT --unpublish'])