code.grnet.gr Git - kamaki/blob - kamaki/cli/commands/pithos.py

   1 # Copyright 2011-2012 GRNET S.A. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or
   4 # without modification, are permitted provided that the following
   5 # conditions are met:
   6 #
   7 #   1. Redistributions of source code must retain the above
   8 #      copyright notice, this list of conditions and the following
   9 #      disclaimer.
  10 #
  11 #   2. Redistributions in binary form must reproduce the above
  12 #      copyright notice, this list of conditions and the following
  13 #      disclaimer in the documentation and/or other materials
  14 #      provided with the distribution.
  15 #
  16 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  17 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  23 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  24 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  26 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27 # POSSIBILITY OF SUCH DAMAGE.
  28 #
  29 # The views and conclusions contained in the software and
  30 # documentation are those of the authors and should not be
  31 # interpreted as representing official policies, either expressed
  32 # or implied, of GRNET S.A.command
  33
  34 from sys import stdout
  35 from time import localtime, strftime
  36 from logging import getLogger
  37 from os import path, makedirs, walk
  38
  39 from kamaki.cli import command
  40 from kamaki.cli.command_tree import CommandTree
  41 from kamaki.cli.errors import raiseCLIError, CLISyntaxError
  42 from kamaki.cli.utils import (
  43     format_size, to_bytes, print_dict, print_items, pretty_keys,
  44     page_hold, bold, ask_user, get_path_size)
  45 from kamaki.cli.argument import FlagArgument, ValueArgument, IntArgument
  46 from kamaki.cli.argument import KeyValueArgument, DateArgument
  47 from kamaki.cli.argument import ProgressBarArgument
  48 from kamaki.cli.commands import _command_init, errors
  49 from kamaki.clients.pithos import PithosClient, ClientError
  50 from kamaki.clients.astakos import AstakosClient
  51
  52
  53 kloger = getLogger('kamaki')
  54
  55 pithos_cmds = CommandTree('file', 'Pithos+/Storage API commands')
  56 _commands = [pithos_cmds]
  57
  58
  59 # Argument functionality
  60
  61 class DelimiterArgument(ValueArgument):
  62     """
  63     :value type: string
  64     :value returns: given string or /
  65     """
  66
  67     def __init__(self, caller_obj, help='', parsed_name=None, default=None):
  68         super(DelimiterArgument, self).__init__(help, parsed_name, default)
  69         self.caller_obj = caller_obj
  70
  71     @property
  72     def value(self):
  73         if self.caller_obj['recursive']:
  74             return '/'
  75         return getattr(self, '_value', self.default)
  76
  77     @value.setter
  78     def value(self, newvalue):
  79         self._value = newvalue
  80
  81
  82 class SharingArgument(ValueArgument):
  83     """Set sharing (read and/or write) groups
  84     .
  85     :value type: "read=term1,term2,... write=term1,term2,..."
  86     .
  87     :value returns: {'read':['term1', 'term2', ...],
  88     .   'write':['term1', 'term2', ...]}
  89     """
  90
  91     @property
  92     def value(self):
  93         return getattr(self, '_value', self.default)
  94
  95     @value.setter
  96     def value(self, newvalue):
  97         perms = {}
  98         try:
  99             permlist = newvalue.split(' ')
 100         except AttributeError:
 101             return
 102         for p in permlist:
 103             try:
 104                 (key, val) = p.split('=')
 105             except ValueError as err:
 106                 raiseCLIError(
 107                     err,
 108                     'Error in --sharing',
 109                     details='Incorrect format',
 110                     importance=1)
 111             if key.lower() not in ('read', 'write'):
 112                 msg = 'Error in --sharing'
 113                 raiseCLIError(err, msg, importance=1, details=[
 114                     'Invalid permission key %s' % key])
 115             val_list = val.split(',')
 116             if not key in perms:
 117                 perms[key] = []
 118             for item in val_list:
 119                 if item not in perms[key]:
 120                     perms[key].append(item)
 121         self._value = perms
 122
 123
 124 class RangeArgument(ValueArgument):
 125     """
 126     :value type: string of the form <start>-<end> where <start> and <end> are
 127         integers
 128     :value returns: the input string, after type checking <start> and <end>
 129     """
 130
 131     @property
 132     def value(self):
 133         return getattr(self, '_value', self.default)
 134
 135     @value.setter
 136     def value(self, newvalue):
 137         if newvalue is None:
 138             self._value = self.default
 139             return
 140         (start, end) = newvalue.split('-')
 141         (start, end) = (int(start), int(end))
 142         self._value = '%s-%s' % (start, end)
 143
 144 # Command specs
 145
 146
 147 class _pithos_init(_command_init):
 148     """Initialize a pithos+ kamaki client"""
 149
 150     @staticmethod
 151     def _is_dir(remote_dict):
 152         return 'application/directory' == remote_dict.get(
 153             'content_type',
 154             remote_dict.get('content-type', ''))
 155
 156     @errors.generic.all
 157     def _run(self):
 158         self.token = self.config.get('file', 'token')\
 159             or self.config.get('global', 'token')\
 160             or self.config.get('store', 'token')
 161         self.base_url = self.config.get('store', 'url')\
 162             or self.config.get('file', 'url')\
 163             or self.config.get('global', 'url')
 164         self._set_account()
 165         self.container = self.config.get('file', 'container')\
 166             or self.config.get('store', 'container')
 167         self.client = PithosClient(
 168             base_url=self.base_url,
 169             token=self.token,
 170             account=self.account,
 171             container=self.container)
 172         self._set_log_params()
 173         self._update_max_threads()
 174
 175     def main(self):
 176         self._run()
 177
 178     def _set_account(self):
 179         user_url = self.config.get('astakos', 'url')\
 180             or self.config.get('user', 'url')
 181         user = AstakosClient(user_url, self.token)
 182         self.account = self['account'] or user.term('uuid')\
 183             or self.config.get('file', 'account')\
 184             or self.config.get('store', 'account')
 185
 186
 187 class _file_account_command(_pithos_init):
 188     """Base class for account level storage commands"""
 189
 190     def __init__(self, arguments={}):
 191         super(_file_account_command, self).__init__(arguments)
 192         self['account'] = ValueArgument(
 193             'Set user account (not permanent)',
 194             ('-A', '--account'))
 195
 196     def _run(self, custom_account=None):
 197         super(_file_account_command, self)._run()
 198         if custom_account:
 199             self.client.account = custom_account
 200         elif self['account']:
 201             self.client.account = self['account']
 202
 203     @errors.generic.all
 204     def main(self):
 205         self._run()
 206
 207
 208 class _file_container_command(_file_account_command):
 209     """Base class for container level storage commands"""
 210
 211     container = None
 212     path = None
 213
 214     def __init__(self, arguments={}):
 215         super(_file_container_command, self).__init__(arguments)
 216         self['container'] = ValueArgument(
 217             'Set container to work with (temporary)',
 218             ('-C', '--container'))
 219
 220     def extract_container_and_path(
 221             self,
 222             container_with_path,
 223             path_is_optional=True):
 224         """Contains all heuristics for deciding what should be used as
 225         container or path. Options are:
 226         * user string of the form container:path
 227         * self.container, self.path variables set by super constructor, or
 228         explicitly by the caller application
 229         Error handling is explicit as these error cases happen only here
 230         """
 231         try:
 232             assert isinstance(container_with_path, str)
 233         except AssertionError as err:
 234             if self['container'] and path_is_optional:
 235                 self.container = self['container']
 236                 self.client.container = self['container']
 237                 return
 238             raiseCLIError(err)
 239
 240         user_cont, sep, userpath = container_with_path.partition(':')
 241
 242         if sep:
 243             if not user_cont:
 244                 raiseCLIError(CLISyntaxError(
 245                     'Container is missing\n',
 246                     details=errors.pithos.container_howto))
 247             alt_cont = self['container']
 248             if alt_cont and user_cont != alt_cont:
 249                 raiseCLIError(CLISyntaxError(
 250                     'Conflict: 2 containers (%s, %s)' % (user_cont, alt_cont),
 251                     details=errors.pithos.container_howto)
 252                 )
 253             self.container = user_cont
 254             if not userpath:
 255                 raiseCLIError(CLISyntaxError(
 256                     'Path is missing for object in container %s' % user_cont,
 257                     details=errors.pithos.container_howto)
 258                 )
 259             self.path = userpath
 260         else:
 261             alt_cont = self['container'] or self.client.container
 262             if alt_cont:
 263                 self.container = alt_cont
 264                 self.path = user_cont
 265             elif path_is_optional:
 266                 self.container = user_cont
 267                 self.path = None
 268             else:
 269                 self.container = user_cont
 270                 raiseCLIError(CLISyntaxError(
 271                     'Both container and path are required',
 272                     details=errors.pithos.container_howto)
 273                 )
 274
 275     @errors.generic.all
 276     def _run(self, container_with_path=None, path_is_optional=True):
 277         super(_file_container_command, self)._run()
 278         if self['container']:
 279             self.client.container = self['container']
 280             if container_with_path:
 281                 self.path = container_with_path
 282             elif not path_is_optional:
 283                 raise CLISyntaxError(
 284                     'Both container and path are required',
 285                     details=errors.pithos.container_howto)
 286         elif container_with_path:
 287             self.extract_container_and_path(
 288                 container_with_path,
 289                 path_is_optional)
 290             self.client.container = self.container
 291         self.container = self.client.container
 292
 293     def main(self, container_with_path=None, path_is_optional=True):
 294         self._run(container_with_path, path_is_optional)
 295
 296
 297 @command(pithos_cmds)
 298 class file_list(_file_container_command):
 299     """List containers, object trees or objects in a directory
 300     Use with:
 301     1 no parameters : containers in current account
 302     2. one parameter (container) or --container : contents of container
 303     3. <container>:<prefix> or --container=<container> <prefix>: objects in
 304     .   container starting with prefix
 305     """
 306
 307     arguments = dict(
 308         detail=FlagArgument('detailed output', ('-l', '--list')),
 309         limit=IntArgument('limit number of listed items', ('-n', '--number')),
 310         marker=ValueArgument('output greater that marker', '--marker'),
 311         prefix=ValueArgument('output starting with prefix', '--prefix'),
 312         delimiter=ValueArgument('show output up to delimiter', '--delimiter'),
 313         path=ValueArgument(
 314             'show output starting with prefix up to /',
 315             '--path'),
 316         meta=ValueArgument(
 317             'show output with specified meta keys',
 318             '--meta',
 319             default=[]),
 320         if_modified_since=ValueArgument(
 321             'show output modified since then',
 322             '--if-modified-since'),
 323         if_unmodified_since=ValueArgument(
 324             'show output not modified since then',
 325             '--if-unmodified-since'),
 326         until=DateArgument('show metadata until then', '--until'),
 327         format=ValueArgument(
 328             'format to parse until data (default: d/m/Y H:M:S )',
 329             '--format'),
 330         shared=FlagArgument('show only shared', '--shared'),
 331         more=FlagArgument(
 332             'output results in pages (-n to set items per page, default 10)',
 333             '--more'),
 334         exact_match=FlagArgument(
 335             'Show only objects that match exactly with path',
 336             '--exact-match')
 337     )
 338
 339     def print_objects(self, object_list):
 340         limit = int(self['limit']) if self['limit'] > 0 else len(object_list)
 341         for index, obj in enumerate(object_list):
 342             if self['exact_match'] and self.path and not (
 343                     obj['name'] == self.path or 'content_type' in obj):
 344                 continue
 345             pretty_obj = obj.copy()
 346             index += 1
 347             empty_space = ' ' * (len(str(len(object_list))) - len(str(index)))
 348             if obj['content_type'] == 'application/directory':
 349                 isDir = True
 350                 size = 'D'
 351             else:
 352                 isDir = False
 353                 size = format_size(obj['bytes'])
 354                 pretty_obj['bytes'] = '%s (%s)' % (obj['bytes'], size)
 355             oname = bold(obj['name'])
 356             if self['detail']:
 357                 print('%s%s. %s' % (empty_space, index, oname))
 358                 print_dict(pretty_keys(pretty_obj), exclude=('name'))
 359                 print
 360             else:
 361                 oname = '%s%s. %6s %s' % (empty_space, index, size, oname)
 362                 oname += '/' if isDir else ''
 363                 print(oname)
 364             if self['more']:
 365                 page_hold(index, limit, len(object_list))
 366
 367     def print_containers(self, container_list):
 368         limit = int(self['limit']) if self['limit'] > 0\
 369             else len(container_list)
 370         for index, container in enumerate(container_list):
 371             if 'bytes' in container:
 372                 size = format_size(container['bytes'])
 373             cname = '%s. %s' % (index + 1, bold(container['name']))
 374             if self['detail']:
 375                 print(cname)
 376                 pretty_c = container.copy()
 377                 if 'bytes' in container:
 378                     pretty_c['bytes'] = '%s (%s)' % (container['bytes'], size)
 379                 print_dict(pretty_keys(pretty_c), exclude=('name'))
 380                 print
 381             else:
 382                 if 'count' in container and 'bytes' in container:
 383                     print('%s (%s, %s objects)' % (
 384                         cname,
 385                         size,
 386                         container['count']))
 387                 else:
 388                     print(cname)
 389             if self['more']:
 390                 page_hold(index + 1, limit, len(container_list))
 391
 392     @errors.generic.all
 393     @errors.pithos.connection
 394     @errors.pithos.object_path
 395     @errors.pithos.container
 396     def _run(self):
 397         if self.container is None:
 398             r = self.client.account_get(
 399                 limit=False if self['more'] else self['limit'],
 400                 marker=self['marker'],
 401                 if_modified_since=self['if_modified_since'],
 402                 if_unmodified_since=self['if_unmodified_since'],
 403                 until=self['until'],
 404                 show_only_shared=self['shared'])
 405             self.print_containers(r.json)
 406         else:
 407             prefix = self.path or self['prefix']
 408             r = self.client.container_get(
 409                 limit=False if self['more'] else self['limit'],
 410                 marker=self['marker'],
 411                 prefix=prefix,
 412                 delimiter=self['delimiter'],
 413                 path=self['path'],
 414                 if_modified_since=self['if_modified_since'],
 415                 if_unmodified_since=self['if_unmodified_since'],
 416                 until=self['until'],
 417                 meta=self['meta'],
 418                 show_only_shared=self['shared'])
 419             self.print_objects(r.json)
 420
 421     def main(self, container____path__=None):
 422         super(self.__class__, self)._run(container____path__)
 423         self._run()
 424
 425
 426 @command(pithos_cmds)
 427 class file_mkdir(_file_container_command):
 428     """Create a directory"""
 429
 430     __doc__ += '\n. '.join([
 431         'Kamaki hanldes directories the same way as OOS Storage and Pithos+:',
 432         'A   directory  is   an  object  with  type  "application/directory"',
 433         'An object with path  dir/name can exist even if  dir does not exist',
 434         'or even if dir  is  a non  directory  object.  Users can modify dir',
 435         'without affecting the dir/name object in any way.'])
 436
 437     @errors.generic.all
 438     @errors.pithos.connection
 439     @errors.pithos.container
 440     def _run(self):
 441         self.client.create_directory(self.path)
 442
 443     def main(self, container___directory):
 444         super(self.__class__, self)._run(
 445             container___directory,
 446             path_is_optional=False)
 447         self._run()
 448
 449
 450 @command(pithos_cmds)
 451 class file_touch(_file_container_command):
 452     """Create an empty object (file)
 453     If object exists, this command will reset it to 0 length
 454     """
 455
 456     arguments = dict(
 457         content_type=ValueArgument(
 458             'Set content type (default: application/octet-stream)',
 459             '--content-type',
 460             default='application/octet-stream')
 461     )
 462
 463     @errors.generic.all
 464     @errors.pithos.connection
 465     @errors.pithos.container
 466     def _run(self):
 467         self.client.create_object(self.path, self['content_type'])
 468
 469     def main(self, container___path):
 470         super(file_touch, self)._run(
 471             container___path,
 472             path_is_optional=False)
 473         self._run()
 474
 475
 476 @command(pithos_cmds)
 477 class file_create(_file_container_command):
 478     """Create a container"""
 479
 480     arguments = dict(
 481         versioning=ValueArgument(
 482             'set container versioning (auto/none)',
 483             '--versioning'),
 484         limit=IntArgument('set default container limit', '--limit'),
 485         meta=KeyValueArgument(
 486             'set container metadata (can be repeated)',
 487             '--meta')
 488     )
 489
 490     @errors.generic.all
 491     @errors.pithos.connection
 492     @errors.pithos.container
 493     def _run(self):
 494         self.client.container_put(
 495             limit=self['limit'],
 496             versioning=self['versioning'],
 497             metadata=self['meta'])
 498
 499     def main(self, container=None):
 500         super(self.__class__, self)._run(container)
 501         if container and self.container != container:
 502             raiseCLIError('Invalid container name %s' % container, details=[
 503                 'Did you mean "%s" ?' % self.container,
 504                 'Use --container for names containing :'])
 505         self._run()
 506
 507
 508 class _source_destination_command(_file_container_command):
 509
 510     arguments = dict(
 511         destination_account=ValueArgument('', ('a', '--dst-account')),
 512         recursive=FlagArgument('', ('-R', '--recursive')),
 513         prefix=FlagArgument('', '--with-prefix', default=''),
 514         suffix=ValueArgument('', '--with-suffix', default=''),
 515         add_prefix=ValueArgument('', '--add-prefix', default=''),
 516         add_suffix=ValueArgument('', '--add-suffix', default=''),
 517         prefix_replace=ValueArgument('', '--prefix-to-replace', default=''),
 518         suffix_replace=ValueArgument('', '--suffix-to-replace', default='')
 519     )
 520
 521     def __init__(self, arguments={}):
 522         self.arguments.update(arguments)
 523         super(_source_destination_command, self).__init__(self.arguments)
 524
 525     def _run(self, source_container___path, path_is_optional=False):
 526         super(_source_destination_command, self)._run(
 527             source_container___path,
 528             path_is_optional)
 529         self.dst_client = PithosClient(
 530             base_url=self.client.base_url,
 531             token=self.client.token,
 532             account=self['destination_account'] or self.client.account)
 533
 534     @errors.generic.all
 535     @errors.pithos.account
 536     def _dest_container_path(self, dest_container_path):
 537         if self['destination_container']:
 538             self.dst_client.container = self['destination_container']
 539             return (self['destination_container'], dest_container_path)
 540         if dest_container_path:
 541             dst = dest_container_path.split(':')
 542             if len(dst) > 1:
 543                 try:
 544                     self.dst_client.container = dst[0]
 545                     self.dst_client.get_container_info(dst[0])
 546                 except ClientError as err:
 547                     if err.status in (404, 204):
 548                         raiseCLIError(
 549                             'Destination container %s not found' % dst[0])
 550                     raise
 551                 else:
 552                     self.dst_client.container = dst[0]
 553                 return (dst[0], dst[1])
 554             return(None, dst[0])
 555         raiseCLIError('No destination container:path provided')
 556
 557     def _get_all(self, prefix):
 558         return self.client.container_get(prefix=prefix).json
 559
 560     def _get_src_objects(self, src_path):
 561         """Get a list of the source objects to be called
 562
 563         :param src_path: (str) source path
 564
 565         :returns: (method, params) a method that returns a list when called
 566         or (object) if it is a single object
 567         """
 568         if src_path and src_path[-1] == '/':
 569             src_path = src_path[:-1]
 570
 571         if self['prefix']:
 572             return (self._get_all, dict(prefix=src_path))
 573         try:
 574             srcobj = self.client.get_object_info(src_path)
 575         except ClientError as srcerr:
 576             if srcerr.status == 404:
 577                 raiseCLIError(
 578                     'Source object %s not in source container %s' % (
 579                         src_path,
 580                         self.client.container),
 581                     details=['Hint: --with-prefix to match multiple objects'])
 582             elif srcerr.status not in (204,):
 583                 raise
 584             return (self.client.list_objects, {})
 585
 586         if self._is_dir(srcobj):
 587             if not self['recursive']:
 588                 raiseCLIError(
 589                     'Object %s of cont. %s is a dir' % (
 590                         src_path,
 591                         self.client.container),
 592                     details=['Use --recursive to access directories'])
 593             return (self._get_all, dict(prefix=src_path))
 594         srcobj['name'] = src_path
 595         return srcobj
 596
 597     def src_dst_pairs(self, ds_path):
 598         src_iter = self._get_src_objects(self.path)
 599         src_N = isinstance(src_iter, tuple)
 600         add_prefix = self['add_prefix'].strip('/')
 601
 602         if dst_path and dst_path.endswith('/'):
 603             dst_path = dst_path[:-1]
 604
 605         try:
 606             dstobj = self.dst_client.get_object_info(dst_path)
 607         except ClientError as trgerr:
 608             if trgerr.status in (404,):
 609                 if src_N:
 610                     raiseCLIError(
 611                         'Cannot merge multiple paths to path %s' % dst_path,
 612                         details=[
 613                             'Try to use / or a directory as destination',
 614                             'or create the destination dir (/file mkdir)',
 615                             'or use a single object as source'])
 616             elif trgerr.status not in (204,):
 617                 raise
 618         else:
 619             if self._is_dir(dstobj):
 620                 add_prefix = '%s/%s' % (dst_path.strip('/'), add_prefix)
 621             elif src_N:
 622                 raiseCLIError(
 623                     'Cannot merge multiple paths to path' % dst_path,
 624                     details=[
 625                         'Try to use / or a directory as destination',
 626                         'or create the destination dir (/file mkdir)',
 627                         'or use a single object as source'])
 628
 629         if src_N:
 630             (method, kwargs) = src_iter
 631             for obj in method(**kwargs):
 632                 name = obj['name']
 633                 if name.endswith(self['suffix']):
 634                     yield (name, self._get_new_object(name, add_prefix))
 635         elif src_iter['name'].endswith(self['suffix']):
 636             name = src_iter['name']
 637             yield (name, self._get_new_object(dst_path or name, add_prefix))
 638         else:
 639             raiseCLIError('Source path %s conflicts with suffix %s' % (
 640                 src_iter['name'],
 641                 self['suffix']))
 642
 643     def _get_new_object(self, obj, add_prefix):
 644         if self['prefix_replace'] and obj.startswith(self['prefix_replace']):
 645             obj = obj[len(self['prefix_replace']):]
 646         if self['suffix_replace'] and obj.endswith(self['suffix_replace']):
 647             obj = obj[:-len(self['suffix_replace'])]
 648         return add_prefix + obj + self['add_suffix']
 649
 650
 651 @command(pithos_cmds)
 652 class file_copy(_source_destination_command):
 653     """Copy objects from container to (another) container
 654     Semantics:
 655     copy cont:path dir
 656     .   transfer path as dir/path
 657     copy cont:path cont2:
 658     .   trasnfer all <obj> prefixed with path to container cont2
 659     copy cont:path [cont2:]path2
 660     .   transfer path to path2
 661     Use options:
 662     1. <container1>:<path1> [container2:]<path2> : if container2 is not given,
 663     destination is container1:path2
 664     2. <container>:<path1> <path2> : make a copy in the same container
 665     3. Can use --container= instead of <container1>
 666     """
 667
 668     arguments = dict(
 669         destination_account=ValueArgument(
 670             'Account to copy to',
 671             ('-a', '--dst-account')),
 672         destination_container=ValueArgument(
 673             'use it if destination container name contains a : character',
 674             ('-D', '--dst-container')),
 675         source_version=ValueArgument(
 676             'copy specific version',
 677             ('-S', '--source-version')),
 678         public=ValueArgument('make object publicly accessible', '--public'),
 679         content_type=ValueArgument(
 680             'change object\'s content type',
 681             '--content-type'),
 682         recursive=FlagArgument(
 683             'copy directory and contents',
 684             ('-R', '--recursive')),
 685         prefix=FlagArgument(
 686             'Match objects prefixed with src path (feels like src_path*)',
 687             '--with-prefix',
 688             default=''),
 689         suffix=ValueArgument(
 690             'Suffix of source objects (feels like *suffix)',
 691             '--with-suffix',
 692             default=''),
 693         add_prefix=ValueArgument('Prefix targets', '--add-prefix', default=''),
 694         add_suffix=ValueArgument('Suffix targets', '--add-suffix', default=''),
 695         prefix_replace=ValueArgument(
 696             'Prefix of src to replace with dst path + add_prefix, if matched',
 697             '--prefix-to-replace',
 698             default=''),
 699         suffix_replace=ValueArgument(
 700             'Suffix of src to replace with add_suffix, if matched',
 701             '--suffix-to-replace',
 702             default='')
 703     )
 704
 705     @errors.generic.all
 706     @errors.pithos.connection
 707     @errors.pithos.container
 708     @errors.pithos.account
 709     def _run(self, dst_path):
 710         no_source_object = True
 711         src_account = self.client.account if (
 712             self['destination_account']) else None
 713         for src_obj, dst_obj in self.src_dst_pairs(dst_path):
 714             no_source_object = False
 715             self.dst_client.copy_object(
 716                 src_container=self.client.container,
 717                 src_object=src_obj,
 718                 dst_container=self.dst_client.container,
 719                 dst_object=dst_obj,
 720                 source_account=src_account,
 721                 source_version=self['source_version'],
 722                 public=self['public'],
 723                 content_type=self['content_type'])
 724         if no_source_object:
 725             raiseCLIError('No object %s in container %s' % (
 726                 self.path,
 727                 self.container))
 728
 729     def main(
 730             self,
 731             source_container___path,
 732             destination_container___path=None):
 733         super(file_copy, self)._run(
 734             source_container___path,
 735             path_is_optional=False)
 736         (dst_cont, dst_path) = self._dest_container_path(
 737             destination_container___path)
 738         self.dst_client.container = dst_cont or self.container
 739         self._run(dst_path=dst_path or '')
 740
 741
 742 @command(pithos_cmds)
 743 class file_move(_source_destination_command):
 744     """Move/rename objects from container to (another) container
 745     Semantics:
 746     move cont:path dir
 747     .   rename path as dir/path
 748     move cont:path cont2:
 749     .   trasnfer all <obj> prefixed with path to container cont2
 750     move cont:path [cont2:]path2
 751     .   transfer path to path2
 752     Use options:
 753     1. <container1>:<path1> [container2:]<path2> : if container2 is not given,
 754     destination is container1:path2
 755     2. <container>:<path1> <path2> : move in the same container
 756     3. Can use --container= instead of <container1>
 757     """
 758
 759     arguments = dict(
 760         destination_account=ValueArgument(
 761             'Account to move to',
 762             ('-a', '--dst-account')),
 763         destination_container=ValueArgument(
 764             'use it if destination container name contains a : character',
 765             ('-D', '--dst-container')),
 766         source_version=ValueArgument(
 767             'copy specific version',
 768             '--source-version'),
 769         public=ValueArgument('make object publicly accessible', '--public'),
 770         content_type=ValueArgument(
 771             'change object\'s content type',
 772             '--content-type'),
 773         recursive=FlagArgument(
 774             'copy directory and contents',
 775             ('-R', '--recursive')),
 776         prefix=FlagArgument(
 777             'Match objects prefixed with src path (feels like src_path*)',
 778             '--with-prefix',
 779             default=''),
 780         suffix=ValueArgument(
 781             'Suffix of source objects (feels like *suffix)',
 782             '--with-suffix',
 783             default=''),
 784         add_prefix=ValueArgument('Prefix targets', '--add-prefix', default=''),
 785         add_suffix=ValueArgument('Suffix targets', '--add-suffix', default=''),
 786         prefix_replace=ValueArgument(
 787             'Prefix of src to replace with dst path + add_prefix, if matched',
 788             '--prefix-to-replace',
 789             default=''),
 790         suffix_replace=ValueArgument(
 791             'Suffix of src to replace with add_suffix, if matched',
 792             '--suffix-to-replace',
 793             default='')
 794     )
 795
 796     @errors.generic.all
 797     @errors.pithos.connection
 798     @errors.pithos.container
 799     def _run(self, dst_path):
 800         no_source_object = True
 801         src_account = self.client.account if (
 802             self['destination_account']) else None
 803         for src_obj, dst_obj in self.src_dst_pairs(dst_path):
 804             no_source_object = False
 805             self.dst_client.move_object(
 806                 src_container=self.container,
 807                 src_object=src_obj,
 808                 dst_container=self.dst_client.container,
 809                 dst_object=dst_obj,
 810                 source_account=src_account,
 811                 source_version=self['source_version'],
 812                 public=self['public'],
 813                 content_type=self['content_type'])
 814         if no_source_object:
 815             raiseCLIError('No object %s in container %s' % (
 816                 self.path,
 817                 self.container))
 818
 819     def main(
 820             self,
 821             source_container___path,
 822             destination_container___path=None):
 823         super(self.__class__, self)._run(
 824             source_container___path,
 825             path_is_optional=False)
 826         (dst_cont, dst_path) = self._dest_container_path(
 827             destination_container___path)
 828         (dst_cont, dst_path) = self._dest_container_path(
 829             destination_container___path)
 830         self.dst_client.container = dst_cont or self.container
 831         self._run(dst_path=dst_path or '')
 832
 833
 834 @command(pithos_cmds)
 835 class file_append(_file_container_command):
 836     """Append local file to (existing) remote object
 837     The remote object should exist.
 838     If the remote object is a directory, it is transformed into a file.
 839     In the later case, objects under the directory remain intact.
 840     """
 841
 842     arguments = dict(
 843         progress_bar=ProgressBarArgument(
 844             'do not show progress bar',
 845             ('-N', '--no-progress-bar'),
 846             default=False)
 847     )
 848
 849     @errors.generic.all
 850     @errors.pithos.connection
 851     @errors.pithos.container
 852     @errors.pithos.object_path
 853     def _run(self, local_path):
 854         (progress_bar, upload_cb) = self._safe_progress_bar('Appending')
 855         try:
 856             f = open(local_path, 'rb')
 857             self.client.append_object(self.path, f, upload_cb)
 858         except Exception:
 859             self._safe_progress_bar_finish(progress_bar)
 860             raise
 861         finally:
 862             self._safe_progress_bar_finish(progress_bar)
 863
 864     def main(self, local_path, container___path):
 865         super(self.__class__, self)._run(
 866             container___path,
 867             path_is_optional=False)
 868         self._run(local_path)
 869
 870
 871 @command(pithos_cmds)
 872 class file_truncate(_file_container_command):
 873     """Truncate remote file up to a size (default is 0)"""
 874
 875     @errors.generic.all
 876     @errors.pithos.connection
 877     @errors.pithos.container
 878     @errors.pithos.object_path
 879     @errors.pithos.object_size
 880     def _run(self, size=0):
 881         self.client.truncate_object(self.path, size)
 882
 883     def main(self, container___path, size=0):
 884         super(self.__class__, self)._run(container___path)
 885         self._run(size=size)
 886
 887
 888 @command(pithos_cmds)
 889 class file_overwrite(_file_container_command):
 890     """Overwrite part (from start to end) of a remote file
 891     overwrite local-path container 10 20
 892     .   will overwrite bytes from 10 to 20 of a remote file with the same name
 893     .   as local-path basename
 894     overwrite local-path container:path 10 20
 895     .   will overwrite as above, but the remote file is named path
 896     """
 897
 898     arguments = dict(
 899         progress_bar=ProgressBarArgument(
 900             'do not show progress bar',
 901             ('-N', '--no-progress-bar'),
 902             default=False)
 903     )
 904
 905     def _open_file(self, local_path, start):
 906         f = open(path.abspath(local_path), 'rb')
 907         f.seek(0, 2)
 908         f_size = f.tell()
 909         f.seek(start, 0)
 910         return (f, f_size)
 911
 912     @errors.generic.all
 913     @errors.pithos.connection
 914     @errors.pithos.container
 915     @errors.pithos.object_path
 916     @errors.pithos.object_size
 917     def _run(self, local_path, start, end):
 918         (start, end) = (int(start), int(end))
 919         (f, f_size) = self._open_file(local_path, start)
 920         (progress_bar, upload_cb) = self._safe_progress_bar(
 921             'Overwrite %s bytes' % (end - start))
 922         try:
 923             self.client.overwrite_object(
 924                 obj=self.path,
 925                 start=start,
 926                 end=end,
 927                 source_file=f,
 928                 upload_cb=upload_cb)
 929         except Exception:
 930             self._safe_progress_bar_finish(progress_bar)
 931             raise
 932         finally:
 933             self._safe_progress_bar_finish(progress_bar)
 934
 935     def main(self, local_path, container___path, start, end):
 936         super(self.__class__, self)._run(
 937             container___path,
 938             path_is_optional=None)
 939         self.path = self.path or path.basename(local_path)
 940         self._run(local_path=local_path, start=start, end=end)
 941
 942
 943 @command(pithos_cmds)
 944 class file_manifest(_file_container_command):
 945     """Create a remote file of uploaded parts by manifestation
 946     Remains functional for compatibility with OOS Storage. Users are advised
 947     to use the upload command instead.
 948     Manifestation is a compliant process for uploading large files. The files
 949     have to be chunked in smalled files and uploaded as <prefix><increment>
 950     where increment is 1, 2, ...
 951     Finally, the manifest command glues partial files together in one file
 952     named <prefix>
 953     The upload command is faster, easier and more intuitive than manifest
 954     """
 955
 956     arguments = dict(
 957         etag=ValueArgument('check written data', '--etag'),
 958         content_encoding=ValueArgument(
 959             'set MIME content type',
 960             '--content-encoding'),
 961         content_disposition=ValueArgument(
 962             'the presentation style of the object',
 963             '--content-disposition'),
 964         content_type=ValueArgument(
 965             'specify content type',
 966             '--content-type',
 967             default='application/octet-stream'),
 968         sharing=SharingArgument(
 969             '\n'.join([
 970                 'define object sharing policy',
 971                 '    ( "read=user1,grp1,user2,... write=user1,grp2,..." )']),
 972             '--sharing'),
 973         public=FlagArgument('make object publicly accessible', '--public')
 974     )
 975
 976     @errors.generic.all
 977     @errors.pithos.connection
 978     @errors.pithos.container
 979     @errors.pithos.object_path
 980     def _run(self):
 981         self.client.create_object_by_manifestation(
 982             self.path,
 983             content_encoding=self['content_encoding'],
 984             content_disposition=self['content_disposition'],
 985             content_type=self['content_type'],
 986             sharing=self['sharing'],
 987             public=self['public'])
 988
 989     def main(self, container___path):
 990         super(self.__class__, self)._run(
 991             container___path,
 992             path_is_optional=False)
 993         self.run()
 994
 995
 996 @command(pithos_cmds)
 997 class file_upload(_file_container_command):
 998     """Upload a file"""
 999
1000     arguments = dict(
1001         use_hashes=FlagArgument(
1002             'provide hashmap file instead of data',
1003             '--use-hashes'),
1004         etag=ValueArgument('check written data', '--etag'),
1005         unchunked=FlagArgument('avoid chunked transfer mode', '--unchunked'),
1006         content_encoding=ValueArgument(
1007             'set MIME content type',
1008             '--content-encoding'),
1009         content_disposition=ValueArgument(
1010             'specify objects presentation style',
1011             '--content-disposition'),
1012         content_type=ValueArgument('specify content type', '--content-type'),
1013         sharing=SharingArgument(
1014             help='\n'.join([
1015                 'define sharing object policy',
1016                 '( "read=user1,grp1,user2,... write=user1,grp2,... )']),
1017             parsed_name='--sharing'),
1018         public=FlagArgument('make object publicly accessible', '--public'),
1019         poolsize=IntArgument('set pool size', '--with-pool-size'),
1020         progress_bar=ProgressBarArgument(
1021             'do not show progress bar',
1022             ('-N', '--no-progress-bar'),
1023             default=False),
1024         overwrite=FlagArgument('Force (over)write', ('-f', '--force')),
1025         recursive=FlagArgument(
1026             'Recursively upload directory *contents* + subdirectories',
1027             ('-R', '--recursive'))
1028     )
1029
1030     def _check_container_limit(self, path):
1031         cl_dict = self.client.get_container_limit()
1032         container_limit = int(cl_dict['x-container-policy-quota'])
1033         r = self.client.container_get()
1034         used_bytes = sum(int(o['bytes']) for o in r.json)
1035         path_size = get_path_size(path)
1036         if container_limit and path_size > (container_limit - used_bytes):
1037             raiseCLIError(
1038                 'Container(%s) (limit(%s) - used(%s)) < size(%s) of %s' % (
1039                     self.client.container,
1040                     format_size(container_limit),
1041                     format_size(used_bytes),
1042                     format_size(path_size),
1043                     path),
1044                 importance=1, details=[
1045                     'Check accound limit: /file quota',
1046                     'Check container limit:',
1047                     '\t/file containerlimit get %s' % self.client.container,
1048                     'Increase container limit:',
1049                     '\t/file containerlimit set <new limit> %s' % (
1050                         self.client.container)])
1051
1052     def _path_pairs(self, local_path, remote_path):
1053         """Get pairs of local and remote paths"""
1054         lpath = path.abspath(local_path)
1055         short_path = lpath.split(path.sep)[-1]
1056         rpath = remote_path or short_path
1057         if path.isdir(lpath):
1058             if not self['recursive']:
1059                 raiseCLIError('%s is a directory' % lpath, details=[
1060                     'Use -R to upload directory contents'])
1061             robj = self.client.container_get(path=rpath)
1062             if robj.json and not self['overwrite']:
1063                 raiseCLIError(
1064                     'Objects prefixed with %s already exist' % rpath,
1065                     importance=1,
1066                     details=['Existing objects:'] + ['\t%s:\t%s' % (
1067                         o['content_type'][12:],
1068                         o['name']) for o in robj.json] + [
1069                         'Use -f to add, overwrite or resume'])
1070             if not self['overwrite']:
1071                 try:
1072                     topobj = self.client.get_object_info(rpath)
1073                     if not self._is_dir(topobj):
1074                         raiseCLIError(
1075                             'Object %s exists but it is not a dir' % rpath,
1076                             importance=1, details=['Use -f to overwrite'])
1077                 except ClientError as ce:
1078                     if ce.status != 404:
1079                         raise
1080             self._check_container_limit(lpath)
1081             prev = ''
1082             for top, subdirs, files in walk(lpath):
1083                 if top != prev:
1084                     prev = top
1085                     try:
1086                         rel_path = rpath + top.split(lpath)[1]
1087                     except IndexError:
1088                         rel_path = rpath
1089                     print('mkdir %s:%s' % (self.client.container, rel_path))
1090                     self.client.create_directory(rel_path)
1091                 for f in files:
1092                     fpath = path.join(top, f)
1093                     if path.isfile(fpath):
1094                         yield open(fpath, 'rb'), '%s/%s' % (rel_path, f)
1095                     else:
1096                         print('%s is not a regular file' % fpath)
1097         else:
1098             if not path.isfile(lpath):
1099                 raiseCLIError('%s is not a regular file' % lpath)
1100             try:
1101                 robj = self.client.get_object_info(rpath)
1102                 if remote_path and self._is_dir(robj):
1103                     rpath += '/%s' % short_path
1104                     self.client.get_object_info(rpath)
1105                 if not self['overwrite']:
1106                     raiseCLIError(
1107                         'Object %s already exists' % rpath,
1108                         importance=1,
1109                         details=['use -f to overwrite or resume'])
1110             except ClientError as ce:
1111                 if ce.status != 404:
1112                     raise
1113             self._check_container_limit(lpath)
1114             yield open(lpath, 'rb'), rpath
1115
1116     @errors.generic.all
1117     @errors.pithos.connection
1118     @errors.pithos.container
1119     @errors.pithos.object_path
1120     @errors.pithos.local_path
1121     def _run(self, local_path, remote_path):
1122         poolsize = self['poolsize']
1123         if poolsize > 0:
1124             self.client.MAX_THREADS = int(poolsize)
1125         params = dict(
1126             content_encoding=self['content_encoding'],
1127             content_type=self['content_type'],
1128             content_disposition=self['content_disposition'],
1129             sharing=self['sharing'],
1130             public=self['public'])
1131         for f, rpath in self._path_pairs(local_path, remote_path):
1132             print('%s --> %s:%s' % (f.name, self.client.container, rpath))
1133             if self['unchunked']:
1134                 self.client.upload_object_unchunked(
1135                     rpath, f,
1136                     etag=self['etag'], withHashFile=self['use_hashes'],
1137                     **params)
1138             else:
1139                 try:
1140                     (progress_bar, upload_cb) = self._safe_progress_bar(
1141                         'Uploading %s' % f.name.split(path.sep)[-1])
1142                     if progress_bar:
1143                         hash_bar = progress_bar.clone()
1144                         hash_cb = hash_bar.get_generator(
1145                             'Calculating block hashes')
1146                     else:
1147                         hash_cb = None
1148                     self.client.upload_object(
1149                         rpath, f,
1150                         hash_cb=hash_cb, upload_cb=upload_cb,
1151                         **params)
1152                 except Exception:
1153                     self._safe_progress_bar_finish(progress_bar)
1154                     raise
1155                 finally:
1156                     self._safe_progress_bar_finish(progress_bar)
1157         print 'Upload completed'
1158
1159     def main(self, local_path, container____path__=None):
1160         super(self.__class__, self)._run(container____path__)
1161         remote_path = self.path or path.basename(local_path)
1162         self._run(local_path=local_path, remote_path=remote_path)
1163
1164
1165 @command(pithos_cmds)
1166 class file_cat(_file_container_command):
1167     """Print remote file contents to console"""
1168
1169     arguments = dict(
1170         range=RangeArgument('show range of data', '--range'),
1171         if_match=ValueArgument('show output if ETags match', '--if-match'),
1172         if_none_match=ValueArgument(
1173             'show output if ETags match',
1174             '--if-none-match'),
1175         if_modified_since=DateArgument(
1176             'show output modified since then',
1177             '--if-modified-since'),
1178         if_unmodified_since=DateArgument(
1179             'show output unmodified since then',
1180             '--if-unmodified-since'),
1181         object_version=ValueArgument(
1182             'get the specific version',
1183             ('-j', '--object-version'))
1184     )
1185
1186     @errors.generic.all
1187     @errors.pithos.connection
1188     @errors.pithos.container
1189     @errors.pithos.object_path
1190     def _run(self):
1191         self.client.download_object(
1192             self.path,
1193             stdout,
1194             range_str=self['range'],
1195             version=self['object_version'],
1196             if_match=self['if_match'],
1197             if_none_match=self['if_none_match'],
1198             if_modified_since=self['if_modified_since'],
1199             if_unmodified_since=self['if_unmodified_since'])
1200
1201     def main(self, container___path):
1202         super(self.__class__, self)._run(
1203             container___path,
1204             path_is_optional=False)
1205         self._run()
1206
1207
1208 @command(pithos_cmds)
1209 class file_download(_file_container_command):
1210     """Download remote object as local file
1211     If local destination is a directory:
1212     *   download <container>:<path> <local dir> -R
1213     will download all files on <container> prefixed as <path>,
1214     to <local dir>/<full path>
1215     *   download <container>:<path> <local dir> --exact-match
1216     will download only one file, exactly matching <path>
1217     ATTENTION: to download cont:dir1/dir2/file there must exist objects
1218     cont:dir1 and cont:dir1/dir2 of type application/directory
1219     To create directory objects, use /file mkdir
1220     """
1221
1222     arguments = dict(
1223         resume=FlagArgument('Resume instead of overwrite', ('-r', '--resume')),
1224         range=RangeArgument('show range of data', '--range'),
1225         if_match=ValueArgument('show output if ETags match', '--if-match'),
1226         if_none_match=ValueArgument(
1227             'show output if ETags match',
1228             '--if-none-match'),
1229         if_modified_since=DateArgument(
1230             'show output modified since then',
1231             '--if-modified-since'),
1232         if_unmodified_since=DateArgument(
1233             'show output unmodified since then',
1234             '--if-unmodified-since'),
1235         object_version=ValueArgument(
1236             'get the specific version',
1237             ('-j', '--object-version')),
1238         poolsize=IntArgument('set pool size', '--with-pool-size'),
1239         progress_bar=ProgressBarArgument(
1240             'do not show progress bar',
1241             ('-N', '--no-progress-bar'),
1242             default=False),
1243         recursive=FlagArgument(
1244             'Download a remote path and all its contents',
1245             ('-R', '--recursive'))
1246     )
1247
1248     def _outputs(self, local_path):
1249         """:returns: (local_file, remote_path)"""
1250         remotes = []
1251         if self['recursive']:
1252             r = self.client.container_get(
1253                 prefix=self.path or '/',
1254                 if_modified_since=self['if_modified_since'],
1255                 if_unmodified_since=self['if_unmodified_since'])
1256             dirlist = dict()
1257             for remote in r.json:
1258                 rname = remote['name'].strip('/')
1259                 tmppath = ''
1260                 for newdir in rname.strip('/').split('/')[:-1]:
1261                     tmppath = '/'.join([tmppath, newdir])
1262                     dirlist.update({tmppath.strip('/'): True})
1263                 remotes.append((rname, file_download._is_dir(remote)))
1264             dir_remotes = [r[0] for r in remotes if r[1]]
1265             if not set(dirlist).issubset(dir_remotes):
1266                 badguys = [bg.strip('/') for bg in set(
1267                     dirlist).difference(dir_remotes)]
1268                 raiseCLIError(
1269                     'Some remote paths contain non existing directories',
1270                     details=['Missing remote directories:'] + badguys)
1271         elif self.path:
1272             r = self.client.get_object_info(
1273                 self.path,
1274                 version=self['object_version'])
1275             if file_download._is_dir(r):
1276                 raiseCLIError(
1277                     'Illegal download: Remote object %s is a directory' % (
1278                         self.path),
1279                     details=['To download a directory, try --recursive'])
1280             if '/' in self.path.strip('/') and not local_path:
1281                 raiseCLIError(
1282                     'Illegal download: remote object %s contains "/"' % (
1283                         self.path),
1284                     details=[
1285                         'To download an object containing "/" characters',
1286                         'either create the remote directories or',
1287                         'specify a non-directory local path for this object'])
1288             remotes = [(self.path, False)]
1289         if not remotes:
1290             if self.path:
1291                 raiseCLIError(
1292                     'No matching path %s on container %s' % (
1293                         self.path,
1294                         self.container),
1295                     details=[
1296                         'To list the contents of %s, try:' % self.container,
1297                         '   /file list %s' % self.container])
1298             raiseCLIError(
1299                 'Illegal download of container %s' % self.container,
1300                 details=[
1301                     'To download a whole container, try:',
1302                     '   /file download --recursive <container>'])
1303
1304         lprefix = path.abspath(local_path or path.curdir)
1305         if path.isdir(lprefix):
1306             for rpath, remote_is_dir in remotes:
1307                 lpath = '/%s/%s' % (lprefix.strip('/'), rpath.strip('/'))
1308                 if remote_is_dir:
1309                     if path.exists(lpath) and path.isdir(lpath):
1310                         continue
1311                     makedirs(lpath)
1312                 elif path.exists(lpath):
1313                     if not self['resume']:
1314                         print('File %s exists, aborting...' % lpath)
1315                         continue
1316                     with open(lpath, 'rwb+') as f:
1317                         yield (f, rpath)
1318                 else:
1319                     with open(lpath, 'wb+') as f:
1320                         yield (f, rpath)
1321         elif path.exists(lprefix):
1322             if len(remotes) > 1:
1323                 raiseCLIError(
1324                     '%s remote objects cannot be merged in local file %s' % (
1325                         len(remotes),
1326                         local_path),
1327                     details=[
1328                         'To download multiple objects, local path should be',
1329                         'a directory, or use download without a local path'])
1330             (rpath, remote_is_dir) = remotes[0]
1331             if remote_is_dir:
1332                 raiseCLIError(
1333                     'Remote directory %s should not replace local file %s' % (
1334                         rpath,
1335                         local_path))
1336             if self['resume']:
1337                 with open(lprefix, 'rwb+') as f:
1338                     yield (f, rpath)
1339             else:
1340                 raiseCLIError(
1341                     'Local file %s already exist' % local_path,
1342                     details=['Try --resume to overwrite it'])
1343         else:
1344             if len(remotes) > 1 or remotes[0][1]:
1345                 raiseCLIError(
1346                     'Local directory %s does not exist' % local_path)
1347             with open(lprefix, 'wb+') as f:
1348                 yield (f, remotes[0][0])
1349
1350     @errors.generic.all
1351     @errors.pithos.connection
1352     @errors.pithos.container
1353     @errors.pithos.object_path
1354     @errors.pithos.local_path
1355     def _run(self, local_path):
1356         #outputs = self._outputs(local_path)
1357         poolsize = self['poolsize']
1358         if poolsize:
1359             self.client.MAX_THREADS = int(poolsize)
1360         progress_bar = None
1361         try:
1362             for f, rpath in self._outputs(local_path):
1363                 (
1364                     progress_bar,
1365                     download_cb) = self._safe_progress_bar(
1366                         'Download %s' % rpath)
1367                 self.client.download_object(
1368                     rpath,
1369                     f,
1370                     download_cb=download_cb,
1371                     range_str=self['range'],
1372                     version=self['object_version'],
1373                     if_match=self['if_match'],
1374                     resume=self['resume'],
1375                     if_none_match=self['if_none_match'],
1376                     if_modified_since=self['if_modified_since'],
1377                     if_unmodified_since=self['if_unmodified_since'])
1378         except KeyboardInterrupt:
1379             from threading import activeCount, enumerate as activethreads
1380             timeout = 0.5
1381             while activeCount() > 1:
1382                 stdout.write('\nCancel %s threads: ' % (activeCount() - 1))
1383                 stdout.flush()
1384                 for thread in activethreads():
1385                     try:
1386                         thread.join(timeout)
1387                         stdout.write('.' if thread.isAlive() else '*')
1388                     except RuntimeError:
1389                         continue
1390                     finally:
1391                         stdout.flush()
1392                         timeout += 0.1
1393
1394             print('\nDownload canceled by user')
1395             if local_path is not None:
1396                 print('to resume, re-run with --resume')
1397         except Exception:
1398             self._safe_progress_bar_finish(progress_bar)
1399             raise
1400         finally:
1401             self._safe_progress_bar_finish(progress_bar)
1402
1403     def main(self, container___path, local_path=None):
1404         super(self.__class__, self)._run(container___path)
1405         self._run(local_path=local_path)
1406
1407
1408 @command(pithos_cmds)
1409 class file_hashmap(_file_container_command):
1410     """Get the hash-map of an object"""
1411
1412     arguments = dict(
1413         if_match=ValueArgument('show output if ETags match', '--if-match'),
1414         if_none_match=ValueArgument(
1415             'show output if ETags match',
1416             '--if-none-match'),
1417         if_modified_since=DateArgument(
1418             'show output modified since then',
1419             '--if-modified-since'),
1420         if_unmodified_since=DateArgument(
1421             'show output unmodified since then',
1422             '--if-unmodified-since'),
1423         object_version=ValueArgument(
1424             'get the specific version',
1425             ('-j', '--object-version'))
1426     )
1427
1428     @errors.generic.all
1429     @errors.pithos.connection
1430     @errors.pithos.container
1431     @errors.pithos.object_path
1432     def _run(self):
1433         data = self.client.get_object_hashmap(
1434             self.path,
1435             version=self['object_version'],
1436             if_match=self['if_match'],
1437             if_none_match=self['if_none_match'],
1438             if_modified_since=self['if_modified_since'],
1439             if_unmodified_since=self['if_unmodified_since'])
1440         print_dict(data)
1441
1442     def main(self, container___path):
1443         super(self.__class__, self)._run(
1444             container___path,
1445             path_is_optional=False)
1446         self._run()
1447
1448
1449 @command(pithos_cmds)
1450 class file_delete(_file_container_command):
1451     """Delete a container [or an object]
1452     How to delete a non-empty container:
1453     - empty the container:  /file delete -R <container>
1454     - delete it:            /file delete <container>
1455     .
1456     Semantics of directory deletion:
1457     .a preserve the contents: /file delete <container>:<directory>
1458     .    objects of the form dir/filename can exist with a dir object
1459     .b delete contents:       /file delete -R <container>:<directory>
1460     .    all dir/* objects are affected, even if dir does not exist
1461     .
1462     To restore a deleted object OBJ in a container CONT:
1463     - get object versions: /file versions CONT:OBJ
1464     .   and choose the version to be restored
1465     - restore the object:  /file copy --source-version=<version> CONT:OBJ OBJ
1466     """
1467
1468     arguments = dict(
1469         until=DateArgument('remove history until that date', '--until'),
1470         yes=FlagArgument('Do not prompt for permission', '--yes'),
1471         recursive=FlagArgument(
1472             'empty dir or container and delete (if dir)',
1473             ('-R', '--recursive'))
1474     )
1475
1476     def __init__(self, arguments={}):
1477         super(self.__class__, self).__init__(arguments)
1478         self['delimiter'] = DelimiterArgument(
1479             self,
1480             parsed_name='--delimiter',
1481             help='delete objects prefixed with <object><delimiter>')
1482
1483     @errors.generic.all
1484     @errors.pithos.connection
1485     @errors.pithos.container
1486     @errors.pithos.object_path
1487     def _run(self):
1488         if self.path:
1489             if self['yes'] or ask_user(
1490                     'Delete %s:%s ?' % (self.container, self.path)):
1491                 self.client.del_object(
1492                     self.path,
1493                     until=self['until'],
1494                     delimiter=self['delimiter'])
1495             else:
1496                 print('Aborted')
1497         else:
1498             if self['recursive']:
1499                 ask_msg = 'Delete container contents'
1500             else:
1501                 ask_msg = 'Delete container'
1502             if self['yes'] or ask_user('%s %s ?' % (ask_msg, self.container)):
1503                 self.client.del_container(
1504                     until=self['until'],
1505                     delimiter=self['delimiter'])
1506             else:
1507                 print('Aborted')
1508
1509     def main(self, container____path__=None):
1510         super(self.__class__, self)._run(container____path__)
1511         self._run()
1512
1513
1514 @command(pithos_cmds)
1515 class file_purge(_file_container_command):
1516     """Delete a container and release related data blocks
1517     Non-empty containers can not purged.
1518     To purge a container with content:
1519     .   /file delete -R <container>
1520     .      objects are deleted, but data blocks remain on server
1521     .   /file purge <container>
1522     .      container and data blocks are released and deleted
1523     """
1524
1525     arguments = dict(
1526         yes=FlagArgument('Do not prompt for permission', '--yes'),
1527     )
1528
1529     @errors.generic.all
1530     @errors.pithos.connection
1531     @errors.pithos.container
1532     def _run(self):
1533         if self['yes'] or ask_user('Purge container %s?' % self.container):
1534                 self.client.purge_container()
1535         else:
1536             print('Aborted')
1537
1538     def main(self, container=None):
1539         super(self.__class__, self)._run(container)
1540         if container and self.container != container:
1541             raiseCLIError('Invalid container name %s' % container, details=[
1542                 'Did you mean "%s" ?' % self.container,
1543                 'Use --container for names containing :'])
1544         self._run()
1545
1546
1547 @command(pithos_cmds)
1548 class file_publish(_file_container_command):
1549     """Publish the object and print the public url"""
1550
1551     @errors.generic.all
1552     @errors.pithos.connection
1553     @errors.pithos.container
1554     @errors.pithos.object_path
1555     def _run(self):
1556         url = self.client.publish_object(self.path)
1557         print(url)
1558
1559     def main(self, container___path):
1560         super(self.__class__, self)._run(
1561             container___path,
1562             path_is_optional=False)
1563         self._run()
1564
1565
1566 @command(pithos_cmds)
1567 class file_unpublish(_file_container_command):
1568     """Unpublish an object"""
1569
1570     @errors.generic.all
1571     @errors.pithos.connection
1572     @errors.pithos.container
1573     @errors.pithos.object_path
1574     def _run(self):
1575             self.client.unpublish_object(self.path)
1576
1577     def main(self, container___path):
1578         super(self.__class__, self)._run(
1579             container___path,
1580             path_is_optional=False)
1581         self._run()
1582
1583
1584 @command(pithos_cmds)
1585 class file_permissions(_file_container_command):
1586     """Get read and write permissions of an object
1587     Permissions are lists of users and user groups. There is read and write
1588     permissions. Users and groups with write permission have also read
1589     permission.
1590     """
1591
1592     @errors.generic.all
1593     @errors.pithos.connection
1594     @errors.pithos.container
1595     @errors.pithos.object_path
1596     def _run(self):
1597         r = self.client.get_object_sharing(self.path)
1598         print_dict(r)
1599
1600     def main(self, container___path):
1601         super(self.__class__, self)._run(
1602             container___path,
1603             path_is_optional=False)
1604         self._run()
1605
1606
1607 @command(pithos_cmds)
1608 class file_setpermissions(_file_container_command):
1609     """Set permissions for an object
1610     New permissions overwrite existing permissions.
1611     Permission format:
1612     -   read=<username>[,usergroup[,...]]
1613     -   write=<username>[,usegroup[,...]]
1614     E.g. to give read permissions for file F to users A and B and write for C:
1615     .       /file setpermissions F read=A,B write=C
1616     """
1617
1618     @errors.generic.all
1619     def format_permition_dict(self, permissions):
1620         read = False
1621         write = False
1622         for perms in permissions:
1623             splstr = perms.split('=')
1624             if 'read' == splstr[0]:
1625                 read = [ug.strip() for ug in splstr[1].split(',')]
1626             elif 'write' == splstr[0]:
1627                 write = [ug.strip() for ug in splstr[1].split(',')]
1628             else:
1629                 msg = 'Usage:\tread=<groups,users> write=<groups,users>'
1630                 raiseCLIError(None, msg)
1631         return (read, write)
1632
1633     @errors.generic.all
1634     @errors.pithos.connection
1635     @errors.pithos.container
1636     @errors.pithos.object_path
1637     def _run(self, read, write):
1638         self.client.set_object_sharing(
1639             self.path,
1640             read_permition=read,
1641             write_permition=write)
1642
1643     def main(self, container___path, *permissions):
1644         super(self.__class__, self)._run(
1645             container___path,
1646             path_is_optional=False)
1647         (read, write) = self.format_permition_dict(permissions)
1648         self._run(read, write)
1649
1650
1651 @command(pithos_cmds)
1652 class file_delpermissions(_file_container_command):
1653     """Delete all permissions set on object
1654     To modify permissions, use /file setpermssions
1655     """
1656
1657     @errors.generic.all
1658     @errors.pithos.connection
1659     @errors.pithos.container
1660     @errors.pithos.object_path
1661     def _run(self):
1662         self.client.del_object_sharing(self.path)
1663
1664     def main(self, container___path):
1665         super(self.__class__, self)._run(
1666             container___path,
1667             path_is_optional=False)
1668         self._run()
1669
1670
1671 @command(pithos_cmds)
1672 class file_info(_file_container_command):
1673     """Get detailed information for user account, containers or objects
1674     to get account info:    /file info
1675     to get container info:  /file info <container>
1676     to get object info:     /file info <container>:<path>
1677     """
1678
1679     arguments = dict(
1680         object_version=ValueArgument(
1681             'show specific version \ (applies only for objects)',
1682             ('-j', '--object-version'))
1683     )
1684
1685     @errors.generic.all
1686     @errors.pithos.connection
1687     @errors.pithos.container
1688     @errors.pithos.object_path
1689     def _run(self):
1690         if self.container is None:
1691             r = self.client.get_account_info()
1692         elif self.path is None:
1693             r = self.client.get_container_info(self.container)
1694         else:
1695             r = self.client.get_object_info(
1696                 self.path,
1697                 version=self['object_version'])
1698         print_dict(r)
1699
1700     def main(self, container____path__=None):
1701         super(self.__class__, self)._run(container____path__)
1702         self._run()
1703
1704
1705 @command(pithos_cmds)
1706 class file_meta(_file_container_command):
1707     """Get metadata for account, containers or objects"""
1708
1709     arguments = dict(
1710         detail=FlagArgument('show detailed output', ('-l', '--details')),
1711         until=DateArgument('show metadata until then', '--until'),
1712         object_version=ValueArgument(
1713             'show specific version \ (applies only for objects)',
1714             ('-j', '--object-version'))
1715     )
1716
1717     @errors.generic.all
1718     @errors.pithos.connection
1719     @errors.pithos.container
1720     @errors.pithos.object_path
1721     def _run(self):
1722         until = self['until']
1723         if self.container is None:
1724             if self['detail']:
1725                 r = self.client.get_account_info(until=until)
1726             else:
1727                 r = self.client.get_account_meta(until=until)
1728                 r = pretty_keys(r, '-')
1729             if r:
1730                 print(bold(self.client.account))
1731         elif self.path is None:
1732             if self['detail']:
1733                 r = self.client.get_container_info(until=until)
1734             else:
1735                 cmeta = self.client.get_container_meta(until=until)
1736                 ometa = self.client.get_container_object_meta(until=until)
1737                 r = {}
1738                 if cmeta:
1739                     r['container-meta'] = pretty_keys(cmeta, '-')
1740                 if ometa:
1741                     r['object-meta'] = pretty_keys(ometa, '-')
1742         else:
1743             if self['detail']:
1744                 r = self.client.get_object_info(
1745                     self.path,
1746                     version=self['object_version'])
1747             else:
1748                 r = self.client.get_object_meta(
1749                     self.path,
1750                     version=self['object_version'])
1751             if r:
1752                 r = pretty_keys(pretty_keys(r, '-'))
1753         if r:
1754             print_dict(r)
1755
1756     def main(self, container____path__=None):
1757         super(self.__class__, self)._run(container____path__)
1758         self._run()
1759
1760
1761 @command(pithos_cmds)
1762 class file_setmeta(_file_container_command):
1763     """Set a piece of metadata for account, container or object
1764     Metadata are formed as key:value pairs
1765     """
1766
1767     @errors.generic.all
1768     @errors.pithos.connection
1769     @errors.pithos.container
1770     @errors.pithos.object_path
1771     def _run(self, metakey, metaval):
1772         if not self.container:
1773             self.client.set_account_meta({metakey: metaval})
1774         elif not self.path:
1775             self.client.set_container_meta({metakey: metaval})
1776         else:
1777             self.client.set_object_meta(self.path, {metakey: metaval})
1778
1779     def main(self, metakey, metaval, container____path__=None):
1780         super(self.__class__, self)._run(container____path__)
1781         self._run(metakey=metakey, metaval=metaval)
1782
1783
1784 @command(pithos_cmds)
1785 class file_delmeta(_file_container_command):
1786     """Delete metadata with given key from account, container or object
1787     Metadata are formed as key:value objects
1788     - to get metadata of current account:     /file meta
1789     - to get metadata of a container:         /file meta <container>
1790     - to get metadata of an object:           /file meta <container>:<path>
1791     """
1792
1793     @errors.generic.all
1794     @errors.pithos.connection
1795     @errors.pithos.container
1796     @errors.pithos.object_path
1797     def _run(self, metakey):
1798         if self.container is None:
1799             self.client.del_account_meta(metakey)
1800         elif self.path is None:
1801             self.client.del_container_meta(metakey)
1802         else:
1803             self.client.del_object_meta(self.path, metakey)
1804
1805     def main(self, metakey, container____path__=None):
1806         super(self.__class__, self)._run(container____path__)
1807         self._run(metakey)
1808
1809
1810 @command(pithos_cmds)
1811 class file_quota(_file_account_command):
1812     """Get account quota"""
1813
1814     arguments = dict(
1815         in_bytes=FlagArgument('Show result in bytes', ('-b', '--bytes'))
1816     )
1817
1818     @errors.generic.all
1819     @errors.pithos.connection
1820     def _run(self):
1821         reply = self.client.get_account_quota()
1822         if not self['in_bytes']:
1823             for k in reply:
1824                 reply[k] = format_size(reply[k])
1825         print_dict(pretty_keys(reply, '-'))
1826
1827     def main(self, custom_uuid=None):
1828         super(self.__class__, self)._run(custom_account=custom_uuid)
1829         self._run()
1830
1831
1832 @command(pithos_cmds)
1833 class file_containerlimit(_pithos_init):
1834     """Container size limit commands"""
1835
1836
1837 @command(pithos_cmds)
1838 class file_containerlimit_get(_file_container_command):
1839     """Get container size limit"""
1840
1841     arguments = dict(
1842         in_bytes=FlagArgument('Show result in bytes', ('-b', '--bytes'))
1843     )
1844
1845     @errors.generic.all
1846     @errors.pithos.container
1847     def _run(self):
1848         reply = self.client.get_container_limit(self.container)
1849         if not self['in_bytes']:
1850             for k, v in reply.items():
1851                 reply[k] = 'unlimited' if '0' == v else format_size(v)
1852         print_dict(pretty_keys(reply, '-'))
1853
1854     def main(self, container=None):
1855         super(self.__class__, self)._run()
1856         self.container = container
1857         self._run()
1858
1859
1860 @command(pithos_cmds)
1861 class file_containerlimit_set(_file_account_command):
1862     """Set new storage limit for a container
1863     By default, the limit is set in bytes
1864     Users may specify a different unit, e.g:
1865     /file containerlimit set 2.3GB mycontainer
1866     Valid units: B, KiB (1024 B), KB (1000 B), MiB, MB, GiB, GB, TiB, TB
1867     To set container limit to "unlimited", use 0
1868     """
1869
1870     @errors.generic.all
1871     def _calculate_limit(self, user_input):
1872         limit = 0
1873         try:
1874             limit = int(user_input)
1875         except ValueError:
1876             index = 0
1877             digits = [str(num) for num in range(0, 10)] + ['.']
1878             while user_input[index] in digits:
1879                 index += 1
1880             limit = user_input[:index]
1881             format = user_input[index:]
1882             try:
1883                 return to_bytes(limit, format)
1884             except Exception as qe:
1885                 msg = 'Failed to convert %s to bytes' % user_input,
1886                 raiseCLIError(qe, msg, details=[
1887                     'Syntax: containerlimit set <limit>[format] [container]',
1888                     'e.g.: containerlimit set 2.3GB mycontainer',
1889                     'Valid formats:',
1890                     '(*1024): B, KiB, MiB, GiB, TiB',
1891                     '(*1000): B, KB, MB, GB, TB'])
1892         return limit
1893
1894     @errors.generic.all
1895     @errors.pithos.connection
1896     @errors.pithos.container
1897     def _run(self, limit):
1898         if self.container:
1899             self.client.container = self.container
1900         self.client.set_container_limit(limit)
1901
1902     def main(self, limit, container=None):
1903         super(self.__class__, self)._run()
1904         limit = self._calculate_limit(limit)
1905         self.container = container
1906         self._run(limit)
1907
1908
1909 @command(pithos_cmds)
1910 class file_versioning(_file_account_command):
1911     """Get  versioning for account or container"""
1912
1913     @errors.generic.all
1914     @errors.pithos.connection
1915     @errors.pithos.container
1916     def _run(self):
1917         if self.container:
1918             r = self.client.get_container_versioning(self.container)
1919         else:
1920             r = self.client.get_account_versioning()
1921         print_dict(r)
1922
1923     def main(self, container=None):
1924         super(self.__class__, self)._run()
1925         self.container = container
1926         self._run()
1927
1928
1929 @command(pithos_cmds)
1930 class file_setversioning(_file_account_command):
1931     """Set versioning mode (auto, none) for account or container"""
1932
1933     def _check_versioning(self, versioning):
1934         if versioning and versioning.lower() in ('auto', 'none'):
1935             return versioning.lower()
1936         raiseCLIError('Invalid versioning %s' % versioning, details=[
1937             'Versioning can be auto or none'])
1938
1939     @errors.generic.all
1940     @errors.pithos.connection
1941     @errors.pithos.container
1942     def _run(self, versioning):
1943         if self.container:
1944             self.client.container = self.container
1945             self.client.set_container_versioning(versioning)
1946         else:
1947             self.client.set_account_versioning(versioning)
1948
1949     def main(self, versioning, container=None):
1950         super(self.__class__, self)._run()
1951         self._run(self._check_versioning(versioning))
1952
1953
1954 @command(pithos_cmds)
1955 class file_group(_file_account_command):
1956     """Get groups and group members"""
1957
1958     @errors.generic.all
1959     @errors.pithos.connection
1960     def _run(self):
1961         r = self.client.get_account_group()
1962         print_dict(pretty_keys(r, '-'))
1963
1964     def main(self):
1965         super(self.__class__, self)._run()
1966         self._run()
1967
1968
1969 @command(pithos_cmds)
1970 class file_setgroup(_file_account_command):
1971     """Set a user group"""
1972
1973     @errors.generic.all
1974     @errors.pithos.connection
1975     def _run(self, groupname, *users):
1976         self.client.set_account_group(groupname, users)
1977
1978     def main(self, groupname, *users):
1979         super(self.__class__, self)._run()
1980         if users:
1981             self._run(groupname, *users)
1982         else:
1983             raiseCLIError('No users to add in group %s' % groupname)
1984
1985
1986 @command(pithos_cmds)
1987 class file_delgroup(_file_account_command):
1988     """Delete a user group"""
1989
1990     @errors.generic.all
1991     @errors.pithos.connection
1992     def _run(self, groupname):
1993         self.client.del_account_group(groupname)
1994
1995     def main(self, groupname):
1996         super(self.__class__, self)._run()
1997         self._run(groupname)
1998
1999
2000 @command(pithos_cmds)
2001 class file_sharers(_file_account_command):
2002     """List the accounts that share objects with current user"""
2003
2004     arguments = dict(
2005         detail=FlagArgument('show detailed output', ('-l', '--details')),
2006         marker=ValueArgument('show output greater then marker', '--marker')
2007     )
2008
2009     @errors.generic.all
2010     @errors.pithos.connection
2011     def _run(self):
2012         accounts = self.client.get_sharing_accounts(marker=self['marker'])
2013         if self['detail']:
2014             print_items(accounts)
2015         else:
2016             print_items([acc['name'] for acc in accounts])
2017
2018     def main(self):
2019         super(self.__class__, self)._run()
2020         self._run()
2021
2022
2023 @command(pithos_cmds)
2024 class file_versions(_file_container_command):
2025     """Get the list of object versions
2026     Deleted objects may still have versions that can be used to restore it and
2027     get information about its previous state.
2028     The version number can be used in a number of other commands, like info,
2029     copy, move, meta. See these commands for more information, e.g.
2030     /file info -h
2031     """
2032
2033     @errors.generic.all
2034     @errors.pithos.connection
2035     @errors.pithos.container
2036     @errors.pithos.object_path
2037     def _run(self):
2038         versions = self.client.get_object_versionlist(self.path)
2039         print_items([dict(id=vitem[0], created=strftime(
2040             '%d-%m-%Y %H:%M:%S',
2041             localtime(float(vitem[1])))) for vitem in versions])
2042
2043     def main(self, container___path):
2044         super(file_versions, self)._run(
2045             container___path,
2046             path_is_optional=False)
2047         self._run()