code.grnet.gr Git - kamaki/blob - kamaki/cli/commands/pithos_cli.py

   1 # Copyright 2011-2012 GRNET S.A. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or
   4 # without modification, are permitted provided that the following
   5 # conditions are met:
   6 #
   7 #   1. Redistributions of source code must retain the above
   8 #      copyright notice, this list of conditions and the following
   9 #      disclaimer.
  10 #
  11 #   2. Redistributions in binary form must reproduce the above
  12 #      copyright notice, this list of conditions and the following
  13 #      disclaimer in the documentation and/or other materials
  14 #      provided with the distribution.
  15 #
  16 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  17 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  23 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  24 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  26 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27 # POSSIBILITY OF SUCH DAMAGE.
  28 #
  29 # The views and conclusions contained in the software and
  30 # documentation are those of the authors and should not be
  31 # interpreted as representing official policies, either expressed
  32 # or implied, of GRNET S.A.command
  33
  34 from sys import stdout
  35 from time import localtime, strftime
  36 from logging import getLogger
  37 from os import path, makedirs
  38
  39 from kamaki.cli import command
  40 from kamaki.cli.command_tree import CommandTree
  41 from kamaki.cli.errors import raiseCLIError, CLISyntaxError
  42 from kamaki.cli.utils import (
  43     format_size,
  44     to_bytes,
  45     print_dict,
  46     print_items,
  47     pretty_keys,
  48     page_hold,
  49     bold,
  50     ask_user)
  51 from kamaki.cli.argument import FlagArgument, ValueArgument, IntArgument
  52 from kamaki.cli.argument import KeyValueArgument, DateArgument
  53 from kamaki.cli.argument import ProgressBarArgument
  54 from kamaki.cli.commands import _command_init, errors
  55 from kamaki.clients.pithos import PithosClient, ClientError
  56 from kamaki.clients.astakos import AstakosClient
  57
  58
  59 kloger = getLogger('kamaki')
  60
  61 pithos_cmds = CommandTree('store', 'Pithos+ storage commands')
  62 _commands = [pithos_cmds]
  63
  64
  65 # Argument functionality
  66
  67 class DelimiterArgument(ValueArgument):
  68     """
  69     :value type: string
  70     :value returns: given string or /
  71     """
  72
  73     def __init__(self, caller_obj, help='', parsed_name=None, default=None):
  74         super(DelimiterArgument, self).__init__(help, parsed_name, default)
  75         self.caller_obj = caller_obj
  76
  77     @property
  78     def value(self):
  79         if self.caller_obj['recursive']:
  80             return '/'
  81         return getattr(self, '_value', self.default)
  82
  83     @value.setter
  84     def value(self, newvalue):
  85         self._value = newvalue
  86
  87
  88 class SharingArgument(ValueArgument):
  89     """Set sharing (read and/or write) groups
  90     .
  91     :value type: "read=term1,term2,... write=term1,term2,..."
  92     .
  93     :value returns: {'read':['term1', 'term2', ...],
  94     .   'write':['term1', 'term2', ...]}
  95     """
  96
  97     @property
  98     def value(self):
  99         return getattr(self, '_value', self.default)
 100
 101     @value.setter
 102     def value(self, newvalue):
 103         perms = {}
 104         try:
 105             permlist = newvalue.split(' ')
 106         except AttributeError:
 107             return
 108         for p in permlist:
 109             try:
 110                 (key, val) = p.split('=')
 111             except ValueError as err:
 112                 raiseCLIError(
 113                     err,
 114                     'Error in --sharing',
 115                     details='Incorrect format',
 116                     importance=1)
 117             if key.lower() not in ('read', 'write'):
 118                 msg = 'Error in --sharing'
 119                 raiseCLIError(err, msg, importance=1, details=[
 120                     'Invalid permission key %s' % key])
 121             val_list = val.split(',')
 122             if not key in perms:
 123                 perms[key] = []
 124             for item in val_list:
 125                 if item not in perms[key]:
 126                     perms[key].append(item)
 127         self._value = perms
 128
 129
 130 class RangeArgument(ValueArgument):
 131     """
 132     :value type: string of the form <start>-<end> where <start> and <end> are
 133         integers
 134     :value returns: the input string, after type checking <start> and <end>
 135     """
 136
 137     @property
 138     def value(self):
 139         return getattr(self, '_value', self.default)
 140
 141     @value.setter
 142     def value(self, newvalue):
 143         if newvalue is None:
 144             self._value = self.default
 145             return
 146         (start, end) = newvalue.split('-')
 147         (start, end) = (int(start), int(end))
 148         self._value = '%s-%s' % (start, end)
 149
 150 # Command specs
 151
 152
 153 class _pithos_init(_command_init):
 154     """Initialize a pithos+ kamaki client"""
 155
 156     @staticmethod
 157     def _is_dir(remote_dict):
 158         return 'application/directory' == remote_dict.get(
 159             'content_type',
 160             remote_dict.get('content-type', ''))
 161
 162     @errors.generic.all
 163     def _run(self):
 164         self.token = self.config.get('store', 'token')\
 165             or self.config.get('global', 'token')
 166         self.base_url = self.config.get('store', 'url')\
 167             or self.config.get('global', 'url')
 168         self._set_account()
 169         self.container = self.config.get('store', 'container')\
 170             or self.config.get('global', 'container')
 171         self.client = PithosClient(
 172             base_url=self.base_url,
 173             token=self.token,
 174             account=self.account,
 175             container=self.container)
 176         self._update_low_level_log()
 177         self._update_max_threads()
 178
 179     def main(self):
 180         self._run()
 181
 182     def _set_account(self):
 183         astakos = AstakosClient(self.config.get('astakos', 'url'), self.token)
 184         self.account = self['account'] or astakos.term('uuid')
 185
 186         """Backwards compatibility"""
 187         self.account = self.account\
 188             or self.config.get('store', 'account')\
 189             or self.config.get('global', 'account')
 190
 191
 192 class _store_account_command(_pithos_init):
 193     """Base class for account level storage commands"""
 194
 195     def __init__(self, arguments={}):
 196         super(_store_account_command, self).__init__(arguments)
 197         self['account'] = ValueArgument(
 198             'Set user account (not permanent)',
 199             ('-A', '--account'))
 200
 201     def _run(self):
 202         super(_store_account_command, self)._run()
 203         if self['account']:
 204             self.client.account = self['account']
 205
 206     @errors.generic.all
 207     def main(self):
 208         self._run()
 209
 210
 211 class _store_container_command(_store_account_command):
 212     """Base class for container level storage commands"""
 213
 214     container = None
 215     path = None
 216
 217     def __init__(self, arguments={}):
 218         super(_store_container_command, self).__init__(arguments)
 219         self['container'] = ValueArgument(
 220             'Set container to work with (temporary)',
 221             ('-C', '--container'))
 222
 223     def extract_container_and_path(
 224             self,
 225             container_with_path,
 226             path_is_optional=True):
 227         """Contains all heuristics for deciding what should be used as
 228         container or path. Options are:
 229         * user string of the form container:path
 230         * self.container, self.path variables set by super constructor, or
 231         explicitly by the caller application
 232         Error handling is explicit as these error cases happen only here
 233         """
 234         try:
 235             assert isinstance(container_with_path, str)
 236         except AssertionError as err:
 237             if self['container'] and path_is_optional:
 238                 self.container = self['container']
 239                 self.client.container = self['container']
 240                 return
 241             raiseCLIError(err)
 242
 243         user_cont, sep, userpath = container_with_path.partition(':')
 244
 245         if sep:
 246             if not user_cont:
 247                 raiseCLIError(CLISyntaxError(
 248                     'Container is missing\n',
 249                     details=errors.pithos.container_howto))
 250             alt_cont = self['container']
 251             if alt_cont and user_cont != alt_cont:
 252                 raiseCLIError(CLISyntaxError(
 253                     'Conflict: 2 containers (%s, %s)' % (user_cont, alt_cont),
 254                     details=errors.pithos.container_howto)
 255                 )
 256             self.container = user_cont
 257             if not userpath:
 258                 raiseCLIError(CLISyntaxError(
 259                     'Path is missing for object in container %s' % user_cont,
 260                     details=errors.pithos.container_howto)
 261                 )
 262             self.path = userpath
 263         else:
 264             alt_cont = self['container'] or self.client.container
 265             if alt_cont:
 266                 self.container = alt_cont
 267                 self.path = user_cont
 268             elif path_is_optional:
 269                 self.container = user_cont
 270                 self.path = None
 271             else:
 272                 self.container = user_cont
 273                 raiseCLIError(CLISyntaxError(
 274                     'Both container and path are required',
 275                     details=errors.pithos.container_howto)
 276                 )
 277
 278     @errors.generic.all
 279     def _run(self, container_with_path=None, path_is_optional=True):
 280         super(_store_container_command, self)._run()
 281         if self['container']:
 282             self.client.container = self['container']
 283             if container_with_path:
 284                 self.path = container_with_path
 285             elif not path_is_optional:
 286                 raise CLISyntaxError(
 287                     'Both container and path are required',
 288                     details=errors.pithos.container_howto)
 289         elif container_with_path:
 290             self.extract_container_and_path(
 291                 container_with_path,
 292                 path_is_optional)
 293             self.client.container = self.container
 294         self.container = self.client.container
 295
 296     def main(self, container_with_path=None, path_is_optional=True):
 297         self._run(container_with_path, path_is_optional)
 298
 299
 300 @command(pithos_cmds)
 301 class store_list(_store_container_command):
 302     """List containers, object trees or objects in a directory
 303     Use with:
 304     1 no parameters : containers in current account
 305     2. one parameter (container) or --container : contents of container
 306     3. <container>:<prefix> or --container=<container> <prefix>: objects in
 307     .   container starting with prefix
 308     """
 309
 310     arguments = dict(
 311         detail=FlagArgument('detailed output', ('-l', '--list')),
 312         limit=IntArgument('limit number of listed items', ('-n', '--number')),
 313         marker=ValueArgument('output greater that marker', '--marker'),
 314         prefix=ValueArgument('output starting with prefix', '--prefix'),
 315         delimiter=ValueArgument('show output up to delimiter', '--delimiter'),
 316         path=ValueArgument(
 317             'show output starting with prefix up to /',
 318             '--path'),
 319         meta=ValueArgument(
 320             'show output with specified meta keys',
 321             '--meta',
 322             default=[]),
 323         if_modified_since=ValueArgument(
 324             'show output modified since then',
 325             '--if-modified-since'),
 326         if_unmodified_since=ValueArgument(
 327             'show output not modified since then',
 328             '--if-unmodified-since'),
 329         until=DateArgument('show metadata until then', '--until'),
 330         format=ValueArgument(
 331             'format to parse until data (default: d/m/Y H:M:S )',
 332             '--format'),
 333         shared=FlagArgument('show only shared', '--shared'),
 334         more=FlagArgument(
 335             'output results in pages (-n to set items per page, default 10)',
 336             '--more'),
 337         exact_match=FlagArgument(
 338             'Show only objects that match exactly with path',
 339             '--exact-match')
 340     )
 341
 342     def print_objects(self, object_list):
 343         limit = int(self['limit']) if self['limit'] > 0 else len(object_list)
 344         for index, obj in enumerate(object_list):
 345             if self['exact_match'] and self.path and not (
 346                     obj['name'] == self.path or 'content_type' in obj):
 347                 continue
 348             pretty_obj = obj.copy()
 349             index += 1
 350             empty_space = ' ' * (len(str(len(object_list))) - len(str(index)))
 351             if obj['content_type'] == 'application/directory':
 352                 isDir = True
 353                 size = 'D'
 354             else:
 355                 isDir = False
 356                 size = format_size(obj['bytes'])
 357                 pretty_obj['bytes'] = '%s (%s)' % (obj['bytes'], size)
 358             oname = bold(obj['name'])
 359             if self['detail']:
 360                 print('%s%s. %s' % (empty_space, index, oname))
 361                 print_dict(pretty_keys(pretty_obj), exclude=('name'))
 362                 print
 363             else:
 364                 oname = '%s%s. %6s %s' % (empty_space, index, size, oname)
 365                 oname += '/' if isDir else ''
 366                 print(oname)
 367             if self['more']:
 368                 page_hold(index, limit, len(object_list))
 369
 370     def print_containers(self, container_list):
 371         limit = int(self['limit']) if self['limit'] > 0\
 372             else len(container_list)
 373         for index, container in enumerate(container_list):
 374             if 'bytes' in container:
 375                 size = format_size(container['bytes'])
 376             cname = '%s. %s' % (index + 1, bold(container['name']))
 377             if self['detail']:
 378                 print(cname)
 379                 pretty_c = container.copy()
 380                 if 'bytes' in container:
 381                     pretty_c['bytes'] = '%s (%s)' % (container['bytes'], size)
 382                 print_dict(pretty_keys(pretty_c), exclude=('name'))
 383                 print
 384             else:
 385                 if 'count' in container and 'bytes' in container:
 386                     print('%s (%s, %s objects)' % (
 387                         cname,
 388                         size,
 389                         container['count']))
 390                 else:
 391                     print(cname)
 392             if self['more']:
 393                 page_hold(index + 1, limit, len(container_list))
 394
 395     @errors.generic.all
 396     @errors.pithos.connection
 397     @errors.pithos.object_path
 398     @errors.pithos.container
 399     def _run(self):
 400         if self.container is None:
 401             r = self.client.account_get(
 402                 limit=False if self['more'] else self['limit'],
 403                 marker=self['marker'],
 404                 if_modified_since=self['if_modified_since'],
 405                 if_unmodified_since=self['if_unmodified_since'],
 406                 until=self['until'],
 407                 show_only_shared=self['shared'])
 408             self.print_containers(r.json)
 409         else:
 410             prefix = self.path or self['prefix']
 411             r = self.client.container_get(
 412                 limit=False if self['more'] else self['limit'],
 413                 marker=self['marker'],
 414                 prefix=prefix,
 415                 delimiter=self['delimiter'],
 416                 path=self['path'],
 417                 if_modified_since=self['if_modified_since'],
 418                 if_unmodified_since=self['if_unmodified_since'],
 419                 until=self['until'],
 420                 meta=self['meta'],
 421                 show_only_shared=self['shared'])
 422             self.print_objects(r.json)
 423
 424     def main(self, container____path__=None):
 425         super(self.__class__, self)._run(container____path__)
 426         self._run()
 427
 428
 429 @command(pithos_cmds)
 430 class store_mkdir(_store_container_command):
 431     """Create a directory"""
 432
 433     __doc__ += '\n. '.join([
 434         'Kamaki hanldes directories the same way as OOS Storage and Pithos+:',
 435         'A   directory  is   an  object  with  type  "application/directory"',
 436         'An object with path  dir/name can exist even if  dir does not exist',
 437         'or even if dir  is  a non  directory  object.  Users can modify dir',
 438         'without affecting the dir/name object in any way.'])
 439
 440     @errors.generic.all
 441     @errors.pithos.connection
 442     @errors.pithos.container
 443     def _run(self):
 444         self.client.create_directory(self.path)
 445
 446     def main(self, container___directory):
 447         super(self.__class__, self)._run(
 448             container___directory,
 449             path_is_optional=False)
 450         self._run()
 451
 452
 453 @command(pithos_cmds)
 454 class store_touch(_store_container_command):
 455     """Create an empty object (file)
 456     If object exists, this command will reset it to 0 length
 457     """
 458
 459     arguments = dict(
 460         content_type=ValueArgument(
 461             'Set content type (default: application/octet-stream)',
 462             '--content-type',
 463             default='application/octet-stream')
 464     )
 465
 466     @errors.generic.all
 467     @errors.pithos.connection
 468     @errors.pithos.container
 469     def _run(self):
 470         self.client.create_object(self.path, self['content_type'])
 471
 472     def main(self, container___path):
 473         super(store_touch, self)._run(
 474             container___path,
 475             path_is_optional=False)
 476         self._run()
 477
 478
 479 @command(pithos_cmds)
 480 class store_create(_store_container_command):
 481     """Create a container"""
 482
 483     arguments = dict(
 484         versioning=ValueArgument(
 485             'set container versioning (auto/none)',
 486             '--versioning'),
 487         quota=IntArgument('set default container quota', '--quota'),
 488         meta=KeyValueArgument(
 489             'set container metadata (can be repeated)',
 490             '--meta')
 491     )
 492
 493     @errors.generic.all
 494     @errors.pithos.connection
 495     @errors.pithos.container
 496     def _run(self):
 497         self.client.container_put(
 498             quota=self['quota'],
 499             versioning=self['versioning'],
 500             metadata=self['meta'])
 501
 502     def main(self, container=None):
 503         super(self.__class__, self)._run(container)
 504         if container and self.container != container:
 505             raiseCLIError('Invalid container name %s' % container, details=[
 506                 'Did you mean "%s" ?' % self.container,
 507                 'Use --container for names containing :'])
 508         self._run()
 509
 510
 511 class _source_destination_command(_store_container_command):
 512
 513     arguments = dict(
 514         destination_account=ValueArgument('', ('a', '--dst-account')),
 515         recursive=FlagArgument('', ('-R', '--recursive')),
 516         prefix=FlagArgument('', '--with-prefix', default=''),
 517         suffix=ValueArgument('', '--with-suffix', default=''),
 518         add_prefix=ValueArgument('', '--add-prefix', default=''),
 519         add_suffix=ValueArgument('', '--add-suffix', default=''),
 520         prefix_replace=ValueArgument('', '--prefix-to-replace', default=''),
 521         suffix_replace=ValueArgument('', '--suffix-to-replace', default='')
 522     )
 523
 524     def __init__(self, arguments={}):
 525         self.arguments.update(arguments)
 526         super(_source_destination_command, self).__init__(self.arguments)
 527
 528     def _run(self, source_container___path, path_is_optional=False):
 529         super(_source_destination_command, self)._run(
 530             source_container___path,
 531             path_is_optional)
 532         self.dst_client = PithosClient(
 533             base_url=self.client.base_url,
 534             token=self.client.token,
 535             account=self['destination_account'] or self.client.account)
 536
 537     @errors.generic.all
 538     @errors.pithos.account
 539     def _dest_container_path(self, dest_container_path):
 540         if self['destination_container']:
 541             self.dst_client.container = self['destination_container']
 542             return (self['destination_container'], dest_container_path)
 543         if dest_container_path:
 544             dst = dest_container_path.split(':')
 545             if len(dst) > 1:
 546                 try:
 547                     self.dst_client.container = dst[0]
 548                     self.dst_client.get_container_info(dst[0])
 549                 except ClientError as err:
 550                     if err.status in (404, 204):
 551                         raiseCLIError(
 552                             'Destination container %s not found' % dst[0])
 553                     raise
 554                 else:
 555                     self.dst_client.container = dst[0]
 556                 return (dst[0], dst[1])
 557             return(None, dst[0])
 558         raiseCLIError('No destination container:path provided')
 559
 560     def _get_all(self, prefix):
 561         return self.client.container_get(prefix=prefix).json
 562
 563     def _get_src_objects(self, src_path):
 564         """Get a list of the source objects to be called
 565
 566         :param src_path: (str) source path
 567
 568         :returns: (method, params) a method that returns a list when called
 569         or (object) if it is a single object
 570         """
 571         if src_path and src_path[-1] == '/':
 572             src_path = src_path[:-1]
 573
 574         if self['prefix']:
 575             return (self._get_all, dict(prefix=src_path))
 576         try:
 577             srcobj = self.client.get_object_info(src_path)
 578         except ClientError as srcerr:
 579             if srcerr.status == 404:
 580                 raiseCLIError(
 581                     'Source object %s not in source container %s' % (
 582                         src_path,
 583                         self.client.container),
 584                     details=['Hint: --with-prefix to match multiple objects'])
 585             elif srcerr.status not in (204,):
 586                 raise
 587             return (self.client.list_objects, {})
 588
 589         if self._is_dir(srcobj):
 590             if not self['recursive']:
 591                 raiseCLIError(
 592                     'Object %s of cont. %s is a dir' % (
 593                         src_path,
 594                         self.client.container),
 595                     details=['Use --recursive to access directories'])
 596             return (self._get_all, dict(prefix=src_path))
 597         srcobj['name'] = src_path
 598         return srcobj
 599
 600     def src_dst_pairs(self, dst_path):
 601         src_iter = self._get_src_objects(self.path)
 602         src_N = isinstance(src_iter, tuple)
 603         add_prefix = self['add_prefix'].strip('/')
 604
 605         if dst_path and dst_path.endswith('/'):
 606             dst_path = dst_path[:-1]
 607
 608         try:
 609             dstobj = self.dst_client.get_object_info(dst_path)
 610         except ClientError as trgerr:
 611             if trgerr.status in (404,):
 612                 if src_N:
 613                     raiseCLIError(
 614                         'Cannot merge multiple paths to path %s' % dst_path,
 615                         details=[
 616                             'Try to use / or a directory as destination',
 617                             'or create the destination dir (/store mkdir)',
 618                             'or use a single object as source'])
 619             elif trgerr.status not in (204,):
 620                 raise
 621         else:
 622             if self._is_dir(dstobj):
 623                 add_prefix = '%s/%s' % (dst_path.strip('/'), add_prefix)
 624             elif src_N:
 625                 raiseCLIError(
 626                     'Cannot merge multiple paths to path' % dst_path,
 627                     details=[
 628                         'Try to use / or a directory as destination',
 629                         'or create the destination dir (/store mkdir)',
 630                         'or use a single object as source'])
 631
 632         if src_N:
 633             (method, kwargs) = src_iter
 634             for obj in method(**kwargs):
 635                 name = obj['name']
 636                 if name.endswith(self['suffix']):
 637                     yield (name, self._get_new_object(name, add_prefix))
 638         elif src_iter['name'].endswith(self['suffix']):
 639             name = src_iter['name']
 640             yield (name, self._get_new_object(dst_path or name, add_prefix))
 641         else:
 642             raiseCLIError('Source path %s conflicts with suffix %s' % (
 643                 src_iter['name'],
 644                 self['suffix']))
 645
 646     def _get_new_object(self, obj, add_prefix):
 647         if self['prefix_replace'] and obj.startswith(self['prefix_replace']):
 648             obj = obj[len(self['prefix_replace']):]
 649         if self['suffix_replace'] and obj.endswith(self['suffix_replace']):
 650             obj = obj[:-len(self['suffix_replace'])]
 651         return add_prefix + obj + self['add_suffix']
 652
 653
 654 @command(pithos_cmds)
 655 class store_copy(_source_destination_command):
 656     """Copy objects from container to (another) container
 657     Semantics:
 658     copy cont:path dir
 659     .   transfer path as dir/path
 660     copy cont:path cont2:
 661     .   trasnfer all <obj> prefixed with path to container cont2
 662     copy cont:path [cont2:]path2
 663     .   transfer path to path2
 664     Use options:
 665     1. <container1>:<path1> [container2:]<path2> : if container2 is not given,
 666     destination is container1:path2
 667     2. <container>:<path1> <path2> : make a copy in the same container
 668     3. Can use --container= instead of <container1>
 669     """
 670
 671     arguments = dict(
 672         destination_account=ValueArgument(
 673             'Account to copy to',
 674             ('-a', '--dst-account')),
 675         destination_container=ValueArgument(
 676             'use it if destination container name contains a : character',
 677             ('-D', '--dst-container')),
 678         source_version=ValueArgument(
 679             'copy specific version',
 680             ('-S', '--source-version')),
 681         public=ValueArgument('make object publicly accessible', '--public'),
 682         content_type=ValueArgument(
 683             'change object\'s content type',
 684             '--content-type'),
 685         recursive=FlagArgument(
 686             'copy directory and contents',
 687             ('-R', '--recursive')),
 688         prefix=FlagArgument(
 689             'Match objects prefixed with src path (feels like src_path*)',
 690             '--with-prefix',
 691             default=''),
 692         suffix=ValueArgument(
 693             'Suffix of source objects (feels like *suffix)',
 694             '--with-suffix',
 695             default=''),
 696         add_prefix=ValueArgument('Prefix targets', '--add-prefix', default=''),
 697         add_suffix=ValueArgument('Suffix targets', '--add-suffix', default=''),
 698         prefix_replace=ValueArgument(
 699             'Prefix of src to replace with dst path + add_prefix, if matched',
 700             '--prefix-to-replace',
 701             default=''),
 702         suffix_replace=ValueArgument(
 703             'Suffix of src to replace with add_suffix, if matched',
 704             '--suffix-to-replace',
 705             default='')
 706     )
 707
 708     @errors.generic.all
 709     @errors.pithos.connection
 710     @errors.pithos.container
 711     @errors.pithos.account
 712     def _run(self, dst_path):
 713         no_source_object = True
 714         src_account = self.client.account if (
 715             self['destination_account']) else None
 716         for src_obj, dst_obj in self.src_dst_pairs(dst_path):
 717             no_source_object = False
 718             self.dst_client.copy_object(
 719                 src_container=self.client.container,
 720                 src_object=src_obj,
 721                 dst_container=self.dst_client.container,
 722                 dst_object=dst_obj,
 723                 source_account=src_account,
 724                 source_version=self['source_version'],
 725                 public=self['public'],
 726                 content_type=self['content_type'])
 727         if no_source_object:
 728             raiseCLIError('No object %s in container %s' % (
 729                 self.path,
 730                 self.container))
 731
 732     def main(
 733             self,
 734             source_container___path,
 735             destination_container___path=None):
 736         super(store_copy, self)._run(
 737             source_container___path,
 738             path_is_optional=False)
 739         (dst_cont, dst_path) = self._dest_container_path(
 740             destination_container___path)
 741         self.dst_client.container = dst_cont or self.container
 742         self._run(dst_path=dst_path or '')
 743
 744
 745 @command(pithos_cmds)
 746 class store_move(_source_destination_command):
 747     """Move/rename objects from container to (another) container
 748     Semantics:
 749     move cont:path dir
 750     .   rename path as dir/path
 751     move cont:path cont2:
 752     .   trasnfer all <obj> prefixed with path to container cont2
 753     move cont:path [cont2:]path2
 754     .   transfer path to path2
 755     Use options:
 756     1. <container1>:<path1> [container2:]<path2> : if container2 is not given,
 757     destination is container1:path2
 758     2. <container>:<path1> <path2> : move in the same container
 759     3. Can use --container= instead of <container1>
 760     """
 761
 762     arguments = dict(
 763         destination_account=ValueArgument(
 764             'Account to move to',
 765             ('-a', '--dst-account')),
 766         destination_container=ValueArgument(
 767             'use it if destination container name contains a : character',
 768             ('-D', '--dst-container')),
 769         source_version=ValueArgument(
 770             'copy specific version',
 771             '--source-version'),
 772         public=ValueArgument('make object publicly accessible', '--public'),
 773         content_type=ValueArgument(
 774             'change object\'s content type',
 775             '--content-type'),
 776         recursive=FlagArgument(
 777             'copy directory and contents',
 778             ('-R', '--recursive')),
 779         prefix=FlagArgument(
 780             'Match objects prefixed with src path (feels like src_path*)',
 781             '--with-prefix',
 782             default=''),
 783         suffix=ValueArgument(
 784             'Suffix of source objects (feels like *suffix)',
 785             '--with-suffix',
 786             default=''),
 787         add_prefix=ValueArgument('Prefix targets', '--add-prefix', default=''),
 788         add_suffix=ValueArgument('Suffix targets', '--add-suffix', default=''),
 789         prefix_replace=ValueArgument(
 790             'Prefix of src to replace with dst path + add_prefix, if matched',
 791             '--prefix-to-replace',
 792             default=''),
 793         suffix_replace=ValueArgument(
 794             'Suffix of src to replace with add_suffix, if matched',
 795             '--suffix-to-replace',
 796             default='')
 797     )
 798
 799     @errors.generic.all
 800     @errors.pithos.connection
 801     @errors.pithos.container
 802     def _run(self, dst_path):
 803         no_source_object = True
 804         src_account = self.client.account if (
 805             self['destination_account']) else None
 806         for src_obj, dst_obj in self.src_dst_pairs(dst_path):
 807             no_source_object = False
 808             self.dst_client.move_object(
 809                 src_container=self.container,
 810                 src_object=src_obj,
 811                 dst_container=self.dst_client.container,
 812                 dst_object=dst_obj,
 813                 source_account=src_account,
 814                 source_version=self['source_version'],
 815                 public=self['public'],
 816                 content_type=self['content_type'])
 817         if no_source_object:
 818             raiseCLIError('No object %s in container %s' % (
 819                 self.path,
 820                 self.container))
 821
 822     def main(
 823             self,
 824             source_container___path,
 825             destination_container___path=None):
 826         super(self.__class__, self)._run(
 827             source_container___path,
 828             path_is_optional=False)
 829         (dst_cont, dst_path) = self._dest_container_path(
 830             destination_container___path)
 831         (dst_cont, dst_path) = self._dest_container_path(
 832             destination_container___path)
 833         self.dst_client.container = dst_cont or self.container
 834         self._run(dst_path=dst_path or '')
 835
 836
 837 @command(pithos_cmds)
 838 class store_append(_store_container_command):
 839     """Append local file to (existing) remote object
 840     The remote object should exist.
 841     If the remote object is a directory, it is transformed into a file.
 842     In the later case, objects under the directory remain intact.
 843     """
 844
 845     arguments = dict(
 846         progress_bar=ProgressBarArgument(
 847             'do not show progress bar',
 848             ('-N', '--no-progress-bar'),
 849             default=False)
 850     )
 851
 852     @errors.generic.all
 853     @errors.pithos.connection
 854     @errors.pithos.container
 855     @errors.pithos.object_path
 856     def _run(self, local_path):
 857         (progress_bar, upload_cb) = self._safe_progress_bar('Appending')
 858         try:
 859             f = open(local_path, 'rb')
 860             self.client.append_object(self.path, f, upload_cb)
 861         except Exception:
 862             self._safe_progress_bar_finish(progress_bar)
 863             raise
 864         finally:
 865             self._safe_progress_bar_finish(progress_bar)
 866
 867     def main(self, local_path, container___path):
 868         super(self.__class__, self)._run(
 869             container___path,
 870             path_is_optional=False)
 871         self._run(local_path)
 872
 873
 874 @command(pithos_cmds)
 875 class store_truncate(_store_container_command):
 876     """Truncate remote file up to a size (default is 0)"""
 877
 878     @errors.generic.all
 879     @errors.pithos.connection
 880     @errors.pithos.container
 881     @errors.pithos.object_path
 882     @errors.pithos.object_size
 883     def _run(self, size=0):
 884         self.client.truncate_object(self.path, size)
 885
 886     def main(self, container___path, size=0):
 887         super(self.__class__, self)._run(container___path)
 888         self._run(size=size)
 889
 890
 891 @command(pithos_cmds)
 892 class store_overwrite(_store_container_command):
 893     """Overwrite part (from start to end) of a remote file
 894     overwrite local-path container 10 20
 895     .   will overwrite bytes from 10 to 20 of a remote file with the same name
 896     .   as local-path basename
 897     overwrite local-path container:path 10 20
 898     .   will overwrite as above, but the remote file is named path
 899     """
 900
 901     arguments = dict(
 902         progress_bar=ProgressBarArgument(
 903             'do not show progress bar',
 904             ('-N', '--no-progress-bar'),
 905             default=False)
 906     )
 907
 908     def _open_file(self, local_path, start):
 909         f = open(path.abspath(local_path), 'rb')
 910         f.seek(0, 2)
 911         f_size = f.tell()
 912         f.seek(start, 0)
 913         return (f, f_size)
 914
 915     @errors.generic.all
 916     @errors.pithos.connection
 917     @errors.pithos.container
 918     @errors.pithos.object_path
 919     @errors.pithos.object_size
 920     def _run(self, local_path, start, end):
 921         (start, end) = (int(start), int(end))
 922         (f, f_size) = self._open_file(local_path, start)
 923         (progress_bar, upload_cb) = self._safe_progress_bar(
 924             'Overwrite %s bytes' % (end - start))
 925         try:
 926             self.client.overwrite_object(
 927                 obj=self.path,
 928                 start=start,
 929                 end=end,
 930                 source_file=f,
 931                 upload_cb=upload_cb)
 932         except Exception:
 933             self._safe_progress_bar_finish(progress_bar)
 934             raise
 935         finally:
 936             self._safe_progress_bar_finish(progress_bar)
 937
 938     def main(self, local_path, container___path, start, end):
 939         super(self.__class__, self)._run(
 940             container___path,
 941             path_is_optional=None)
 942         self.path = self.path or path.basename(local_path)
 943         self._run(local_path=local_path, start=start, end=end)
 944
 945
 946 @command(pithos_cmds)
 947 class store_manifest(_store_container_command):
 948     """Create a remote file of uploaded parts by manifestation
 949     Remains functional for compatibility with OOS Storage. Users are advised
 950     to use the upload command instead.
 951     Manifestation is a compliant process for uploading large files. The files
 952     have to be chunked in smalled files and uploaded as <prefix><increment>
 953     where increment is 1, 2, ...
 954     Finally, the manifest command glues partial files together in one file
 955     named <prefix>
 956     The upload command is faster, easier and more intuitive than manifest
 957     """
 958
 959     arguments = dict(
 960         etag=ValueArgument('check written data', '--etag'),
 961         content_encoding=ValueArgument(
 962             'set MIME content type',
 963             '--content-encoding'),
 964         content_disposition=ValueArgument(
 965             'the presentation style of the object',
 966             '--content-disposition'),
 967         content_type=ValueArgument(
 968             'specify content type',
 969             '--content-type',
 970             default='application/octet-stream'),
 971         sharing=SharingArgument(
 972             '\n'.join([
 973                 'define object sharing policy',
 974                 '    ( "read=user1,grp1,user2,... write=user1,grp2,..." )']),
 975             '--sharing'),
 976         public=FlagArgument('make object publicly accessible', '--public')
 977     )
 978
 979     @errors.generic.all
 980     @errors.pithos.connection
 981     @errors.pithos.container
 982     @errors.pithos.object_path
 983     def _run(self):
 984         self.client.create_object_by_manifestation(
 985             self.path,
 986             content_encoding=self['content_encoding'],
 987             content_disposition=self['content_disposition'],
 988             content_type=self['content_type'],
 989             sharing=self['sharing'],
 990             public=self['public'])
 991
 992     def main(self, container___path):
 993         super(self.__class__, self)._run(
 994             container___path,
 995             path_is_optional=False)
 996         self.run()
 997
 998
 999 @command(pithos_cmds)
1000 class store_upload(_store_container_command):
1001     """Upload a file"""
1002
1003     arguments = dict(
1004         use_hashes=FlagArgument(
1005             'provide hashmap file instead of data',
1006             '--use-hashes'),
1007         etag=ValueArgument('check written data', '--etag'),
1008         unchunked=FlagArgument('avoid chunked transfer mode', '--unchunked'),
1009         content_encoding=ValueArgument(
1010             'set MIME content type',
1011             '--content-encoding'),
1012         content_disposition=ValueArgument(
1013             'specify objects presentation style',
1014             '--content-disposition'),
1015         content_type=ValueArgument('specify content type', '--content-type'),
1016         sharing=SharingArgument(
1017             help='\n'.join([
1018                 'define sharing object policy',
1019                 '( "read=user1,grp1,user2,... write=user1,grp2,... )']),
1020             parsed_name='--sharing'),
1021         public=FlagArgument('make object publicly accessible', '--public'),
1022         poolsize=IntArgument('set pool size', '--with-pool-size'),
1023         progress_bar=ProgressBarArgument(
1024             'do not show progress bar',
1025             ('-N', '--no-progress-bar'),
1026             default=False),
1027         overwrite=FlagArgument('Force (over)write', ('-f', '--force'))
1028     )
1029
1030     def _remote_path(self, remote_path, local_path=''):
1031         if self['overwrite']:
1032             return remote_path
1033         try:
1034             r = self.client.get_object_info(remote_path)
1035         except ClientError as ce:
1036             if ce.status == 404:
1037                 return remote_path
1038             raise ce
1039         ctype = r.get('content-type', '')
1040         if 'application/directory' == ctype.lower():
1041             ret = '%s/%s' % (remote_path, local_path)
1042             return self._remote_path(ret) if local_path else ret
1043         raiseCLIError(
1044             'Object %s already exists' % remote_path,
1045             importance=1,
1046             details=['use -f to overwrite or resume'])
1047
1048     @errors.generic.all
1049     @errors.pithos.connection
1050     @errors.pithos.container
1051     @errors.pithos.object_path
1052     @errors.pithos.local_path
1053     def _run(self, local_path, remote_path):
1054         poolsize = self['poolsize']
1055         if poolsize > 0:
1056             self.client.MAX_THREADS = int(poolsize)
1057         params = dict(
1058             content_encoding=self['content_encoding'],
1059             content_type=self['content_type'],
1060             content_disposition=self['content_disposition'],
1061             sharing=self['sharing'],
1062             public=self['public'])
1063         remote_path = self._remote_path(remote_path, local_path)
1064         with open(path.abspath(local_path), 'rb') as f:
1065             if self['unchunked']:
1066                 self.client.upload_object_unchunked(
1067                     remote_path,
1068                     f,
1069                     etag=self['etag'],
1070                     withHashFile=self['use_hashes'],
1071                     **params)
1072             else:
1073                 try:
1074                     (progress_bar, upload_cb) = self._safe_progress_bar(
1075                         'Uploading')
1076                     if progress_bar:
1077                         hash_bar = progress_bar.clone()
1078                         hash_cb = hash_bar.get_generator(
1079                             'Calculating block hashes')
1080                     else:
1081                         hash_cb = None
1082                     self.client.upload_object(
1083                         remote_path,
1084                         f,
1085                         hash_cb=hash_cb,
1086                         upload_cb=upload_cb,
1087                         **params)
1088                 except Exception:
1089                     self._safe_progress_bar_finish(progress_bar)
1090                     raise
1091                 finally:
1092                     self._safe_progress_bar_finish(progress_bar)
1093         print 'Upload completed'
1094
1095     def main(self, local_path, container____path__=None):
1096         super(self.__class__, self)._run(container____path__)
1097         remote_path = self.path or path.basename(local_path)
1098         self._run(local_path=local_path, remote_path=remote_path)
1099
1100
1101 @command(pithos_cmds)
1102 class store_cat(_store_container_command):
1103     """Print remote file contents to console"""
1104
1105     arguments = dict(
1106         range=RangeArgument('show range of data', '--range'),
1107         if_match=ValueArgument('show output if ETags match', '--if-match'),
1108         if_none_match=ValueArgument(
1109             'show output if ETags match',
1110             '--if-none-match'),
1111         if_modified_since=DateArgument(
1112             'show output modified since then',
1113             '--if-modified-since'),
1114         if_unmodified_since=DateArgument(
1115             'show output unmodified since then',
1116             '--if-unmodified-since'),
1117         object_version=ValueArgument(
1118             'get the specific version',
1119             ('-j', '--object-version'))
1120     )
1121
1122     @errors.generic.all
1123     @errors.pithos.connection
1124     @errors.pithos.container
1125     @errors.pithos.object_path
1126     def _run(self):
1127         self.client.download_object(
1128             self.path,
1129             stdout,
1130             range_str=self['range'],
1131             version=self['object_version'],
1132             if_match=self['if_match'],
1133             if_none_match=self['if_none_match'],
1134             if_modified_since=self['if_modified_since'],
1135             if_unmodified_since=self['if_unmodified_since'])
1136
1137     def main(self, container___path):
1138         super(self.__class__, self)._run(
1139             container___path,
1140             path_is_optional=False)
1141         self._run()
1142
1143
1144 @command(pithos_cmds)
1145 class store_download(_store_container_command):
1146     """Download remote object as local file
1147     If local destination is a directory:
1148     *   download <container>:<path> <local dir> -R
1149     will download all files on <container> prefixed as <path>,
1150     to <local dir>/<full path>
1151     *   download <container>:<path> <local dir> --exact-match
1152     will download only one file, exactly matching <path>
1153     ATTENTION: to download cont:dir1/dir2/file there must exist objects
1154     cont:dir1 and cont:dir1/dir2 of type application/directory
1155     To create directory objects, use /store mkdir
1156     """
1157
1158     arguments = dict(
1159         resume=FlagArgument('Resume instead of overwrite', ('-r', '--resume')),
1160         range=RangeArgument('show range of data', '--range'),
1161         if_match=ValueArgument('show output if ETags match', '--if-match'),
1162         if_none_match=ValueArgument(
1163             'show output if ETags match',
1164             '--if-none-match'),
1165         if_modified_since=DateArgument(
1166             'show output modified since then',
1167             '--if-modified-since'),
1168         if_unmodified_since=DateArgument(
1169             'show output unmodified since then',
1170             '--if-unmodified-since'),
1171         object_version=ValueArgument(
1172             'get the specific version',
1173             ('-j', '--object-version')),
1174         poolsize=IntArgument('set pool size', '--with-pool-size'),
1175         progress_bar=ProgressBarArgument(
1176             'do not show progress bar',
1177             ('-N', '--no-progress-bar'),
1178             default=False),
1179         recursive=FlagArgument(
1180             'Download a remote path and all its contents',
1181             ('-R', '--recursive'))
1182     )
1183
1184     def _outputs(self, local_path):
1185         """:returns: (local_file, remote_path)"""
1186         remotes = []
1187         if self['recursive']:
1188             r = self.client.container_get(
1189                 prefix=self.path or '/',
1190                 if_modified_since=self['if_modified_since'],
1191                 if_unmodified_since=self['if_unmodified_since'])
1192             dirlist = dict()
1193             for remote in r.json:
1194                 rname = remote['name'].strip('/')
1195                 tmppath = ''
1196                 for newdir in rname.strip('/').split('/')[:-1]:
1197                     tmppath = '/'.join([tmppath, newdir])
1198                     dirlist.update({tmppath.strip('/'): True})
1199                 remotes.append((rname, store_download._is_dir(remote)))
1200             dir_remotes = [r[0] for r in remotes if r[1]]
1201             if not set(dirlist).issubset(dir_remotes):
1202                 badguys = [bg.strip('/') for bg in set(
1203                     dirlist).difference(dir_remotes)]
1204                 raiseCLIError(
1205                     'Some remote paths contain non existing directories',
1206                     details=['Missing remote directories:'] + badguys)
1207         elif self.path:
1208             r = self.client.get_object_info(
1209                 self.path,
1210                 version=self['object_version'])
1211             if store_download._is_dir(r):
1212                 raiseCLIError(
1213                     'Illegal download: Remote object %s is a directory' % (
1214                         self.path),
1215                     details=['To download a directory, try --recursive'])
1216             if '/' in self.path.strip('/') and not local_path:
1217                 raiseCLIError(
1218                     'Illegal download: remote object %s contains "/"' % (
1219                         self.path),
1220                     details=[
1221                         'To download an object containing "/" characters',
1222                         'either create the remote directories or',
1223                         'specify a non-directory local path for this object'])
1224             remotes = [(self.path, False)]
1225         if not remotes:
1226             if self.path:
1227                 raiseCLIError(
1228                     'No matching path %s on container %s' % (
1229                         self.path,
1230                         self.container),
1231                     details=[
1232                         'To list the contents of %s, try:' % self.container,
1233                         '   /store list %s' % self.container])
1234             raiseCLIError(
1235                 'Illegal download of container %s' % self.container,
1236                 details=[
1237                     'To download a whole container, try:',
1238                     '   /store download --recursive <container>'])
1239
1240         lprefix = path.abspath(local_path or path.curdir)
1241         if path.isdir(lprefix):
1242             for rpath, remote_is_dir in remotes:
1243                 lpath = '/%s/%s' % (lprefix.strip('/'), rpath.strip('/'))
1244                 if remote_is_dir:
1245                     if path.exists(lpath) and path.isdir(lpath):
1246                         continue
1247                     makedirs(lpath)
1248                 elif path.exists(lpath):
1249                     if not self['resume']:
1250                         print('File %s exists, aborting...' % lpath)
1251                         continue
1252                     with open(lpath, 'rwb+') as f:
1253                         yield (f, rpath)
1254                 else:
1255                     with open(lpath, 'wb+') as f:
1256                         yield (f, rpath)
1257         elif path.exists(lprefix):
1258             if len(remotes) > 1:
1259                 raiseCLIError(
1260                     '%s remote objects cannot be merged in local file %s' % (
1261                         len(remotes),
1262                         local_path),
1263                     details=[
1264                         'To download multiple objects, local path should be',
1265                         'a directory, or use download without a local path'])
1266             (rpath, remote_is_dir) = remotes[0]
1267             if remote_is_dir:
1268                 raiseCLIError(
1269                     'Remote directory %s should not replace local file %s' % (
1270                         rpath,
1271                         local_path))
1272             if self['resume']:
1273                 with open(lprefix, 'rwb+') as f:
1274                     yield (f, rpath)
1275             else:
1276                 raiseCLIError(
1277                     'Local file %s already exist' % local_path,
1278                     details=['Try --resume to overwrite it'])
1279         else:
1280             if len(remotes) > 1 or remotes[0][1]:
1281                 raiseCLIError(
1282                     'Local directory %s does not exist' % local_path)
1283             with open(lprefix, 'wb+') as f:
1284                 yield (f, remotes[0][0])
1285
1286     @errors.generic.all
1287     @errors.pithos.connection
1288     @errors.pithos.container
1289     @errors.pithos.object_path
1290     @errors.pithos.local_path
1291     def _run(self, local_path):
1292         #outputs = self._outputs(local_path)
1293         poolsize = self['poolsize']
1294         if poolsize:
1295             self.client.MAX_THREADS = int(poolsize)
1296         progress_bar = None
1297         try:
1298             for f, rpath in self._outputs(local_path):
1299                 (
1300                     progress_bar,
1301                     download_cb) = self._safe_progress_bar(
1302                         'Download %s' % rpath)
1303                 self.client.download_object(
1304                     rpath,
1305                     f,
1306                     download_cb=download_cb,
1307                     range_str=self['range'],
1308                     version=self['object_version'],
1309                     if_match=self['if_match'],
1310                     resume=self['resume'],
1311                     if_none_match=self['if_none_match'],
1312                     if_modified_since=self['if_modified_since'],
1313                     if_unmodified_since=self['if_unmodified_since'])
1314         except KeyboardInterrupt:
1315             from threading import activeCount, enumerate as activethreads
1316             while activeCount() > 1:
1317                 stdout.write('\nTry stop %s threads: ' % (activeCount() - 1))
1318                 for thread in activethreads():
1319                     stdout.flush()
1320                     try:
1321                         thread.join(0.5)
1322                         stdout.write('*')
1323                     except RuntimeError:
1324                         continue
1325             print('\nDownload canceled by user')
1326             if local_path is not None:
1327                 print('to resume, re-run with --resume')
1328         except Exception:
1329             self._safe_progress_bar_finish(progress_bar)
1330             raise
1331         finally:
1332             self._safe_progress_bar_finish(progress_bar)
1333
1334     def main(self, container___path, local_path=None):
1335         super(self.__class__, self)._run(container___path)
1336         self._run(local_path=local_path)
1337
1338
1339 @command(pithos_cmds)
1340 class store_hashmap(_store_container_command):
1341     """Get the hash-map of an object"""
1342
1343     arguments = dict(
1344         if_match=ValueArgument('show output if ETags match', '--if-match'),
1345         if_none_match=ValueArgument(
1346             'show output if ETags match',
1347             '--if-none-match'),
1348         if_modified_since=DateArgument(
1349             'show output modified since then',
1350             '--if-modified-since'),
1351         if_unmodified_since=DateArgument(
1352             'show output unmodified since then',
1353             '--if-unmodified-since'),
1354         object_version=ValueArgument(
1355             'get the specific version',
1356             ('-j', '--object-version'))
1357     )
1358
1359     @errors.generic.all
1360     @errors.pithos.connection
1361     @errors.pithos.container
1362     @errors.pithos.object_path
1363     def _run(self):
1364         data = self.client.get_object_hashmap(
1365             self.path,
1366             version=self['object_version'],
1367             if_match=self['if_match'],
1368             if_none_match=self['if_none_match'],
1369             if_modified_since=self['if_modified_since'],
1370             if_unmodified_since=self['if_unmodified_since'])
1371         print_dict(data)
1372
1373     def main(self, container___path):
1374         super(self.__class__, self)._run(
1375             container___path,
1376             path_is_optional=False)
1377         self._run()
1378
1379
1380 @command(pithos_cmds)
1381 class store_delete(_store_container_command):
1382     """Delete a container [or an object]
1383     How to delete a non-empty container:
1384     - empty the container:  /store delete -R <container>
1385     - delete it:            /store delete <container>
1386     .
1387     Semantics of directory deletion:
1388     .a preserve the contents: /store delete <container>:<directory>
1389     .    objects of the form dir/filename can exist with a dir object
1390     .b delete contents:       /store delete -R <container>:<directory>
1391     .    all dir/* objects are affected, even if dir does not exist
1392     .
1393     To restore a deleted object OBJ in a container CONT:
1394     - get object versions: /store versions CONT:OBJ
1395     .   and choose the version to be restored
1396     - restore the object:  /store copy --source-version=<version> CONT:OBJ OBJ
1397     """
1398
1399     arguments = dict(
1400         until=DateArgument('remove history until that date', '--until'),
1401         yes=FlagArgument('Do not prompt for permission', '--yes'),
1402         recursive=FlagArgument(
1403             'empty dir or container and delete (if dir)',
1404             ('-R', '--recursive'))
1405     )
1406
1407     def __init__(self, arguments={}):
1408         super(self.__class__, self).__init__(arguments)
1409         self['delimiter'] = DelimiterArgument(
1410             self,
1411             parsed_name='--delimiter',
1412             help='delete objects prefixed with <object><delimiter>')
1413
1414     @errors.generic.all
1415     @errors.pithos.connection
1416     @errors.pithos.container
1417     @errors.pithos.object_path
1418     def _run(self):
1419         if self.path:
1420             if self['yes'] or ask_user(
1421                     'Delete %s:%s ?' % (self.container, self.path)):
1422                 self.client.del_object(
1423                     self.path,
1424                     until=self['until'],
1425                     delimiter=self['delimiter'])
1426             else:
1427                 print('Aborted')
1428         else:
1429             if self['recursive']:
1430                 ask_msg = 'Delete container contents'
1431             else:
1432                 ask_msg = 'Delete container'
1433             if self['yes'] or ask_user('%s %s ?' % (ask_msg, self.container)):
1434                 self.client.del_container(
1435                     until=self['until'],
1436                     delimiter=self['delimiter'])
1437             else:
1438                 print('Aborted')
1439
1440     def main(self, container____path__=None):
1441         super(self.__class__, self)._run(container____path__)
1442         self._run()
1443
1444
1445 @command(pithos_cmds)
1446 class store_purge(_store_container_command):
1447     """Delete a container and release related data blocks
1448     Non-empty containers can not purged.
1449     To purge a container with content:
1450     .   /store delete -R <container>
1451     .      objects are deleted, but data blocks remain on server
1452     .   /store purge <container>
1453     .      container and data blocks are released and deleted
1454     """
1455
1456     arguments = dict(
1457         yes=FlagArgument('Do not prompt for permission', '--yes'),
1458     )
1459
1460     @errors.generic.all
1461     @errors.pithos.connection
1462     @errors.pithos.container
1463     def _run(self):
1464         if self['yes'] or ask_user('Purge container %s?' % self.container):
1465                 self.client.purge_container()
1466         else:
1467             print('Aborted')
1468
1469     def main(self, container=None):
1470         super(self.__class__, self)._run(container)
1471         if container and self.container != container:
1472             raiseCLIError('Invalid container name %s' % container, details=[
1473                 'Did you mean "%s" ?' % self.container,
1474                 'Use --container for names containing :'])
1475         self._run()
1476
1477
1478 @command(pithos_cmds)
1479 class store_publish(_store_container_command):
1480     """Publish the object and print the public url"""
1481
1482     @errors.generic.all
1483     @errors.pithos.connection
1484     @errors.pithos.container
1485     @errors.pithos.object_path
1486     def _run(self):
1487         url = self.client.publish_object(self.path)
1488         print(url)
1489
1490     def main(self, container___path):
1491         super(self.__class__, self)._run(
1492             container___path,
1493             path_is_optional=False)
1494         self._run()
1495
1496
1497 @command(pithos_cmds)
1498 class store_unpublish(_store_container_command):
1499     """Unpublish an object"""
1500
1501     @errors.generic.all
1502     @errors.pithos.connection
1503     @errors.pithos.container
1504     @errors.pithos.object_path
1505     def _run(self):
1506             self.client.unpublish_object(self.path)
1507
1508     def main(self, container___path):
1509         super(self.__class__, self)._run(
1510             container___path,
1511             path_is_optional=False)
1512         self._run()
1513
1514
1515 @command(pithos_cmds)
1516 class store_permissions(_store_container_command):
1517     """Get read and write permissions of an object
1518     Permissions are lists of users and user groups. There is read and write
1519     permissions. Users and groups with write permission have also read
1520     permission.
1521     """
1522
1523     @errors.generic.all
1524     @errors.pithos.connection
1525     @errors.pithos.container
1526     @errors.pithos.object_path
1527     def _run(self):
1528         r = self.client.get_object_sharing(self.path)
1529         print_dict(r)
1530
1531     def main(self, container___path):
1532         super(self.__class__, self)._run(
1533             container___path,
1534             path_is_optional=False)
1535         self._run()
1536
1537
1538 @command(pithos_cmds)
1539 class store_setpermissions(_store_container_command):
1540     """Set permissions for an object
1541     New permissions overwrite existing permissions.
1542     Permission format:
1543     -   read=<username>[,usergroup[,...]]
1544     -   write=<username>[,usegroup[,...]]
1545     E.g. to give read permissions for file F to users A and B and write for C:
1546     .       /store setpermissions F read=A,B write=C
1547     """
1548
1549     @errors.generic.all
1550     def format_permition_dict(self, permissions):
1551         read = False
1552         write = False
1553         for perms in permissions:
1554             splstr = perms.split('=')
1555             if 'read' == splstr[0]:
1556                 read = [ug.strip() for ug in splstr[1].split(',')]
1557             elif 'write' == splstr[0]:
1558                 write = [ug.strip() for ug in splstr[1].split(',')]
1559             else:
1560                 msg = 'Usage:\tread=<groups,users> write=<groups,users>'
1561                 raiseCLIError(None, msg)
1562         return (read, write)
1563
1564     @errors.generic.all
1565     @errors.pithos.connection
1566     @errors.pithos.container
1567     @errors.pithos.object_path
1568     def _run(self, read, write):
1569         self.client.set_object_sharing(
1570             self.path,
1571             read_permition=read,
1572             write_permition=write)
1573
1574     def main(self, container___path, *permissions):
1575         super(self.__class__, self)._run(
1576             container___path,
1577             path_is_optional=False)
1578         (read, write) = self.format_permition_dict(permissions)
1579         self._run(read, write)
1580
1581
1582 @command(pithos_cmds)
1583 class store_delpermissions(_store_container_command):
1584     """Delete all permissions set on object
1585     To modify permissions, use /store setpermssions
1586     """
1587
1588     @errors.generic.all
1589     @errors.pithos.connection
1590     @errors.pithos.container
1591     @errors.pithos.object_path
1592     def _run(self):
1593         self.client.del_object_sharing(self.path)
1594
1595     def main(self, container___path):
1596         super(self.__class__, self)._run(
1597             container___path,
1598             path_is_optional=False)
1599         self._run()
1600
1601
1602 @command(pithos_cmds)
1603 class store_info(_store_container_command):
1604     """Get detailed information for user account, containers or objects
1605     to get account info:    /store info
1606     to get container info:  /store info <container>
1607     to get object info:     /store info <container>:<path>
1608     """
1609
1610     arguments = dict(
1611         object_version=ValueArgument(
1612             'show specific version \ (applies only for objects)',
1613             ('-j', '--object-version'))
1614     )
1615
1616     @errors.generic.all
1617     @errors.pithos.connection
1618     @errors.pithos.container
1619     @errors.pithos.object_path
1620     def _run(self):
1621         if self.container is None:
1622             r = self.client.get_account_info()
1623         elif self.path is None:
1624             r = self.client.get_container_info(self.container)
1625         else:
1626             r = self.client.get_object_info(
1627                 self.path,
1628                 version=self['object_version'])
1629         print_dict(r)
1630
1631     def main(self, container____path__=None):
1632         super(self.__class__, self)._run(container____path__)
1633         self._run()
1634
1635
1636 @command(pithos_cmds)
1637 class store_meta(_store_container_command):
1638     """Get metadata for account, containers or objects"""
1639
1640     arguments = dict(
1641         detail=FlagArgument('show detailed output', ('-l', '--details')),
1642         until=DateArgument('show metadata until then', '--until'),
1643         object_version=ValueArgument(
1644             'show specific version \ (applies only for objects)',
1645             ('-j', '--object-version'))
1646     )
1647
1648     @errors.generic.all
1649     @errors.pithos.connection
1650     @errors.pithos.container
1651     @errors.pithos.object_path
1652     def _run(self):
1653         until = self['until']
1654         if self.container is None:
1655             if self['detail']:
1656                 r = self.client.get_account_info(until=until)
1657             else:
1658                 r = self.client.get_account_meta(until=until)
1659                 r = pretty_keys(r, '-')
1660             if r:
1661                 print(bold(self.client.account))
1662         elif self.path is None:
1663             if self['detail']:
1664                 r = self.client.get_container_info(until=until)
1665             else:
1666                 cmeta = self.client.get_container_meta(until=until)
1667                 ometa = self.client.get_container_object_meta(until=until)
1668                 r = {}
1669                 if cmeta:
1670                     r['container-meta'] = pretty_keys(cmeta, '-')
1671                 if ometa:
1672                     r['object-meta'] = pretty_keys(ometa, '-')
1673         else:
1674             if self['detail']:
1675                 r = self.client.get_object_info(
1676                     self.path,
1677                     version=self['object_version'])
1678             else:
1679                 r = self.client.get_object_meta(
1680                     self.path,
1681                     version=self['object_version'])
1682             if r:
1683                 r = pretty_keys(pretty_keys(r, '-'))
1684         if r:
1685             print_dict(r)
1686
1687     def main(self, container____path__=None):
1688         super(self.__class__, self)._run(container____path__)
1689         self._run()
1690
1691
1692 @command(pithos_cmds)
1693 class store_setmeta(_store_container_command):
1694     """Set a piece of metadata for account, container or object
1695     Metadata are formed as key:value pairs
1696     """
1697
1698     @errors.generic.all
1699     @errors.pithos.connection
1700     @errors.pithos.container
1701     @errors.pithos.object_path
1702     def _run(self, metakey, metaval):
1703         if not self.container:
1704             self.client.set_account_meta({metakey: metaval})
1705         elif not self.path:
1706             self.client.set_container_meta({metakey: metaval})
1707         else:
1708             self.client.set_object_meta(self.path, {metakey: metaval})
1709
1710     def main(self, metakey, metaval, container____path__=None):
1711         super(self.__class__, self)._run(container____path__)
1712         self._run(metakey=metakey, metaval=metaval)
1713
1714
1715 @command(pithos_cmds)
1716 class store_delmeta(_store_container_command):
1717     """Delete metadata with given key from account, container or object
1718     Metadata are formed as key:value objects
1719     - to get metadata of current account:     /store meta
1720     - to get metadata of a container:         /store meta <container>
1721     - to get metadata of an object:           /store meta <container>:<path>
1722     """
1723
1724     @errors.generic.all
1725     @errors.pithos.connection
1726     @errors.pithos.container
1727     @errors.pithos.object_path
1728     def _run(self, metakey):
1729         if self.container is None:
1730             self.client.del_account_meta(metakey)
1731         elif self.path is None:
1732             self.client.del_container_meta(metakey)
1733         else:
1734             self.client.del_object_meta(self.path, metakey)
1735
1736     def main(self, metakey, container____path__=None):
1737         super(self.__class__, self)._run(container____path__)
1738         self._run(metakey)
1739
1740
1741 @command(pithos_cmds)
1742 class store_quota(_store_account_command):
1743     """Get quota for account or container"""
1744
1745     arguments = dict(
1746         in_bytes=FlagArgument('Show result in bytes', ('-b', '--bytes'))
1747     )
1748
1749     @errors.generic.all
1750     @errors.pithos.connection
1751     @errors.pithos.container
1752     def _run(self):
1753         if self.container:
1754             reply = self.client.get_container_quota(self.container)
1755         else:
1756             reply = self.client.get_account_quota()
1757         if not self['in_bytes']:
1758             for k in reply:
1759                 reply[k] = format_size(reply[k])
1760         print_dict(pretty_keys(reply, '-'))
1761
1762     def main(self, container=None):
1763         super(self.__class__, self)._run()
1764         self.container = container
1765         self._run()
1766
1767
1768 @command(pithos_cmds)
1769 class store_setquota(_store_account_command):
1770     """Set new quota for account or container
1771     By default, quota is set in bytes
1772     Users may specify a different unit, e.g:
1773     /store setquota 2.3GB mycontainer
1774     Accepted units: B, KiB (1024 B), KB (1000 B), MiB, MB, GiB, GB, TiB, TB
1775     """
1776
1777     @errors.generic.all
1778     def _calculate_quota(self, user_input):
1779         quota = 0
1780         try:
1781             quota = int(user_input)
1782         except ValueError:
1783             index = 0
1784             digits = [str(num) for num in range(0, 10)] + ['.']
1785             while user_input[index] in digits:
1786                 index += 1
1787             quota = user_input[:index]
1788             format = user_input[index:]
1789             try:
1790                 return to_bytes(quota, format)
1791             except Exception as qe:
1792                 msg = 'Failed to convert %s to bytes' % user_input,
1793                 raiseCLIError(qe, msg, details=[
1794                     'Syntax: setquota <quota>[format] [container]',
1795                     'e.g.: setquota 2.3GB mycontainer',
1796                     'Acceptable formats:',
1797                     '(*1024): B, KiB, MiB, GiB, TiB',
1798                     '(*1000): B, KB, MB, GB, TB'])
1799         return quota
1800
1801     @errors.generic.all
1802     @errors.pithos.connection
1803     @errors.pithos.container
1804     def _run(self, quota):
1805         if self.container:
1806             self.client.container = self.container
1807             self.client.set_container_quota(quota)
1808         else:
1809             self.client.set_account_quota(quota)
1810
1811     def main(self, quota, container=None):
1812         super(self.__class__, self)._run()
1813         quota = self._calculate_quota(quota)
1814         self.container = container
1815         self._run(quota)
1816
1817
1818 @command(pithos_cmds)
1819 class store_versioning(_store_account_command):
1820     """Get  versioning for account or container"""
1821
1822     @errors.generic.all
1823     @errors.pithos.connection
1824     @errors.pithos.container
1825     def _run(self):
1826         if self.container:
1827             r = self.client.get_container_versioning(self.container)
1828         else:
1829             r = self.client.get_account_versioning()
1830         print_dict(r)
1831
1832     def main(self, container=None):
1833         super(self.__class__, self)._run()
1834         self.container = container
1835         self._run()
1836
1837
1838 @command(pithos_cmds)
1839 class store_setversioning(_store_account_command):
1840     """Set versioning mode (auto, none) for account or container"""
1841
1842     def _check_versioning(self, versioning):
1843         if versioning and versioning.lower() in ('auto', 'none'):
1844             return versioning.lower()
1845         raiseCLIError('Invalid versioning %s' % versioning, details=[
1846             'Versioning can be auto or none'])
1847
1848     @errors.generic.all
1849     @errors.pithos.connection
1850     @errors.pithos.container
1851     def _run(self, versioning):
1852         if self.container:
1853             self.client.container = self.container
1854             self.client.set_container_versioning(versioning)
1855         else:
1856             self.client.set_account_versioning(versioning)
1857
1858     def main(self, versioning, container=None):
1859         super(self.__class__, self)._run()
1860         self._run(self._check_versioning(versioning))
1861
1862
1863 @command(pithos_cmds)
1864 class store_group(_store_account_command):
1865     """Get groups and group members"""
1866
1867     @errors.generic.all
1868     @errors.pithos.connection
1869     def _run(self):
1870         r = self.client.get_account_group()
1871         print_dict(pretty_keys(r, '-'))
1872
1873     def main(self):
1874         super(self.__class__, self)._run()
1875         self._run()
1876
1877
1878 @command(pithos_cmds)
1879 class store_setgroup(_store_account_command):
1880     """Set a user group"""
1881
1882     @errors.generic.all
1883     @errors.pithos.connection
1884     def _run(self, groupname, *users):
1885         self.client.set_account_group(groupname, users)
1886
1887     def main(self, groupname, *users):
1888         super(self.__class__, self)._run()
1889         if users:
1890             self._run(groupname, *users)
1891         else:
1892             raiseCLIError('No users to add in group %s' % groupname)
1893
1894
1895 @command(pithos_cmds)
1896 class store_delgroup(_store_account_command):
1897     """Delete a user group"""
1898
1899     @errors.generic.all
1900     @errors.pithos.connection
1901     def _run(self, groupname):
1902         self.client.del_account_group(groupname)
1903
1904     def main(self, groupname):
1905         super(self.__class__, self)._run()
1906         self._run(groupname)
1907
1908
1909 @command(pithos_cmds)
1910 class store_sharers(_store_account_command):
1911     """List the accounts that share objects with current user"""
1912
1913     arguments = dict(
1914         detail=FlagArgument('show detailed output', ('-l', '--details')),
1915         marker=ValueArgument('show output greater then marker', '--marker')
1916     )
1917
1918     @errors.generic.all
1919     @errors.pithos.connection
1920     def _run(self):
1921         accounts = self.client.get_sharing_accounts(marker=self['marker'])
1922         if self['detail']:
1923             print_items(accounts)
1924         else:
1925             print_items([acc['name'] for acc in accounts])
1926
1927     def main(self):
1928         super(self.__class__, self)._run()
1929         self._run()
1930
1931
1932 @command(pithos_cmds)
1933 class store_versions(_store_container_command):
1934     """Get the list of object versions
1935     Deleted objects may still have versions that can be used to restore it and
1936     get information about its previous state.
1937     The version number can be used in a number of other commands, like info,
1938     copy, move, meta. See these commands for more information, e.g.
1939     /store info -h
1940     """
1941
1942     @errors.generic.all
1943     @errors.pithos.connection
1944     @errors.pithos.container
1945     @errors.pithos.object_path
1946     def _run(self):
1947         versions = self.client.get_object_versionlist(self.path)
1948         print_items([dict(id=vitem[0], created=strftime(
1949             '%d-%m-%Y %H:%M:%S',
1950             localtime(float(vitem[1])))) for vitem in versions])
1951
1952     def main(self, container___path):
1953         super(store_versions, self)._run(
1954             container___path,
1955             path_is_optional=False)
1956         self._run()