code.grnet.gr Git - kamaki/blob - kamaki/cli/commands/pithos_cli.py

   1 # Copyright 2011-2012 GRNET S.A. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or
   4 # without modification, are permitted provided that the following
   5 # conditions are met:
   6 #
   7 #   1. Redistributions of source code must retain the above
   8 #      copyright notice, this list of conditions and the following
   9 #      disclaimer.
  10 #
  11 #   2. Redistributions in binary form must reproduce the above
  12 #      copyright notice, this list of conditions and the following
  13 #      disclaimer in the documentation and/or other materials
  14 #      provided with the distribution.
  15 #
  16 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  17 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  23 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  24 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  26 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27 # POSSIBILITY OF SUCH DAMAGE.
  28 #
  29 # The views and conclusions contained in the software and
  30 # documentation are those of the authors and should not be
  31 # interpreted as representing official policies, either expressed
  32 # or implied, of GRNET S.A.command
  33
  34 from sys import stdout
  35 from time import localtime, strftime
  36 from logging import getLogger
  37 from os import path, makedirs
  38
  39 from kamaki.cli import command
  40 from kamaki.cli.command_tree import CommandTree
  41 from kamaki.cli.errors import raiseCLIError, CLISyntaxError
  42 from kamaki.cli.utils import (
  43     format_size,
  44     to_bytes,
  45     print_dict,
  46     print_items,
  47     pretty_keys,
  48     page_hold,
  49     bold,
  50     ask_user)
  51 from kamaki.cli.argument import FlagArgument, ValueArgument, IntArgument
  52 from kamaki.cli.argument import KeyValueArgument, DateArgument
  53 from kamaki.cli.argument import ProgressBarArgument
  54 from kamaki.cli.commands import _command_init, errors
  55 from kamaki.clients.pithos import PithosClient, ClientError
  56 from kamaki.clients.astakos import AstakosClient
  57
  58
  59 kloger = getLogger('kamaki')
  60
  61 pithos_cmds = CommandTree('store', 'Pithos+ storage commands')
  62 _commands = [pithos_cmds]
  63
  64
  65 # Argument functionality
  66
  67 class DelimiterArgument(ValueArgument):
  68     """
  69     :value type: string
  70     :value returns: given string or /
  71     """
  72
  73     def __init__(self, caller_obj, help='', parsed_name=None, default=None):
  74         super(DelimiterArgument, self).__init__(help, parsed_name, default)
  75         self.caller_obj = caller_obj
  76
  77     @property
  78     def value(self):
  79         if self.caller_obj['recursive']:
  80             return '/'
  81         return getattr(self, '_value', self.default)
  82
  83     @value.setter
  84     def value(self, newvalue):
  85         self._value = newvalue
  86
  87
  88 class SharingArgument(ValueArgument):
  89     """Set sharing (read and/or write) groups
  90     .
  91     :value type: "read=term1,term2,... write=term1,term2,..."
  92     .
  93     :value returns: {'read':['term1', 'term2', ...],
  94     .   'write':['term1', 'term2', ...]}
  95     """
  96
  97     @property
  98     def value(self):
  99         return getattr(self, '_value', self.default)
 100
 101     @value.setter
 102     def value(self, newvalue):
 103         perms = {}
 104         try:
 105             permlist = newvalue.split(' ')
 106         except AttributeError:
 107             return
 108         for p in permlist:
 109             try:
 110                 (key, val) = p.split('=')
 111             except ValueError as err:
 112                 raiseCLIError(
 113                     err,
 114                     'Error in --sharing',
 115                     details='Incorrect format',
 116                     importance=1)
 117             if key.lower() not in ('read', 'write'):
 118                 msg = 'Error in --sharing'
 119                 raiseCLIError(err, msg, importance=1, details=[
 120                     'Invalid permission key %s' % key])
 121             val_list = val.split(',')
 122             if not key in perms:
 123                 perms[key] = []
 124             for item in val_list:
 125                 if item not in perms[key]:
 126                     perms[key].append(item)
 127         self._value = perms
 128
 129
 130 class RangeArgument(ValueArgument):
 131     """
 132     :value type: string of the form <start>-<end> where <start> and <end> are
 133         integers
 134     :value returns: the input string, after type checking <start> and <end>
 135     """
 136
 137     @property
 138     def value(self):
 139         return getattr(self, '_value', self.default)
 140
 141     @value.setter
 142     def value(self, newvalue):
 143         if newvalue is None:
 144             self._value = self.default
 145             return
 146         (start, end) = newvalue.split('-')
 147         (start, end) = (int(start), int(end))
 148         self._value = '%s-%s' % (start, end)
 149
 150 # Command specs
 151
 152
 153 class _pithos_init(_command_init):
 154     """Initialize a pithos+ kamaki client"""
 155
 156     @staticmethod
 157     def _is_dir(remote_dict):
 158         return 'application/directory' == remote_dict.get(
 159             'content_type',
 160             remote_dict.get('content-type', ''))
 161
 162     @errors.generic.all
 163     def _run(self):
 164         self.token = self.config.get('store', 'token')\
 165             or self.config.get('global', 'token')
 166         self.base_url = self.config.get('store', 'url')\
 167             or self.config.get('global', 'url')
 168         self._set_account()
 169         self.container = self.config.get('store', 'container')\
 170             or self.config.get('global', 'container')
 171         self.client = PithosClient(
 172             base_url=self.base_url,
 173             token=self.token,
 174             account=self.account,
 175             container=self.container)
 176
 177     def main(self):
 178         self._run()
 179
 180     def _set_account(self):
 181         astakos = AstakosClient(self.config.get('astakos', 'url'), self.token)
 182         self.account = self['account'] or astakos.term('uuid')
 183
 184         """Backwards compatibility"""
 185         self.account = self.account\
 186             or self.config.get('store', 'account')\
 187             or self.config.get('global', 'account')
 188
 189
 190 class _store_account_command(_pithos_init):
 191     """Base class for account level storage commands"""
 192
 193     def __init__(self, arguments={}):
 194         super(_store_account_command, self).__init__(arguments)
 195         self['account'] = ValueArgument(
 196             'Set user account (not permanent)',
 197             ('-A', '--account'))
 198
 199     def _run(self):
 200         super(_store_account_command, self)._run()
 201         if self['account']:
 202             self.client.account = self['account']
 203
 204     @errors.generic.all
 205     def main(self):
 206         self._run()
 207
 208
 209 class _store_container_command(_store_account_command):
 210     """Base class for container level storage commands"""
 211
 212     container = None
 213     path = None
 214
 215     def __init__(self, arguments={}):
 216         super(_store_container_command, self).__init__(arguments)
 217         self['container'] = ValueArgument(
 218             'Set container to work with (temporary)',
 219             ('-C', '--container'))
 220
 221     def extract_container_and_path(
 222             self,
 223             container_with_path,
 224             path_is_optional=True):
 225         """Contains all heuristics for deciding what should be used as
 226         container or path. Options are:
 227         * user string of the form container:path
 228         * self.container, self.path variables set by super constructor, or
 229         explicitly by the caller application
 230         Error handling is explicit as these error cases happen only here
 231         """
 232         try:
 233             assert isinstance(container_with_path, str)
 234         except AssertionError as err:
 235             if self['container'] and path_is_optional:
 236                 self.container = self['container']
 237                 self.client.container = self['container']
 238                 return
 239             raiseCLIError(err)
 240
 241         user_cont, sep, userpath = container_with_path.partition(':')
 242
 243         if sep:
 244             if not user_cont:
 245                 raiseCLIError(CLISyntaxError(
 246                     'Container is missing\n',
 247                     details=errors.pithos.container_howto))
 248             alt_cont = self['container']
 249             if alt_cont and user_cont != alt_cont:
 250                 raiseCLIError(CLISyntaxError(
 251                     'Conflict: 2 containers (%s, %s)' % (user_cont, alt_cont),
 252                     details=errors.pithos.container_howto)
 253                 )
 254             self.container = user_cont
 255             if not userpath:
 256                 raiseCLIError(CLISyntaxError(
 257                     'Path is missing for object in container %s' % user_cont,
 258                     details=errors.pithos.container_howto)
 259                 )
 260             self.path = userpath
 261         else:
 262             alt_cont = self['container'] or self.client.container
 263             if alt_cont:
 264                 self.container = alt_cont
 265                 self.path = user_cont
 266             elif path_is_optional:
 267                 self.container = user_cont
 268                 self.path = None
 269             else:
 270                 self.container = user_cont
 271                 raiseCLIError(CLISyntaxError(
 272                     'Both container and path are required',
 273                     details=errors.pithos.container_howto)
 274                 )
 275
 276     @errors.generic.all
 277     def _run(self, container_with_path=None, path_is_optional=True):
 278         super(_store_container_command, self)._run()
 279         if self['container']:
 280             self.client.container = self['container']
 281             if container_with_path:
 282                 self.path = container_with_path
 283             elif not path_is_optional:
 284                 raise CLISyntaxError(
 285                     'Both container and path are required',
 286                     details=errors.pithos.container_howto)
 287         elif container_with_path:
 288             self.extract_container_and_path(
 289                 container_with_path,
 290                 path_is_optional)
 291             self.client.container = self.container
 292         self.container = self.client.container
 293
 294     def main(self, container_with_path=None, path_is_optional=True):
 295         self._run(container_with_path, path_is_optional)
 296
 297
 298 @command(pithos_cmds)
 299 class store_list(_store_container_command):
 300     """List containers, object trees or objects in a directory
 301     Use with:
 302     1 no parameters : containers in current account
 303     2. one parameter (container) or --container : contents of container
 304     3. <container>:<prefix> or --container=<container> <prefix>: objects in
 305     .   container starting with prefix
 306     """
 307
 308     arguments = dict(
 309         detail=FlagArgument('detailed output', ('-l', '--list')),
 310         limit=IntArgument('limit number of listed items', ('-n', '--number')),
 311         marker=ValueArgument('output greater that marker', '--marker'),
 312         prefix=ValueArgument('output starting with prefix', '--prefix'),
 313         delimiter=ValueArgument('show output up to delimiter', '--delimiter'),
 314         path=ValueArgument(
 315             'show output starting with prefix up to /',
 316             '--path'),
 317         meta=ValueArgument(
 318             'show output with specified meta keys',
 319             '--meta',
 320             default=[]),
 321         if_modified_since=ValueArgument(
 322             'show output modified since then',
 323             '--if-modified-since'),
 324         if_unmodified_since=ValueArgument(
 325             'show output not modified since then',
 326             '--if-unmodified-since'),
 327         until=DateArgument('show metadata until then', '--until'),
 328         format=ValueArgument(
 329             'format to parse until data (default: d/m/Y H:M:S )',
 330             '--format'),
 331         shared=FlagArgument('show only shared', '--shared'),
 332         public=FlagArgument('show only public', '--public'),
 333         more=FlagArgument(
 334             'output results in pages (-n to set items per page, default 10)',
 335             '--more'),
 336         exact_match=FlagArgument(
 337             'Show only objects that match exactly with path',
 338             '--exact-match')
 339     )
 340
 341     def print_objects(self, object_list):
 342         limit = int(self['limit']) if self['limit'] > 0 else len(object_list)
 343         for index, obj in enumerate(object_list):
 344             if self['exact_match'] and self.path and not (
 345                     obj['name'] == self.path or 'content_type' in obj):
 346                 continue
 347             pretty_obj = obj.copy()
 348             index += 1
 349             empty_space = ' ' * (len(str(len(object_list))) - len(str(index)))
 350             if obj['content_type'] == 'application/directory':
 351                 isDir = True
 352                 size = 'D'
 353             else:
 354                 isDir = False
 355                 size = format_size(obj['bytes'])
 356                 pretty_obj['bytes'] = '%s (%s)' % (obj['bytes'], size)
 357             oname = bold(obj['name'])
 358             if self['detail']:
 359                 print('%s%s. %s' % (empty_space, index, oname))
 360                 print_dict(pretty_keys(pretty_obj), exclude=('name'))
 361                 print
 362             else:
 363                 oname = '%s%s. %6s %s' % (empty_space, index, size, oname)
 364                 oname += '/' if isDir else ''
 365                 print(oname)
 366             if self['more']:
 367                 page_hold(index, limit, len(object_list))
 368
 369     def print_containers(self, container_list):
 370         limit = int(self['limit']) if self['limit'] > 0\
 371             else len(container_list)
 372         for index, container in enumerate(container_list):
 373             if 'bytes' in container:
 374                 size = format_size(container['bytes'])
 375             cname = '%s. %s' % (index + 1, bold(container['name']))
 376             if self['detail']:
 377                 print(cname)
 378                 pretty_c = container.copy()
 379                 if 'bytes' in container:
 380                     pretty_c['bytes'] = '%s (%s)' % (container['bytes'], size)
 381                 print_dict(pretty_keys(pretty_c), exclude=('name'))
 382                 print
 383             else:
 384                 if 'count' in container and 'bytes' in container:
 385                     print('%s (%s, %s objects)' % (
 386                         cname,
 387                         size,
 388                         container['count']))
 389                 else:
 390                     print(cname)
 391             if self['more']:
 392                 page_hold(index + 1, limit, len(container_list))
 393
 394     @errors.generic.all
 395     @errors.pithos.connection
 396     @errors.pithos.object_path
 397     @errors.pithos.container
 398     def _run(self):
 399         if self.container is None:
 400             r = self.client.account_get(
 401                 limit=False if self['more'] else self['limit'],
 402                 marker=self['marker'],
 403                 if_modified_since=self['if_modified_since'],
 404                 if_unmodified_since=self['if_unmodified_since'],
 405                 until=self['until'],
 406                 show_only_shared=self['shared'])
 407             self.print_containers(r.json)
 408         else:
 409             prefix = self.path or self['prefix']
 410             r = self.client.container_get(
 411                 limit=False if self['more'] else self['limit'],
 412                 marker=self['marker'],
 413                 prefix=prefix,
 414                 delimiter=self['delimiter'],
 415                 path=self['path'],
 416                 if_modified_since=self['if_modified_since'],
 417                 if_unmodified_since=self['if_unmodified_since'],
 418                 until=self['until'],
 419                 meta=self['meta'],
 420                 show_only_shared=self['shared'])
 421             self.print_objects(r.json)
 422
 423     def main(self, container____path__=None):
 424         super(self.__class__, self)._run(container____path__)
 425         self._run()
 426
 427
 428 @command(pithos_cmds)
 429 class store_mkdir(_store_container_command):
 430     """Create a directory"""
 431
 432     __doc__ += '\n. '.join([
 433         'Kamaki hanldes directories the same way as OOS Storage and Pithos+:',
 434         'A   directory  is   an  object  with  type  "application/directory"',
 435         'An object with path  dir/name can exist even if  dir does not exist',
 436         'or even if dir  is  a non  directory  object.  Users can modify dir',
 437         'without affecting the dir/name object in any way.'])
 438
 439     @errors.generic.all
 440     @errors.pithos.connection
 441     @errors.pithos.container
 442     def _run(self):
 443         self.client.create_directory(self.path)
 444
 445     def main(self, container___directory):
 446         super(self.__class__, self)._run(
 447             container___directory,
 448             path_is_optional=False)
 449         self._run()
 450
 451
 452 @command(pithos_cmds)
 453 class store_touch(_store_container_command):
 454     """Create an empty object (file)
 455     If object exists, this command will reset it to 0 length
 456     """
 457
 458     arguments = dict(
 459         content_type=ValueArgument(
 460             'Set content type (default: application/octet-stream)',
 461             '--content-type',
 462             default='application/octet-stream')
 463     )
 464
 465     @errors.generic.all
 466     @errors.pithos.connection
 467     @errors.pithos.container
 468     def _run(self):
 469         self.client.create_object(self.path, self['content_type'])
 470
 471     def main(self, container___path):
 472         super(store_touch, self)._run(
 473             container___path,
 474             path_is_optional=False)
 475         self._run()
 476
 477
 478 @command(pithos_cmds)
 479 class store_create(_store_container_command):
 480     """Create a container"""
 481
 482     arguments = dict(
 483         versioning=ValueArgument(
 484             'set container versioning (auto/none)',
 485             '--versioning'),
 486         quota=IntArgument('set default container quota', '--quota'),
 487         meta=KeyValueArgument(
 488             'set container metadata (can be repeated)',
 489             '--meta')
 490     )
 491
 492     @errors.generic.all
 493     @errors.pithos.connection
 494     @errors.pithos.container
 495     def _run(self):
 496         self.client.container_put(
 497             quota=self['quota'],
 498             versioning=self['versioning'],
 499             metadata=self['meta'])
 500
 501     def main(self, container=None):
 502         super(self.__class__, self)._run(container)
 503         if container and self.container != container:
 504             raiseCLIError('Invalid container name %s' % container, details=[
 505                 'Did you mean "%s" ?' % self.container,
 506                 'Use --container for names containing :'])
 507         self._run()
 508
 509
 510 class _source_destination_command(_store_container_command):
 511
 512     arguments = dict(
 513         destination_account=ValueArgument('', ('a', '--dst-account')),
 514         recursive=FlagArgument('', ('-R', '--recursive')),
 515         prefix=FlagArgument('', '--with-prefix', default=''),
 516         suffix=ValueArgument('', '--with-suffix', default=''),
 517         add_prefix=ValueArgument('', '--add-prefix', default=''),
 518         add_suffix=ValueArgument('', '--add-suffix', default=''),
 519         prefix_replace=ValueArgument('', '--prefix-to-replace', default=''),
 520         suffix_replace=ValueArgument('', '--suffix-to-replace', default='')
 521     )
 522
 523     def __init__(self, arguments={}):
 524         self.arguments.update(arguments)
 525         super(_source_destination_command, self).__init__(self.arguments)
 526
 527     def _run(self, source_container___path, path_is_optional=False):
 528         super(_source_destination_command, self)._run(
 529             source_container___path,
 530             path_is_optional)
 531         self.dst_client = PithosClient(
 532             base_url=self.client.base_url,
 533             token=self.client.token,
 534             account=self['destination_account'] or self.client.account)
 535
 536     @errors.generic.all
 537     @errors.pithos.account
 538     def _dest_container_path(self, dest_container_path):
 539         if self['destination_container']:
 540             self.dst_client.container = self['destination_container']
 541             return (self['destination_container'], dest_container_path)
 542         if dest_container_path:
 543             dst = dest_container_path.split(':')
 544             if len(dst) > 1:
 545                 try:
 546                     self.dst_client.container = dst[0]
 547                     self.dst_client.get_container_info(dst[0])
 548                 except ClientError as err:
 549                     if err.status in (404, 204):
 550                         raiseCLIError(
 551                             'Destination container %s not found' % dst[0])
 552                     raise
 553                 else:
 554                     self.dst_client.container = dst[0]
 555                 return (dst[0], dst[1])
 556             return(None, dst[0])
 557         raiseCLIError('No destination container:path provided')
 558
 559     def _get_all(self, prefix):
 560         return self.client.container_get(prefix=prefix).json
 561
 562     def _get_src_objects(self, src_path):
 563         """Get a list of the source objects to be called
 564
 565         :param src_path: (str) source path
 566
 567         :returns: (method, params) a method that returns a list when called
 568         or (object) if it is a single object
 569         """
 570         if src_path and src_path[-1] == '/':
 571             src_path = src_path[:-1]
 572
 573         if self['prefix']:
 574             return (self._get_all, dict(prefix=src_path))
 575         try:
 576             srcobj = self.client.get_object_info(src_path)
 577         except ClientError as srcerr:
 578             if srcerr.status == 404:
 579                 raiseCLIError(
 580                     'Source object %s not in source container %s' % (
 581                         src_path,
 582                         self.client.container),
 583                     details=['Hint: --with-prefix to match multiple objects'])
 584             elif srcerr.status not in (204,):
 585                 raise
 586             return (self.client.list_objects, {})
 587
 588         if self._is_dir(srcobj):
 589             if not self['recursive']:
 590                 raiseCLIError(
 591                     'Object %s of cont. %s is a dir' % (
 592                         src_path,
 593                         self.client.container),
 594                     details=['Use --recursive to access directories'])
 595             return (self._get_all, dict(prefix=src_path))
 596         srcobj['name'] = src_path
 597         return srcobj
 598
 599     def src_dst_pairs(self, dst_path):
 600         src_iter = self._get_src_objects(self.path)
 601         src_N = isinstance(src_iter, tuple)
 602         add_prefix = self['add_prefix'].strip('/')
 603
 604         if dst_path and dst_path.endswith('/'):
 605             dst_path = dst_path[:-1]
 606
 607         try:
 608             dstobj = self.dst_client.get_object_info(dst_path)
 609         except ClientError as trgerr:
 610             if trgerr.status in (404,):
 611                 if src_N:
 612                     raiseCLIError(
 613                         'Cannot merge multiple paths to path %s' % dst_path,
 614                         details=[
 615                             'Try to use / or a directory as destination',
 616                             'or create the destination dir (/store mkdir)',
 617                             'or use a single object as source'])
 618             elif trgerr.status not in (204,):
 619                 raise
 620         else:
 621             if self._is_dir(dstobj):
 622                 add_prefix = '%s/%s' % (dst_path.strip('/'), add_prefix)
 623             elif src_N:
 624                 raiseCLIError(
 625                     'Cannot merge multiple paths to path' % dst_path,
 626                     details=[
 627                         'Try to use / or a directory as destination',
 628                         'or create the destination dir (/store mkdir)',
 629                         'or use a single object as source'])
 630
 631         if src_N:
 632             (method, kwargs) = src_iter
 633             for obj in method(**kwargs):
 634                 name = obj['name']
 635                 if name.endswith(self['suffix']):
 636                     yield (name, self._get_new_object(name, add_prefix))
 637         elif src_iter['name'].endswith(self['suffix']):
 638             name = src_iter['name']
 639             yield (name, self._get_new_object(dst_path or name, add_prefix))
 640         else:
 641             raiseCLIError('Source path %s conflicts with suffix %s' % (
 642                 src_iter['name'],
 643                 self['suffix']))
 644
 645     def _get_new_object(self, obj, add_prefix):
 646         if self['prefix_replace'] and obj.startswith(self['prefix_replace']):
 647             obj = obj[len(self['prefix_replace']):]
 648         if self['suffix_replace'] and obj.endswith(self['suffix_replace']):
 649             obj = obj[:-len(self['suffix_replace'])]
 650         return add_prefix + obj + self['add_suffix']
 651
 652
 653 @command(pithos_cmds)
 654 class store_copy(_source_destination_command):
 655     """Copy objects from container to (another) container
 656     Semantics:
 657     copy cont:path dir
 658     .   transfer path as dir/path
 659     copy cont:path cont2:
 660     .   trasnfer all <obj> prefixed with path to container cont2
 661     copy cont:path [cont2:]path2
 662     .   transfer path to path2
 663     Use options:
 664     1. <container1>:<path1> [container2:]<path2> : if container2 is not given,
 665     destination is container1:path2
 666     2. <container>:<path1> <path2> : make a copy in the same container
 667     3. Can use --container= instead of <container1>
 668     """
 669
 670     arguments = dict(
 671         destination_account=ValueArgument(
 672             'Account to copy to',
 673             ('-a', '--dst-account')),
 674         destination_container=ValueArgument(
 675             'use it if destination container name contains a : character',
 676             ('-D', '--dst-container')),
 677         source_version=ValueArgument(
 678             'copy specific version',
 679             ('-S', '--source-version')),
 680         public=ValueArgument('make object publicly accessible', '--public'),
 681         content_type=ValueArgument(
 682             'change object\'s content type',
 683             '--content-type'),
 684         recursive=FlagArgument(
 685             'copy directory and contents',
 686             ('-R', '--recursive')),
 687         prefix=FlagArgument(
 688             'Match objects prefixed with src path (feels like src_path*)',
 689             '--with-prefix',
 690             default=''),
 691         suffix=ValueArgument(
 692             'Suffix of source objects (feels like *suffix)',
 693             '--with-suffix',
 694             default=''),
 695         add_prefix=ValueArgument('Prefix targets', '--add-prefix', default=''),
 696         add_suffix=ValueArgument('Suffix targets', '--add-suffix', default=''),
 697         prefix_replace=ValueArgument(
 698             'Prefix of src to replace with dst path + add_prefix, if matched',
 699             '--prefix-to-replace',
 700             default=''),
 701         suffix_replace=ValueArgument(
 702             'Suffix of src to replace with add_suffix, if matched',
 703             '--suffix-to-replace',
 704             default='')
 705     )
 706
 707     @errors.generic.all
 708     @errors.pithos.connection
 709     @errors.pithos.container
 710     @errors.pithos.account
 711     def _run(self, dst_path):
 712         no_source_object = True
 713         src_account = self.client.account if (
 714             self['destination_account']) else None
 715         for src_obj, dst_obj in self.src_dst_pairs(dst_path):
 716             no_source_object = False
 717             self.dst_client.copy_object(
 718                 src_container=self.client.container,
 719                 src_object=src_obj,
 720                 dst_container=self.dst_client.container,
 721                 dst_object=dst_obj,
 722                 source_account=src_account,
 723                 source_version=self['source_version'],
 724                 public=self['public'],
 725                 content_type=self['content_type'])
 726         if no_source_object:
 727             raiseCLIError('No object %s in container %s' % (
 728                 self.path,
 729                 self.container))
 730
 731     def main(
 732             self,
 733             source_container___path,
 734             destination_container___path=None):
 735         super(store_copy, self)._run(
 736             source_container___path,
 737             path_is_optional=False)
 738         (dst_cont, dst_path) = self._dest_container_path(
 739             destination_container___path)
 740         self.dst_client.container = dst_cont or self.container
 741         self._run(dst_path=dst_path or '')
 742
 743
 744 @command(pithos_cmds)
 745 class store_move(_source_destination_command):
 746     """Move/rename objects from container to (another) container
 747     Semantics:
 748     move cont:path dir
 749     .   rename path as dir/path
 750     move cont:path cont2:
 751     .   trasnfer all <obj> prefixed with path to container cont2
 752     move cont:path [cont2:]path2
 753     .   transfer path to path2
 754     Use options:
 755     1. <container1>:<path1> [container2:]<path2> : if container2 is not given,
 756     destination is container1:path2
 757     2. <container>:<path1> <path2> : move in the same container
 758     3. Can use --container= instead of <container1>
 759     """
 760
 761     arguments = dict(
 762         destination_account=ValueArgument(
 763             'Account to move to',
 764             ('-a', '--dst-account')),
 765         destination_container=ValueArgument(
 766             'use it if destination container name contains a : character',
 767             ('-D', '--dst-container')),
 768         source_version=ValueArgument(
 769             'copy specific version',
 770             '--source-version'),
 771         public=ValueArgument('make object publicly accessible', '--public'),
 772         content_type=ValueArgument(
 773             'change object\'s content type',
 774             '--content-type'),
 775         recursive=FlagArgument(
 776             'copy directory and contents',
 777             ('-R', '--recursive')),
 778         prefix=FlagArgument(
 779             'Match objects prefixed with src path (feels like src_path*)',
 780             '--with-prefix',
 781             default=''),
 782         suffix=ValueArgument(
 783             'Suffix of source objects (feels like *suffix)',
 784             '--with-suffix',
 785             default=''),
 786         add_prefix=ValueArgument('Prefix targets', '--add-prefix', default=''),
 787         add_suffix=ValueArgument('Suffix targets', '--add-suffix', default=''),
 788         prefix_replace=ValueArgument(
 789             'Prefix of src to replace with dst path + add_prefix, if matched',
 790             '--prefix-to-replace',
 791             default=''),
 792         suffix_replace=ValueArgument(
 793             'Suffix of src to replace with add_suffix, if matched',
 794             '--suffix-to-replace',
 795             default='')
 796     )
 797
 798     @errors.generic.all
 799     @errors.pithos.connection
 800     @errors.pithos.container
 801     def _run(self, dst_path):
 802         no_source_object = True
 803         src_account = self.client.account if (
 804             self['destination_account']) else None
 805         for src_obj, dst_obj in self.src_dst_pairs(dst_path):
 806             no_source_object = False
 807             self.dst_client.move_object(
 808                 src_container=self.container,
 809                 src_object=src_obj,
 810                 dst_container=self.dst_client.container,
 811                 dst_object=dst_obj,
 812                 source_account=src_account,
 813                 source_version=self['source_version'],
 814                 public=self['public'],
 815                 content_type=self['content_type'])
 816         if no_source_object:
 817             raiseCLIError('No object %s in container %s' % (
 818                 self.path,
 819                 self.container))
 820
 821     def main(
 822             self,
 823             source_container___path,
 824             destination_container___path=None):
 825         super(self.__class__, self)._run(
 826             source_container___path,
 827             path_is_optional=False)
 828         (dst_cont, dst_path) = self._dest_container_path(
 829             destination_container___path)
 830         (dst_cont, dst_path) = self._dest_container_path(
 831             destination_container___path)
 832         self.dst_client.container = dst_cont or self.container
 833         self._run(dst_path=dst_path or '')
 834
 835
 836 @command(pithos_cmds)
 837 class store_append(_store_container_command):
 838     """Append local file to (existing) remote object
 839     The remote object should exist.
 840     If the remote object is a directory, it is transformed into a file.
 841     In the later case, objects under the directory remain intact.
 842     """
 843
 844     arguments = dict(
 845         progress_bar=ProgressBarArgument(
 846             'do not show progress bar',
 847             ('-N', '--no-progress-bar'),
 848             default=False)
 849     )
 850
 851     @errors.generic.all
 852     @errors.pithos.connection
 853     @errors.pithos.container
 854     @errors.pithos.object_path
 855     def _run(self, local_path):
 856         (progress_bar, upload_cb) = self._safe_progress_bar('Appending')
 857         try:
 858             f = open(local_path, 'rb')
 859             self.client.append_object(self.path, f, upload_cb)
 860         except Exception:
 861             self._safe_progress_bar_finish(progress_bar)
 862             raise
 863         finally:
 864             self._safe_progress_bar_finish(progress_bar)
 865
 866     def main(self, local_path, container___path):
 867         super(self.__class__, self)._run(
 868             container___path,
 869             path_is_optional=False)
 870         self._run(local_path)
 871
 872
 873 @command(pithos_cmds)
 874 class store_truncate(_store_container_command):
 875     """Truncate remote file up to a size (default is 0)"""
 876
 877     @errors.generic.all
 878     @errors.pithos.connection
 879     @errors.pithos.container
 880     @errors.pithos.object_path
 881     @errors.pithos.object_size
 882     def _run(self, size=0):
 883         self.client.truncate_object(self.path, size)
 884
 885     def main(self, container___path, size=0):
 886         super(self.__class__, self)._run(container___path)
 887         self._run(size=size)
 888
 889
 890 @command(pithos_cmds)
 891 class store_overwrite(_store_container_command):
 892     """Overwrite part (from start to end) of a remote file
 893     overwrite local-path container 10 20
 894     .   will overwrite bytes from 10 to 20 of a remote file with the same name
 895     .   as local-path basename
 896     overwrite local-path container:path 10 20
 897     .   will overwrite as above, but the remote file is named path
 898     """
 899
 900     arguments = dict(
 901         progress_bar=ProgressBarArgument(
 902             'do not show progress bar',
 903             ('-N', '--no-progress-bar'),
 904             default=False)
 905     )
 906
 907     def _open_file(self, local_path, start):
 908         f = open(path.abspath(local_path), 'rb')
 909         f.seek(0, 2)
 910         f_size = f.tell()
 911         f.seek(start, 0)
 912         return (f, f_size)
 913
 914     @errors.generic.all
 915     @errors.pithos.connection
 916     @errors.pithos.container
 917     @errors.pithos.object_path
 918     @errors.pithos.object_size
 919     def _run(self, local_path, start, end):
 920         (start, end) = (int(start), int(end))
 921         (f, f_size) = self._open_file(local_path, start)
 922         (progress_bar, upload_cb) = self._safe_progress_bar(
 923             'Overwrite %s bytes' % (end - start))
 924         try:
 925             self.client.overwrite_object(
 926                 obj=self.path,
 927                 start=start,
 928                 end=end,
 929                 source_file=f,
 930                 upload_cb=upload_cb)
 931         except Exception:
 932             self._safe_progress_bar_finish(progress_bar)
 933             raise
 934         finally:
 935             self._safe_progress_bar_finish(progress_bar)
 936
 937     def main(self, local_path, container___path, start, end):
 938         super(self.__class__, self)._run(
 939             container___path,
 940             path_is_optional=None)
 941         self.path = self.path or path.basename(local_path)
 942         self._run(local_path=local_path, start=start, end=end)
 943
 944
 945 @command(pithos_cmds)
 946 class store_manifest(_store_container_command):
 947     """Create a remote file of uploaded parts by manifestation
 948     Remains functional for compatibility with OOS Storage. Users are advised
 949     to use the upload command instead.
 950     Manifestation is a compliant process for uploading large files. The files
 951     have to be chunked in smalled files and uploaded as <prefix><increment>
 952     where increment is 1, 2, ...
 953     Finally, the manifest command glues partial files together in one file
 954     named <prefix>
 955     The upload command is faster, easier and more intuitive than manifest
 956     """
 957
 958     arguments = dict(
 959         etag=ValueArgument('check written data', '--etag'),
 960         content_encoding=ValueArgument(
 961             'set MIME content type',
 962             '--content-encoding'),
 963         content_disposition=ValueArgument(
 964             'the presentation style of the object',
 965             '--content-disposition'),
 966         content_type=ValueArgument(
 967             'specify content type',
 968             '--content-type',
 969             default='application/octet-stream'),
 970         sharing=SharingArgument(
 971             '\n'.join([
 972                 'define object sharing policy',
 973                 '    ( "read=user1,grp1,user2,... write=user1,grp2,..." )']),
 974             '--sharing'),
 975         public=FlagArgument('make object publicly accessible', '--public')
 976     )
 977
 978     @errors.generic.all
 979     @errors.pithos.connection
 980     @errors.pithos.container
 981     @errors.pithos.object_path
 982     def _run(self):
 983         self.client.create_object_by_manifestation(
 984             self.path,
 985             content_encoding=self['content_encoding'],
 986             content_disposition=self['content_disposition'],
 987             content_type=self['content_type'],
 988             sharing=self['sharing'],
 989             public=self['public'])
 990
 991     def main(self, container___path):
 992         super(self.__class__, self)._run(
 993             container___path,
 994             path_is_optional=False)
 995         self.run()
 996
 997
 998 @command(pithos_cmds)
 999 class store_upload(_store_container_command):
1000     """Upload a file"""
1001
1002     arguments = dict(
1003         use_hashes=FlagArgument(
1004             'provide hashmap file instead of data',
1005             '--use-hashes'),
1006         etag=ValueArgument('check written data', '--etag'),
1007         unchunked=FlagArgument('avoid chunked transfer mode', '--unchunked'),
1008         content_encoding=ValueArgument(
1009             'set MIME content type',
1010             '--content-encoding'),
1011         content_disposition=ValueArgument(
1012             'specify objects presentation style',
1013             '--content-disposition'),
1014         content_type=ValueArgument('specify content type', '--content-type'),
1015         sharing=SharingArgument(
1016             help='\n'.join([
1017                 'define sharing object policy',
1018                 '( "read=user1,grp1,user2,... write=user1,grp2,... )']),
1019             parsed_name='--sharing'),
1020         public=FlagArgument('make object publicly accessible', '--public'),
1021         poolsize=IntArgument('set pool size', '--with-pool-size'),
1022         progress_bar=ProgressBarArgument(
1023             'do not show progress bar',
1024             ('-N', '--no-progress-bar'),
1025             default=False),
1026         overwrite=FlagArgument('Force (over)write', ('-f', '--force'))
1027     )
1028
1029     def _remote_path(self, remote_path, local_path=''):
1030         if self['overwrite']:
1031             return remote_path
1032         try:
1033             r = self.client.get_object_info(remote_path)
1034         except ClientError as ce:
1035             if ce.status == 404:
1036                 return remote_path
1037             raise ce
1038         ctype = r.get('content-type', '')
1039         if 'application/directory' == ctype.lower():
1040             ret = '%s/%s' % (remote_path, local_path)
1041             return self._remote_path(ret) if local_path else ret
1042         raiseCLIError(
1043             'Object %s already exists' % remote_path,
1044             importance=1,
1045             details=['use -f to overwrite or resume'])
1046
1047     @errors.generic.all
1048     @errors.pithos.connection
1049     @errors.pithos.container
1050     @errors.pithos.object_path
1051     @errors.pithos.local_path
1052     def _run(self, local_path, remote_path):
1053         poolsize = self['poolsize']
1054         if poolsize > 0:
1055             self.client.MAX_THREADS = int(poolsize)
1056         params = dict(
1057             content_encoding=self['content_encoding'],
1058             content_type=self['content_type'],
1059             content_disposition=self['content_disposition'],
1060             sharing=self['sharing'],
1061             public=self['public'])
1062         remote_path = self._remote_path(remote_path, local_path)
1063         with open(path.abspath(local_path), 'rb') as f:
1064             if self['unchunked']:
1065                 self.client.upload_object_unchunked(
1066                     remote_path,
1067                     f,
1068                     etag=self['etag'],
1069                     withHashFile=self['use_hashes'],
1070                     **params)
1071             else:
1072                 try:
1073                     (progress_bar, upload_cb) = self._safe_progress_bar(
1074                         'Uploading')
1075                     if progress_bar:
1076                         hash_bar = progress_bar.clone()
1077                         hash_cb = hash_bar.get_generator(
1078                             'Calculating block hashes')
1079                     else:
1080                         hash_cb = None
1081                     self.client.upload_object(
1082                         remote_path,
1083                         f,
1084                         hash_cb=hash_cb,
1085                         upload_cb=upload_cb,
1086                         **params)
1087                 except Exception:
1088                     self._safe_progress_bar_finish(progress_bar)
1089                     raise
1090                 finally:
1091                     self._safe_progress_bar_finish(progress_bar)
1092         print 'Upload completed'
1093
1094     def main(self, local_path, container____path__=None):
1095         super(self.__class__, self)._run(container____path__)
1096         remote_path = self.path or path.basename(local_path)
1097         self._run(local_path=local_path, remote_path=remote_path)
1098
1099
1100 @command(pithos_cmds)
1101 class store_cat(_store_container_command):
1102     """Print remote file contents to console"""
1103
1104     arguments = dict(
1105         range=RangeArgument('show range of data', '--range'),
1106         if_match=ValueArgument('show output if ETags match', '--if-match'),
1107         if_none_match=ValueArgument(
1108             'show output if ETags match',
1109             '--if-none-match'),
1110         if_modified_since=DateArgument(
1111             'show output modified since then',
1112             '--if-modified-since'),
1113         if_unmodified_since=DateArgument(
1114             'show output unmodified since then',
1115             '--if-unmodified-since'),
1116         object_version=ValueArgument(
1117             'get the specific version',
1118             ('-j', '--object-version'))
1119     )
1120
1121     @errors.generic.all
1122     @errors.pithos.connection
1123     @errors.pithos.container
1124     @errors.pithos.object_path
1125     def _run(self):
1126         self.client.download_object(
1127             self.path,
1128             stdout,
1129             range_str=self['range'],
1130             version=self['object_version'],
1131             if_match=self['if_match'],
1132             if_none_match=self['if_none_match'],
1133             if_modified_since=self['if_modified_since'],
1134             if_unmodified_since=self['if_unmodified_since'])
1135
1136     def main(self, container___path):
1137         super(self.__class__, self)._run(
1138             container___path,
1139             path_is_optional=False)
1140         self._run()
1141
1142
1143 @command(pithos_cmds)
1144 class store_download(_store_container_command):
1145     """Download remote object as local file
1146     If local destination is a directory:
1147     *   download <container>:<path> <local dir> -R
1148     will download all files on <container> prefixed as <path>,
1149     to <local dir>/<full path>
1150     *   download <container>:<path> <local dir> --exact-match
1151     will download only one file, exactly matching <path>
1152     ATTENTION: to download cont:dir1/dir2/file there must exist objects
1153     cont:dir1 and cont:dir1/dir2 of type application/directory
1154     To create directory objects, use /store mkdir
1155     """
1156
1157     arguments = dict(
1158         resume=FlagArgument('Resume instead of overwrite', ('-r', '--resume')),
1159         range=RangeArgument('show range of data', '--range'),
1160         if_match=ValueArgument('show output if ETags match', '--if-match'),
1161         if_none_match=ValueArgument(
1162             'show output if ETags match',
1163             '--if-none-match'),
1164         if_modified_since=DateArgument(
1165             'show output modified since then',
1166             '--if-modified-since'),
1167         if_unmodified_since=DateArgument(
1168             'show output unmodified since then',
1169             '--if-unmodified-since'),
1170         object_version=ValueArgument(
1171             'get the specific version',
1172             ('-j', '--object-version')),
1173         poolsize=IntArgument('set pool size', '--with-pool-size'),
1174         progress_bar=ProgressBarArgument(
1175             'do not show progress bar',
1176             ('-N', '--no-progress-bar'),
1177             default=False),
1178         recursive=FlagArgument(
1179             'Download a remote path and all its contents',
1180             ('-R', '--recursive'))
1181     )
1182
1183     def _outputs(self, local_path):
1184         """:returns: (local_file, remote_path)"""
1185         remotes = []
1186         if self['recursive']:
1187             r = self.client.container_get(
1188                 prefix=self.path or '/',
1189                 if_modified_since=self['if_modified_since'],
1190                 if_unmodified_since=self['if_unmodified_since'])
1191             dirlist = dict()
1192             for remote in r.json:
1193                 rname = remote['name'].strip('/')
1194                 tmppath = ''
1195                 for newdir in rname.strip('/').split('/')[:-1]:
1196                     tmppath = '/'.join([tmppath, newdir])
1197                     dirlist.update({tmppath.strip('/'): True})
1198                 remotes.append((rname, store_download._is_dir(remote)))
1199             dir_remotes = [r[0] for r in remotes if r[1]]
1200             if not set(dirlist).issubset(dir_remotes):
1201                 badguys = [bg.strip('/') for bg in set(
1202                     dirlist).difference(dir_remotes)]
1203                 raiseCLIError(
1204                     'Some remote paths contain non existing directories',
1205                     details=['Missing remote directories:'] + badguys)
1206         elif self.path:
1207             r = self.client.get_object_info(
1208                 self.path,
1209                 version=self['object_version'])
1210             if store_download._is_dir(r):
1211                 raiseCLIError(
1212                     'Illegal download: Remote object %s is a directory' % (
1213                         self.path),
1214                     details=['To download a directory, try --recursive'])
1215             if '/' in self.path.strip('/') and not local_path:
1216                 raiseCLIError(
1217                     'Illegal download: remote object %s contains "/"' % (
1218                         self.path),
1219                     details=[
1220                         'To download an object containing "/" characters',
1221                         'either create the remote directories or',
1222                         'specify a non-directory local path for this object'])
1223             remotes = [(self.path, False)]
1224         if not remotes:
1225             if self.path:
1226                 raiseCLIError(
1227                     'No matching path %s on container %s' % (
1228                         self.path,
1229                         self.container),
1230                     details=[
1231                         'To list the contents of %s, try:' % self.container,
1232                         '   /store list %s' % self.container])
1233             raiseCLIError(
1234                 'Illegal download of container %s' % self.container,
1235                 details=[
1236                     'To download a whole container, try:',
1237                     '   /store download --recursive <container>'])
1238
1239         lprefix = path.abspath(local_path or path.curdir)
1240         if path.isdir(lprefix):
1241             for rpath, remote_is_dir in remotes:
1242                 lpath = '/%s/%s' % (lprefix.strip('/'), rpath.strip('/'))
1243                 if remote_is_dir:
1244                     if path.exists(lpath) and path.isdir(lpath):
1245                         continue
1246                     makedirs(lpath)
1247                 elif path.exists(lpath):
1248                     if not self['resume']:
1249                         print('File %s exists, aborting...' % lpath)
1250                         continue
1251                     with open(lpath, 'rwb+') as f:
1252                         yield (f, rpath)
1253                 else:
1254                     with open(lpath, 'wb+') as f:
1255                         yield (f, rpath)
1256         elif path.exists(lprefix):
1257             if len(remotes) > 1:
1258                 raiseCLIError(
1259                     '%s remote objects cannot be merged in local file %s' % (
1260                         len(remotes),
1261                         local_path),
1262                     details=[
1263                         'To download multiple objects, local path should be',
1264                         'a directory, or use download without a local path'])
1265             (rpath, remote_is_dir) = remotes[0]
1266             if remote_is_dir:
1267                 raiseCLIError(
1268                     'Remote directory %s should not replace local file %s' % (
1269                         rpath,
1270                         local_path))
1271             if self['resume']:
1272                 with open(lprefix, 'rwb+') as f:
1273                     yield (f, rpath)
1274             else:
1275                 raiseCLIError(
1276                     'Local file %s already exist' % local_path,
1277                     details=['Try --resume to overwrite it'])
1278         else:
1279             if len(remotes) > 1 or remotes[0][1]:
1280                 raiseCLIError(
1281                     'Local directory %s does not exist' % local_path)
1282             with open(lprefix, 'wb+') as f:
1283                 yield (f, remotes[0][0])
1284
1285     @errors.generic.all
1286     @errors.pithos.connection
1287     @errors.pithos.container
1288     @errors.pithos.object_path
1289     @errors.pithos.local_path
1290     def _run(self, local_path):
1291         #outputs = self._outputs(local_path)
1292         poolsize = self['poolsize']
1293         if poolsize:
1294             self.client.MAX_THREADS = int(poolsize)
1295         progress_bar = None
1296         try:
1297             for f, rpath in self._outputs(local_path):
1298                 (
1299                     progress_bar,
1300                     download_cb) = self._safe_progress_bar(
1301                         'Download %s' % rpath)
1302                 self.client.download_object(
1303                     rpath,
1304                     f,
1305                     download_cb=download_cb,
1306                     range_str=self['range'],
1307                     version=self['object_version'],
1308                     if_match=self['if_match'],
1309                     resume=self['resume'],
1310                     if_none_match=self['if_none_match'],
1311                     if_modified_since=self['if_modified_since'],
1312                     if_unmodified_since=self['if_unmodified_since'])
1313         except KeyboardInterrupt:
1314             from threading import enumerate as activethreads
1315             stdout.write('\nFinishing active threads ')
1316             for thread in activethreads():
1317                 stdout.flush()
1318                 try:
1319                     thread.join()
1320                     stdout.write('.')
1321                 except RuntimeError:
1322                     continue
1323             print('\ndownload canceled by user')
1324             if local_path is not None:
1325                 print('to resume, re-run with --resume')
1326         except Exception:
1327             self._safe_progress_bar_finish(progress_bar)
1328             raise
1329         finally:
1330             self._safe_progress_bar_finish(progress_bar)
1331
1332     def main(self, container___path, local_path=None):
1333         super(self.__class__, self)._run(container___path)
1334         self._run(local_path=local_path)
1335
1336
1337 @command(pithos_cmds)
1338 class store_hashmap(_store_container_command):
1339     """Get the hash-map of an object"""
1340
1341     arguments = dict(
1342         if_match=ValueArgument('show output if ETags match', '--if-match'),
1343         if_none_match=ValueArgument(
1344             'show output if ETags match',
1345             '--if-none-match'),
1346         if_modified_since=DateArgument(
1347             'show output modified since then',
1348             '--if-modified-since'),
1349         if_unmodified_since=DateArgument(
1350             'show output unmodified since then',
1351             '--if-unmodified-since'),
1352         object_version=ValueArgument(
1353             'get the specific version',
1354             ('-j', '--object-version'))
1355     )
1356
1357     @errors.generic.all
1358     @errors.pithos.connection
1359     @errors.pithos.container
1360     @errors.pithos.object_path
1361     def _run(self):
1362         data = self.client.get_object_hashmap(
1363             self.path,
1364             version=self['object_version'],
1365             if_match=self['if_match'],
1366             if_none_match=self['if_none_match'],
1367             if_modified_since=self['if_modified_since'],
1368             if_unmodified_since=self['if_unmodified_since'])
1369         print_dict(data)
1370
1371     def main(self, container___path):
1372         super(self.__class__, self)._run(
1373             container___path,
1374             path_is_optional=False)
1375         self._run()
1376
1377
1378 @command(pithos_cmds)
1379 class store_delete(_store_container_command):
1380     """Delete a container [or an object]
1381     How to delete a non-empty container:
1382     - empty the container:  /store delete -R <container>
1383     - delete it:            /store delete <container>
1384     .
1385     Semantics of directory deletion:
1386     .a preserve the contents: /store delete <container>:<directory>
1387     .    objects of the form dir/filename can exist with a dir object
1388     .b delete contents:       /store delete -R <container>:<directory>
1389     .    all dir/* objects are affected, even if dir does not exist
1390     .
1391     To restore a deleted object OBJ in a container CONT:
1392     - get object versions: /store versions CONT:OBJ
1393     .   and choose the version to be restored
1394     - restore the object:  /store copy --source-version=<version> CONT:OBJ OBJ
1395     """
1396
1397     arguments = dict(
1398         until=DateArgument('remove history until that date', '--until'),
1399         yes=FlagArgument('Do not prompt for permission', '--yes'),
1400         recursive=FlagArgument(
1401             'empty dir or container and delete (if dir)',
1402             ('-R', '--recursive'))
1403     )
1404
1405     def __init__(self, arguments={}):
1406         super(self.__class__, self).__init__(arguments)
1407         self['delimiter'] = DelimiterArgument(
1408             self,
1409             parsed_name='--delimiter',
1410             help='delete objects prefixed with <object><delimiter>')
1411
1412     @errors.generic.all
1413     @errors.pithos.connection
1414     @errors.pithos.container
1415     @errors.pithos.object_path
1416     def _run(self):
1417         if self.path:
1418             if self['yes'] or ask_user(
1419                     'Delete %s:%s ?' % (self.container, self.path)):
1420                 self.client.del_object(
1421                     self.path,
1422                     until=self['until'],
1423                     delimiter=self['delimiter'])
1424             else:
1425                 print('Aborted')
1426         else:
1427             if self['recursive']:
1428                 ask_msg = 'Delete container contents'
1429             else:
1430                 ask_msg = 'Delete container'
1431             if self['yes'] or ask_user('%s %s ?' % (ask_msg, self.container)):
1432                 self.client.del_container(
1433                     until=self['until'],
1434                     delimiter=self['delimiter'])
1435             else:
1436                 print('Aborted')
1437
1438     def main(self, container____path__=None):
1439         super(self.__class__, self)._run(container____path__)
1440         self._run()
1441
1442
1443 @command(pithos_cmds)
1444 class store_purge(_store_container_command):
1445     """Delete a container and release related data blocks
1446     Non-empty containers can not purged.
1447     To purge a container with content:
1448     .   /store delete -R <container>
1449     .      objects are deleted, but data blocks remain on server
1450     .   /store purge <container>
1451     .      container and data blocks are released and deleted
1452     """
1453
1454     arguments = dict(
1455         yes=FlagArgument('Do not prompt for permission', '--yes'),
1456     )
1457
1458     @errors.generic.all
1459     @errors.pithos.connection
1460     @errors.pithos.container
1461     def _run(self):
1462         if self['yes'] or ask_user('Purge container %s?' % self.container):
1463                 self.client.purge_container()
1464         else:
1465             print('Aborted')
1466
1467     def main(self, container=None):
1468         super(self.__class__, self)._run(container)
1469         if container and self.container != container:
1470             raiseCLIError('Invalid container name %s' % container, details=[
1471                 'Did you mean "%s" ?' % self.container,
1472                 'Use --container for names containing :'])
1473         self._run()
1474
1475
1476 @command(pithos_cmds)
1477 class store_publish(_store_container_command):
1478     """Publish the object and print the public url"""
1479
1480     @errors.generic.all
1481     @errors.pithos.connection
1482     @errors.pithos.container
1483     @errors.pithos.object_path
1484     def _run(self):
1485         url = self.client.publish_object(self.path)
1486         print(url)
1487
1488     def main(self, container___path):
1489         super(self.__class__, self)._run(
1490             container___path,
1491             path_is_optional=False)
1492         self._run()
1493
1494
1495 @command(pithos_cmds)
1496 class store_unpublish(_store_container_command):
1497     """Unpublish an object"""
1498
1499     @errors.generic.all
1500     @errors.pithos.connection
1501     @errors.pithos.container
1502     @errors.pithos.object_path
1503     def _run(self):
1504             self.client.unpublish_object(self.path)
1505
1506     def main(self, container___path):
1507         super(self.__class__, self)._run(
1508             container___path,
1509             path_is_optional=False)
1510         self._run()
1511
1512
1513 @command(pithos_cmds)
1514 class store_permissions(_store_container_command):
1515     """Get read and write permissions of an object
1516     Permissions are lists of users and user groups. There is read and write
1517     permissions. Users and groups with write permission have also read
1518     permission.
1519     """
1520
1521     @errors.generic.all
1522     @errors.pithos.connection
1523     @errors.pithos.container
1524     @errors.pithos.object_path
1525     def _run(self):
1526         r = self.client.get_object_sharing(self.path)
1527         print_dict(r)
1528
1529     def main(self, container___path):
1530         super(self.__class__, self)._run(
1531             container___path,
1532             path_is_optional=False)
1533         self._run()
1534
1535
1536 @command(pithos_cmds)
1537 class store_setpermissions(_store_container_command):
1538     """Set permissions for an object
1539     New permissions overwrite existing permissions.
1540     Permission format:
1541     -   read=<username>[,usergroup[,...]]
1542     -   write=<username>[,usegroup[,...]]
1543     E.g. to give read permissions for file F to users A and B and write for C:
1544     .       /store setpermissions F read=A,B write=C
1545     """
1546
1547     @errors.generic.all
1548     def format_permition_dict(self, permissions):
1549         read = False
1550         write = False
1551         for perms in permissions:
1552             splstr = perms.split('=')
1553             if 'read' == splstr[0]:
1554                 read = [ug.strip() for ug in splstr[1].split(',')]
1555             elif 'write' == splstr[0]:
1556                 write = [ug.strip() for ug in splstr[1].split(',')]
1557             else:
1558                 msg = 'Usage:\tread=<groups,users> write=<groups,users>'
1559                 raiseCLIError(None, msg)
1560         return (read, write)
1561
1562     @errors.generic.all
1563     @errors.pithos.connection
1564     @errors.pithos.container
1565     @errors.pithos.object_path
1566     def _run(self, read, write):
1567         self.client.set_object_sharing(
1568             self.path,
1569             read_permition=read,
1570             write_permition=write)
1571
1572     def main(self, container___path, *permissions):
1573         super(self.__class__, self)._run(
1574             container___path,
1575             path_is_optional=False)
1576         (read, write) = self.format_permition_dict(permissions)
1577         self._run(read, write)
1578
1579
1580 @command(pithos_cmds)
1581 class store_delpermissions(_store_container_command):
1582     """Delete all permissions set on object
1583     To modify permissions, use /store setpermssions
1584     """
1585
1586     @errors.generic.all
1587     @errors.pithos.connection
1588     @errors.pithos.container
1589     @errors.pithos.object_path
1590     def _run(self):
1591         self.client.del_object_sharing(self.path)
1592
1593     def main(self, container___path):
1594         super(self.__class__, self)._run(
1595             container___path,
1596             path_is_optional=False)
1597         self._run()
1598
1599
1600 @command(pithos_cmds)
1601 class store_info(_store_container_command):
1602     """Get detailed information for user account, containers or objects
1603     to get account info:    /store info
1604     to get container info:  /store info <container>
1605     to get object info:     /store info <container>:<path>
1606     """
1607
1608     arguments = dict(
1609         object_version=ValueArgument(
1610             'show specific version \ (applies only for objects)',
1611             ('-j', '--object-version'))
1612     )
1613
1614     @errors.generic.all
1615     @errors.pithos.connection
1616     @errors.pithos.container
1617     @errors.pithos.object_path
1618     def _run(self):
1619         if self.container is None:
1620             r = self.client.get_account_info()
1621         elif self.path is None:
1622             r = self.client.get_container_info(self.container)
1623         else:
1624             r = self.client.get_object_info(
1625                 self.path,
1626                 version=self['object_version'])
1627         print_dict(r)
1628
1629     def main(self, container____path__=None):
1630         super(self.__class__, self)._run(container____path__)
1631         self._run()
1632
1633
1634 @command(pithos_cmds)
1635 class store_meta(_store_container_command):
1636     """Get metadata for account, containers or objects"""
1637
1638     arguments = dict(
1639         detail=FlagArgument('show detailed output', ('-l', '--details')),
1640         until=DateArgument('show metadata until then', '--until'),
1641         object_version=ValueArgument(
1642             'show specific version \ (applies only for objects)',
1643             ('-j', '--object-version'))
1644     )
1645
1646     @errors.generic.all
1647     @errors.pithos.connection
1648     @errors.pithos.container
1649     @errors.pithos.object_path
1650     def _run(self):
1651         until = self['until']
1652         if self.container is None:
1653             if self['detail']:
1654                 r = self.client.get_account_info(until=until)
1655             else:
1656                 r = self.client.get_account_meta(until=until)
1657                 r = pretty_keys(r, '-')
1658             if r:
1659                 print(bold(self.client.account))
1660         elif self.path is None:
1661             if self['detail']:
1662                 r = self.client.get_container_info(until=until)
1663             else:
1664                 cmeta = self.client.get_container_meta(until=until)
1665                 ometa = self.client.get_container_object_meta(until=until)
1666                 r = {}
1667                 if cmeta:
1668                     r['container-meta'] = pretty_keys(cmeta, '-')
1669                 if ometa:
1670                     r['object-meta'] = pretty_keys(ometa, '-')
1671         else:
1672             if self['detail']:
1673                 r = self.client.get_object_info(
1674                     self.path,
1675                     version=self['object_version'])
1676             else:
1677                 r = self.client.get_object_meta(
1678                     self.path,
1679                     version=self['object_version'])
1680             if r:
1681                 r = pretty_keys(pretty_keys(r, '-'))
1682         if r:
1683             print_dict(r)
1684
1685     def main(self, container____path__=None):
1686         super(self.__class__, self)._run(container____path__)
1687         self._run()
1688
1689
1690 @command(pithos_cmds)
1691 class store_setmeta(_store_container_command):
1692     """Set a piece of metadata for account, container or object
1693     Metadata are formed as key:value pairs
1694     """
1695
1696     @errors.generic.all
1697     @errors.pithos.connection
1698     @errors.pithos.container
1699     @errors.pithos.object_path
1700     def _run(self, metakey, metaval):
1701         if not self.container:
1702             self.client.set_account_meta({metakey: metaval})
1703         elif not self.path:
1704             self.client.set_container_meta({metakey: metaval})
1705         else:
1706             self.client.set_object_meta(self.path, {metakey: metaval})
1707
1708     def main(self, metakey, metaval, container____path__=None):
1709         super(self.__class__, self)._run(container____path__)
1710         self._run(metakey=metakey, metaval=metaval)
1711
1712
1713 @command(pithos_cmds)
1714 class store_delmeta(_store_container_command):
1715     """Delete metadata with given key from account, container or object
1716     Metadata are formed as key:value objects
1717     - to get metadata of current account:     /store meta
1718     - to get metadata of a container:         /store meta <container>
1719     - to get metadata of an object:           /store meta <container>:<path>
1720     """
1721
1722     @errors.generic.all
1723     @errors.pithos.connection
1724     @errors.pithos.container
1725     @errors.pithos.object_path
1726     def _run(self, metakey):
1727         if self.container is None:
1728             self.client.del_account_meta(metakey)
1729         elif self.path is None:
1730             self.client.del_container_meta(metakey)
1731         else:
1732             self.client.del_object_meta(self.path, metakey)
1733
1734     def main(self, metakey, container____path__=None):
1735         super(self.__class__, self)._run(container____path__)
1736         self._run(metakey)
1737
1738
1739 @command(pithos_cmds)
1740 class store_quota(_store_account_command):
1741     """Get quota for account or container"""
1742
1743     arguments = dict(
1744         in_bytes=FlagArgument('Show result in bytes', ('-b', '--bytes'))
1745     )
1746
1747     @errors.generic.all
1748     @errors.pithos.connection
1749     @errors.pithos.container
1750     def _run(self):
1751         if self.container:
1752             reply = self.client.get_container_quota(self.container)
1753         else:
1754             reply = self.client.get_account_quota()
1755         if not self['in_bytes']:
1756             for k in reply:
1757                 reply[k] = format_size(reply[k])
1758         print_dict(pretty_keys(reply, '-'))
1759
1760     def main(self, container=None):
1761         super(self.__class__, self)._run()
1762         self.container = container
1763         self._run()
1764
1765
1766 @command(pithos_cmds)
1767 class store_setquota(_store_account_command):
1768     """Set new quota for account or container
1769     By default, quota is set in bytes
1770     Users may specify a different unit, e.g:
1771     /store setquota 2.3GB mycontainer
1772     Accepted units: B, KiB (1024 B), KB (1000 B), MiB, MB, GiB, GB, TiB, TB
1773     """
1774
1775     @errors.generic.all
1776     def _calculate_quota(self, user_input):
1777         quota = 0
1778         try:
1779             quota = int(user_input)
1780         except ValueError:
1781             index = 0
1782             digits = [str(num) for num in range(0, 10)] + ['.']
1783             while user_input[index] in digits:
1784                 index += 1
1785             quota = user_input[:index]
1786             format = user_input[index:]
1787             try:
1788                 return to_bytes(quota, format)
1789             except Exception as qe:
1790                 msg = 'Failed to convert %s to bytes' % user_input,
1791                 raiseCLIError(qe, msg, details=[
1792                     'Syntax: setquota <quota>[format] [container]',
1793                     'e.g.: setquota 2.3GB mycontainer',
1794                     'Acceptable formats:',
1795                     '(*1024): B, KiB, MiB, GiB, TiB',
1796                     '(*1000): B, KB, MB, GB, TB'])
1797         return quota
1798
1799     @errors.generic.all
1800     @errors.pithos.connection
1801     @errors.pithos.container
1802     def _run(self, quota):
1803         if self.container:
1804             self.client.container = self.container
1805             self.client.set_container_quota(quota)
1806         else:
1807             self.client.set_account_quota(quota)
1808
1809     def main(self, quota, container=None):
1810         super(self.__class__, self)._run()
1811         quota = self._calculate_quota(quota)
1812         self.container = container
1813         self._run(quota)
1814
1815
1816 @command(pithos_cmds)
1817 class store_versioning(_store_account_command):
1818     """Get  versioning for account or container"""
1819
1820     @errors.generic.all
1821     @errors.pithos.connection
1822     @errors.pithos.container
1823     def _run(self):
1824         if self.container:
1825             r = self.client.get_container_versioning(self.container)
1826         else:
1827             r = self.client.get_account_versioning()
1828         print_dict(r)
1829
1830     def main(self, container=None):
1831         super(self.__class__, self)._run()
1832         self.container = container
1833         self._run()
1834
1835
1836 @command(pithos_cmds)
1837 class store_setversioning(_store_account_command):
1838     """Set versioning mode (auto, none) for account or container"""
1839
1840     def _check_versioning(self, versioning):
1841         if versioning and versioning.lower() in ('auto', 'none'):
1842             return versioning.lower()
1843         raiseCLIError('Invalid versioning %s' % versioning, details=[
1844             'Versioning can be auto or none'])
1845
1846     @errors.generic.all
1847     @errors.pithos.connection
1848     @errors.pithos.container
1849     def _run(self, versioning):
1850         if self.container:
1851             self.client.container = self.container
1852             self.client.set_container_versioning(versioning)
1853         else:
1854             self.client.set_account_versioning(versioning)
1855
1856     def main(self, versioning, container=None):
1857         super(self.__class__, self)._run()
1858         self._run(self._check_versioning(versioning))
1859
1860
1861 @command(pithos_cmds)
1862 class store_group(_store_account_command):
1863     """Get groups and group members"""
1864
1865     @errors.generic.all
1866     @errors.pithos.connection
1867     def _run(self):
1868         r = self.client.get_account_group()
1869         print_dict(pretty_keys(r, '-'))
1870
1871     def main(self):
1872         super(self.__class__, self)._run()
1873         self._run()
1874
1875
1876 @command(pithos_cmds)
1877 class store_setgroup(_store_account_command):
1878     """Set a user group"""
1879
1880     @errors.generic.all
1881     @errors.pithos.connection
1882     def _run(self, groupname, *users):
1883         self.client.set_account_group(groupname, users)
1884
1885     def main(self, groupname, *users):
1886         super(self.__class__, self)._run()
1887         if users:
1888             self._run(groupname, *users)
1889         else:
1890             raiseCLIError('No users to add in group %s' % groupname)
1891
1892
1893 @command(pithos_cmds)
1894 class store_delgroup(_store_account_command):
1895     """Delete a user group"""
1896
1897     @errors.generic.all
1898     @errors.pithos.connection
1899     def _run(self, groupname):
1900         self.client.del_account_group(groupname)
1901
1902     def main(self, groupname):
1903         super(self.__class__, self)._run()
1904         self._run(groupname)
1905
1906
1907 @command(pithos_cmds)
1908 class store_sharers(_store_account_command):
1909     """List the accounts that share objects with current user"""
1910
1911     arguments = dict(
1912         detail=FlagArgument('show detailed output', ('-l', '--details')),
1913         marker=ValueArgument('show output greater then marker', '--marker')
1914     )
1915
1916     @errors.generic.all
1917     @errors.pithos.connection
1918     def _run(self):
1919         accounts = self.client.get_sharing_accounts(marker=self['marker'])
1920         if self['detail']:
1921             print_items(accounts)
1922         else:
1923             print_items([acc['name'] for acc in accounts])
1924
1925     def main(self):
1926         super(self.__class__, self)._run()
1927         self._run()
1928
1929
1930 @command(pithos_cmds)
1931 class store_versions(_store_container_command):
1932     """Get the list of object versions
1933     Deleted objects may still have versions that can be used to restore it and
1934     get information about its previous state.
1935     The version number can be used in a number of other commands, like info,
1936     copy, move, meta. See these commands for more information, e.g.
1937     /store info -h
1938     """
1939
1940     @errors.generic.all
1941     @errors.pithos.connection
1942     @errors.pithos.container
1943     @errors.pithos.object_path
1944     def _run(self):
1945         versions = self.client.get_object_versionlist(self.path)
1946         print_items([dict(id=vitem[0], created=strftime(
1947             '%d-%m-%Y %H:%M:%S',
1948             localtime(float(vitem[1])))) for vitem in versions])
1949
1950     def main(self, container___path):
1951         super(store_versions, self)._run(
1952             container___path,
1953             path_is_optional=False)
1954         self._run()