Statistics
| Branch: | Tag: | Revision:

root / kamaki / cli / commands / pithos.py @ cfba9587

History | View | Annotate | Download (68.2 kB)

1
# Copyright 2011-2013 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.command
33

    
34
from time import localtime, strftime
35
from io import StringIO
36
from pydoc import pager
37
from os import path, walk, makedirs
38

    
39
from kamaki.clients.pithos import PithosClient, ClientError
40

    
41
from kamaki.cli import command
42
from kamaki.cli.command_tree import CommandTree
43
from kamaki.cli.commands import (
44
    _command_init, errors, addLogSettings, DontRaiseKeyError, _optional_json,
45
    _name_filter, _optional_output_cmd)
46
from kamaki.cli.errors import (
47
    CLIBaseUrlError, CLIError, CLIInvalidArgument, raiseCLIError,
48
    CLISyntaxError)
49
from kamaki.cli.argument import (
50
    FlagArgument, IntArgument, ValueArgument, DateArgument, KeyValueArgument,
51
    ProgressBarArgument, RepeatableArgument, DataSizeArgument)
52
from kamaki.cli.utils import (
53
    format_size, bold, get_path_size, guess_mime_type)
54

    
55
file_cmds = CommandTree('file', 'Pithos+/Storage object level API commands')
56
container_cmds = CommandTree(
57
    'container', 'Pithos+/Storage container level API commands')
58
sharer_cmds = CommandTree('sharer', 'Pithos+/Storage sharers')
59
group_cmds = CommandTree('group', 'Pithos+/Storage user groups')
60
_commands = [file_cmds, container_cmds, sharer_cmds, group_cmds]
61

    
62

    
63
class _pithos_init(_command_init):
64
    """Initilize a pithos+ client
65
    There is always a default account (current user uuid)
66
    There is always a default container (pithos)
67
    """
68

    
69
    @DontRaiseKeyError
70
    def _custom_container(self):
71
        return self.config.get_cloud(self.cloud, 'pithos_container')
72

    
73
    @DontRaiseKeyError
74
    def _custom_uuid(self):
75
        return self.config.get_cloud(self.cloud, 'pithos_uuid')
76

    
77
    def _set_account(self):
78
        self.account = self._custom_uuid()
79
        if self.account:
80
            return
81
        astakos = getattr(self, 'auth_base', None)
82
        if astakos:
83
            self.account = astakos.user_term('id', self.token)
84
        else:
85
            raise CLIBaseUrlError(service='astakos')
86

    
87
    @errors.generic.all
88
    @addLogSettings
89
    def _run(self):
90
        cloud = getattr(self, 'cloud', None)
91
        if cloud:
92
            self.base_url = self._custom_url('pithos')
93
        else:
94
            self.cloud = 'default'
95
        self.token = self._custom_token('pithos')
96
        self.container = self._custom_container() or 'pithos'
97

    
98
        astakos = getattr(self, 'auth_base', None)
99
        if astakos:
100
            self.token = self.token or astakos.token
101
            if not self.base_url:
102
                pithos_endpoints = astakos.get_service_endpoints(
103
                    self._custom_type('pithos') or 'object-store',
104
                    self._custom_version('pithos') or '')
105
                self.base_url = pithos_endpoints['publicURL']
106
        else:
107
            raise CLIBaseUrlError(service='astakos')
108

    
109
        self._set_account()
110
        self.client = PithosClient(
111
            self.base_url, self.token, self.account, self.container)
112

    
113
    def main(self):
114
        self._run()
115

    
116

    
117
class _pithos_account(_pithos_init):
118
    """Setup account"""
119

    
120
    def __init__(self, arguments={}, auth_base=None, cloud=None):
121
        super(_pithos_account, self).__init__(arguments, auth_base, cloud)
122
        self['account'] = ValueArgument(
123
            'Use (a different) user uuid', ('-A', '--account'))
124

    
125
    def print_objects(self, object_list):
126
        for index, obj in enumerate(object_list):
127
            pretty_obj = obj.copy()
128
            index += 1
129
            empty_space = ' ' * (len(str(len(object_list))) - len(str(index)))
130
            if 'subdir' in obj:
131
                continue
132
            if self._is_dir(obj):
133
                size = 'D'
134
            else:
135
                size = format_size(obj['bytes'])
136
                pretty_obj['bytes'] = '%s (%s)' % (obj['bytes'], size)
137
            oname = obj['name'] if self['more'] else bold(obj['name'])
138
            prfx = ('%s%s. ' % (empty_space, index)) if self['enum'] else ''
139
            if self['detail']:
140
                self.writeln('%s%s' % (prfx, oname))
141
                self.print_dict(pretty_obj, exclude=('name'))
142
                self.writeln()
143
            else:
144
                oname = '%s%9s %s' % (prfx, size, oname)
145
                oname += '/' if self._is_dir(obj) else u''
146
                self.writeln(oname)
147

    
148
    @staticmethod
149
    def _is_dir(remote_dict):
150
        return 'application/directory' == remote_dict.get(
151
            'content_type', remote_dict.get('content-type', ''))
152

    
153
    def _run(self):
154
        super(_pithos_account, self)._run()
155
        self.client.account = self['account'] or getattr(
156
            self, 'account', getattr(self.client, 'account', None))
157

    
158

    
159
class _pithos_container(_pithos_account):
160
    """Setup container"""
161

    
162
    def __init__(self, arguments={}, auth_base=None, cloud=None):
163
        super(_pithos_container, self).__init__(arguments, auth_base, cloud)
164
        self['container'] = ValueArgument(
165
            'Use this container (default: pithos)', ('-C', '--container'))
166

    
167
    @staticmethod
168
    def _resolve_pithos_url(url):
169
        """Match urls of one of the following formats:
170
        pithos://ACCOUNT/CONTAINER/OBJECT_PATH
171
        /CONTAINER/OBJECT_PATH
172
        return account, container, path
173
        """
174
        account, container, obj_path, prefix = '', '', url, 'pithos://'
175
        if url.startswith(prefix):
176
            account, sep, url = url[len(prefix):].partition('/')
177
            url = '/%s' % url
178
        if url.startswith('/'):
179
            container, sep, obj_path = url[1:].partition('/')
180
        return account, container, obj_path
181

    
182
    def _run(self, url=None):
183
        acc, con, self.path = self._resolve_pithos_url(url or '')
184
        #  self.account = acc or getattr(self, 'account', '')
185
        super(_pithos_container, self)._run()
186
        self.container = con or self['container'] or getattr(
187
            self, 'container', None) or getattr(self.client, 'container', '')
188
        self.client.account = acc or self.client.account
189
        self.client.container = self.container
190

    
191

    
192
@command(file_cmds)
193
class file_info(_pithos_container, _optional_json):
194
    """Get information/details about a file"""
195

    
196
    arguments = dict(
197
        object_version=ValueArgument(
198
            'download a file of a specific version', '--object-version'),
199
        hashmap=FlagArgument(
200
            'Get file hashmap instead of details', '--hashmap'),
201
        matching_etag=ValueArgument(
202
            'show output if ETags match', '--if-match'),
203
        non_matching_etag=ValueArgument(
204
            'show output if ETags DO NOT match', '--if-none-match'),
205
        modified_since_date=DateArgument(
206
            'show output modified since then', '--if-modified-since'),
207
        unmodified_since_date=DateArgument(
208
            'show output unmodified since then', '--if-unmodified-since'),
209
        sharing=FlagArgument(
210
            'show object permissions and sharing information', '--sharing'),
211
        metadata=FlagArgument('show only object metadata', '--metadata'),
212
        versions=FlagArgument(
213
            'show the list of versions for the file', '--object-versions')
214
    )
215

    
216
    def version_print(self, versions):
217
        return {'/%s/%s' % (self.container, self.path): [
218
            dict(version_id=vitem[0], created=strftime(
219
                '%d-%m-%Y %H:%M:%S',
220
                localtime(float(vitem[1])))) for vitem in versions]}
221

    
222
    @errors.generic.all
223
    @errors.pithos.connection
224
    @errors.pithos.container
225
    @errors.pithos.object_path
226
    def _run(self):
227
        if self['hashmap']:
228
            r = self.client.get_object_hashmap(
229
                self.path,
230
                version=self['object_version'],
231
                if_match=self['matching_etag'],
232
                if_none_match=self['non_matching_etag'],
233
                if_modified_since=self['modified_since_date'],
234
                if_unmodified_since=self['unmodified_since_date'])
235
        elif self['sharing']:
236
            r = self.client.get_object_sharing(self.path)
237
            r['public url'] = self.client.get_object_info(
238
                self.path, version=self['object_version']).get(
239
                    'x-object-public', None)
240
        elif self['metadata']:
241
            r, preflen = dict(), len('x-object-meta-')
242
            for k, v in self.client.get_object_meta(self.path).items():
243
                r[k[preflen:]] = v
244
        elif self['versions']:
245
            r = self.version_print(
246
                self.client.get_object_versionlist(self.path))
247
        else:
248
            r = self.client.get_object_info(
249
                self.path, version=self['object_version'])
250
        self._print(r, self.print_dict)
251

    
252
    def main(self, path_or_url):
253
        super(self.__class__, self)._run(path_or_url)
254
        self._run()
255

    
256

    
257
@command(file_cmds)
258
class file_list(_pithos_container, _optional_json, _name_filter):
259
    """List all objects in a container or a directory object"""
260

    
261
    arguments = dict(
262
        detail=FlagArgument('detailed output', ('-l', '--list')),
263
        limit=IntArgument('limit number of listed items', ('-n', '--number')),
264
        marker=ValueArgument('output greater that marker', '--marker'),
265
        delimiter=ValueArgument('show output up to delimiter', '--delimiter'),
266
        meta=ValueArgument(
267
            'show output with specified meta keys', '--meta',
268
            default=[]),
269
        if_modified_since=ValueArgument(
270
            'show output modified since then', '--if-modified-since'),
271
        if_unmodified_since=ValueArgument(
272
            'show output not modified since then', '--if-unmodified-since'),
273
        until=DateArgument('show metadata until then', '--until'),
274
        format=ValueArgument(
275
            'format to parse until data (default: d/m/Y H:M:S )', '--format'),
276
        shared_by_me=FlagArgument(
277
            'show only files shared to other users', '--shared-by-me'),
278
        public=FlagArgument('show only published objects', '--public'),
279
        more=FlagArgument('read long results', '--more'),
280
        enum=FlagArgument('Enumerate results', '--enumerate'),
281
        recursive=FlagArgument(
282
            'Recursively list containers and their contents',
283
            ('-R', '--recursive'))
284
    )
285

    
286
    @errors.generic.all
287
    @errors.pithos.connection
288
    @errors.pithos.container
289
    @errors.pithos.object_path
290
    def _run(self):
291
        r = self.client.container_get(
292
            limit=False if self['more'] else self['limit'],
293
            marker=self['marker'],
294
            prefix=self['name_pref'],
295
            delimiter=self['delimiter'],
296
            path=self.path or '',
297
            show_only_shared=self['shared_by_me'],
298
            public=self['public'],
299
            if_modified_since=self['if_modified_since'],
300
            if_unmodified_since=self['if_unmodified_since'],
301
            until=self['until'],
302
            meta=self['meta'])
303
        files = self._filter_by_name(r.json)
304
        if self['more']:
305
            outbu, self._out = self._out, StringIO()
306
        try:
307
            if self['json_output'] or self['output_format']:
308
                self._print(files)
309
            else:
310
                self.print_objects(files)
311
        finally:
312
            if self['more']:
313
                pager(self._out.getvalue())
314
                self._out = outbu
315

    
316
    def main(self, path_or_url=''):
317
        super(self.__class__, self)._run(path_or_url)
318
        self._run()
319

    
320

    
321
@command(file_cmds)
322
class file_modify(_pithos_container):
323
    """Modify the attributes of a file or directory object"""
324

    
325
    arguments = dict(
326
        publish=FlagArgument(
327
            'Make an object public (returns the public URL)', '--publish'),
328
        unpublish=FlagArgument(
329
            'Make an object unpublic', '--unpublish'),
330
        uuid_for_read_permission=RepeatableArgument(
331
            'Give read access to user/group (can be repeated, accumulative). '
332
            'Format for users: UUID . Format for groups: UUID:GROUP . '
333
            'Use * for all users/groups', '--read-permission'),
334
        uuid_for_write_permission=RepeatableArgument(
335
            'Give write access to user/group (can be repeated, accumulative). '
336
            'Format for users: UUID . Format for groups: UUID:GROUP . '
337
            'Use * for all users/groups', '--write-permission'),
338
        no_permissions=FlagArgument('Remove permissions', '--no-permissions'),
339
        metadata_to_set=KeyValueArgument(
340
            'Add metadata (KEY=VALUE) to an object (can be repeated)',
341
            '--metadata-add'),
342
        metadata_key_to_delete=RepeatableArgument(
343
            'Delete object metadata (can be repeated)', '--metadata-del'),
344
    )
345
    required = [
346
        'publish', 'unpublish', 'uuid_for_read_permission', 'metadata_to_set',
347
        'uuid_for_write_permission', 'no_permissions',
348
        'metadata_key_to_delete']
349

    
350
    @errors.generic.all
351
    @errors.pithos.connection
352
    @errors.pithos.container
353
    @errors.pithos.object_path
354
    def _run(self):
355
        if self['publish']:
356
            self.writeln(self.client.publish_object(self.path))
357
        if self['unpublish']:
358
            self.client.unpublish_object(self.path)
359
        if self['uuid_for_read_permission'] or self[
360
                'uuid_for_write_permission']:
361
            perms = self.client.get_object_sharing(self.path)
362
            read, write = perms.get('read', ''), perms.get('write', '')
363
            read = read.split(',') if read else []
364
            write = write.split(',') if write else []
365
            read += (self['uuid_for_read_permission'] or [])
366
            write += (self['uuid_for_write_permission'] or [])
367
            self.client.set_object_sharing(
368
                self.path, read_permission=read, write_permission=write)
369
            self.print_dict(self.client.get_object_sharing(self.path))
370
        if self['no_permissions']:
371
            self.client.del_object_sharing(self.path)
372
        metadata = self['metadata_to_set'] or dict()
373
        for k in (self['metadata_key_to_delete'] or []):
374
            metadata[k] = ''
375
        if metadata:
376
            self.client.set_object_meta(self.path, metadata)
377
            self.print_dict(self.client.get_object_meta(self.path))
378

    
379
    def main(self, path_or_url):
380
        super(self.__class__, self)._run(path_or_url)
381
        if self['publish'] and self['unpublish']:
382
            raise CLIInvalidArgument(
383
                'Arguments %s and %s cannot be used together' % (
384
                    self.arguments['publish'].lvalue,
385
                    self.arguments['publish'].lvalue))
386
        if self['no_permissions'] and (
387
                self['uuid_for_read_permission'] or self[
388
                    'uuid_for_write_permission']):
389
            raise CLIInvalidArgument(
390
                '%s cannot be used with other permission arguments' % (
391
                    self.arguments['no_permissions'].lvalue))
392
        self._run()
393

    
394

    
395
@command(file_cmds)
396
class file_create(_pithos_container, _optional_output_cmd):
397
    """Create an empty file"""
398

    
399
    arguments = dict(
400
        content_type=ValueArgument(
401
            'Set content type (default: application/octet-stream)',
402
            '--content-type',
403
            default='application/octet-stream')
404
    )
405

    
406
    @errors.generic.all
407
    @errors.pithos.connection
408
    @errors.pithos.container
409
    def _run(self):
410
        self._optional_output(
411
            self.client.create_object(self.path, self['content_type']))
412

    
413
    def main(self, path_or_url):
414
        super(self.__class__, self)._run(path_or_url)
415
        self._run()
416

    
417

    
418
@command(file_cmds)
419
class file_mkdir(_pithos_container, _optional_output_cmd):
420
    """Create a directory: /file create --content-type='applcation/directory'
421
    """
422

    
423
    @errors.generic.all
424
    @errors.pithos.connection
425
    @errors.pithos.container
426
    def _run(self):
427
        self._optional_output(self.client.create_directory(self.path))
428

    
429
    def main(self, path_or_url):
430
        super(self.__class__, self)._run(path_or_url)
431
        self._run()
432

    
433

    
434
@command(file_cmds)
435
class file_delete(_pithos_container):
436
    """Delete a file or directory object"""
437

    
438
    arguments = dict(
439
        until_date=DateArgument('remove history until then', '--until'),
440
        yes=FlagArgument('Do not prompt for permission', '--yes'),
441
        recursive=FlagArgument(
442
            'If a directory, empty first', ('-r', '--recursive')),
443
        delimiter=ValueArgument(
444
            'delete objects prefixed with <object><delimiter>', '--delimiter')
445
    )
446

    
447
    @errors.generic.all
448
    @errors.pithos.connection
449
    @errors.pithos.container
450
    @errors.pithos.object_path
451
    def _run(self):
452
        if self.path:
453
            if self['yes'] or self.ask_user(
454
                    'Delete /%s/%s ?' % (self.container, self.path)):
455
                self.client.del_object(
456
                    self.path,
457
                    until=self['until_date'],
458
                    delimiter='/' if self['recursive'] else self['delimiter'])
459
            else:
460
                self.error('Aborted')
461
        else:
462
            if self['yes'] or self.ask_user(
463
                    'Empty container /%s ?' % self.container):
464
                self.client.container_delete(self.container, delimiter='/')
465
            else:
466
                self.error('Aborted')
467

    
468
    def main(self, path_or_url):
469
        super(self.__class__, self)._run(path_or_url)
470
        self._run()
471

    
472

    
473
class _source_destination(_pithos_container, _optional_output_cmd):
474

    
475
    sd_arguments = dict(
476
        destination_user_uuid=ValueArgument(
477
            'default: current user uuid', '--to-account'),
478
        destination_container=ValueArgument(
479
            'default: pithos', '--to-container'),
480
        source_prefix=FlagArgument(
481
            'Transfer all files that are prefixed with SOURCE PATH If the '
482
            'destination path is specified, replace SOURCE_PATH with '
483
            'DESTINATION_PATH',
484
            ('-r', '--recursive')),
485
        force=FlagArgument(
486
            'Overwrite destination objects, if needed', ('-f', '--force')),
487
        source_version=ValueArgument(
488
            'The version of the source object', '--source-version')
489
    )
490

    
491
    def __init__(self, arguments={}, auth_base=None, cloud=None):
492
        self.arguments.update(arguments)
493
        self.arguments.update(self.sd_arguments)
494
        super(_source_destination, self).__init__(
495
            self.arguments, auth_base, cloud)
496

    
497
    def _report_transfer(self, src, dst, transfer_name):
498
        if not dst:
499
            if transfer_name in ('move', ):
500
                self.error('  delete source directory %s' % src)
501
            return
502
        dst_prf = '' if self.account == self.dst_client.account else (
503
                'pithos://%s' % self.dst_client.account)
504
        if src:
505
            src_prf = '' if self.account == self.dst_client.account else (
506
                    'pithos://%s' % self.account)
507
            self.error('  %s %s/%s/%s\n  -->  %s/%s/%s' % (
508
                transfer_name,
509
                src_prf, self.container, src,
510
                dst_prf, self.dst_client.container, dst))
511
        else:
512
            self.error('  mkdir %s/%s/%s' % (
513
                dst_prf, self.dst_client.container, dst))
514

    
515
    @errors.generic.all
516
    @errors.pithos.account
517
    def _src_dst(self, version=None):
518
        """Preconditions:
519
        self.account, self.container, self.path
520
        self.dst_acc, self.dst_con, self.dst_path
521
        They should all be configured properly
522
        :returns: [(src_path, dst_path), ...], if src_path is None, create
523
            destination directory
524
        """
525
        src_objects, dst_objects, pairs = dict(), dict(), []
526
        try:
527
            for obj in self.dst_client.list_objects(
528
                    prefix=self.dst_path or self.path or '/'):
529
                dst_objects[obj['name']] = obj
530
        except ClientError as ce:
531
            if ce.status in (404, ):
532
                raise CLIError(
533
                    'Destination container pithos://%s/%s not found' % (
534
                        self.dst_client.account, self.dst_client.container))
535
            raise ce
536
        if self['source_prefix']:
537
            #  Copy and replace prefixes
538
            for src_obj in self.client.list_objects(prefix=self.path):
539
                src_objects[src_obj['name']] = src_obj
540
            for src_path, src_obj in src_objects.items():
541
                dst_path = '%s%s' % (
542
                    self.dst_path or self.path, src_path[len(self.path):])
543
                dst_obj = dst_objects.get(dst_path, None)
544
                if self['force'] or not dst_obj:
545
                    #  Just do it
546
                    pairs.append((
547
                        None if self._is_dir(src_obj) else src_path, dst_path))
548
                    if self._is_dir(src_obj):
549
                        pairs.append((self.path or dst_path, None))
550
                elif not (self._is_dir(dst_obj) and self._is_dir(src_obj)):
551
                    raise CLIError(
552
                        'Destination object exists', importance=2, details=[
553
                            'Failed while transfering:',
554
                            '    pithos://%s/%s/%s' % (
555
                                    self.account,
556
                                    self.container,
557
                                    src_path),
558
                            '--> pithos://%s/%s/%s' % (
559
                                    self.dst_client.account,
560
                                    self.dst_client.container,
561
                                    dst_path),
562
                            'Use %s to transfer overwrite' % (
563
                                    self.arguments['force'].lvalue)])
564
        else:
565
            #  One object transfer
566
            try:
567
                src_version_arg = self.arguments.get('source_version', None)
568
                src_obj = self.client.get_object_info(
569
                    self.path,
570
                    version=src_version_arg.value if src_version_arg else None)
571
            except ClientError as ce:
572
                if ce.status in (204, ):
573
                    raise CLIError(
574
                        'Missing specific path container %s' % self.container,
575
                        importance=2, details=[
576
                            'To transfer container contents %s' % (
577
                                self.arguments['source_prefix'].lvalue)])
578
                raise
579
            dst_path = self.dst_path or self.path
580
            dst_obj = dst_objects.get(dst_path or self.path, None)
581
            if self['force'] or not dst_obj:
582
                pairs.append(
583
                    (None if self._is_dir(src_obj) else self.path, dst_path))
584
                if self._is_dir(src_obj):
585
                    pairs.append((self.path or dst_path, None))
586
            elif self._is_dir(src_obj):
587
                raise CLIError(
588
                    'Cannot transfer an application/directory object',
589
                    importance=2, details=[
590
                        'The object pithos://%s/%s/%s is a directory' % (
591
                            self.account,
592
                            self.container,
593
                            self.path),
594
                        'To recursively copy a directory, use',
595
                        '  %s' % self.arguments['source_prefix'].lvalue,
596
                        'To create a file, use',
597
                        '  /file create  (general purpose)',
598
                        '  /file mkdir   (a directory object)'])
599
            else:
600
                raise CLIError(
601
                    'Destination object exists',
602
                    importance=2, details=[
603
                        'Failed while transfering:',
604
                        '    pithos://%s/%s/%s' % (
605
                                self.account,
606
                                self.container,
607
                                self.path),
608
                        '--> pithos://%s/%s/%s' % (
609
                                self.dst_client.account,
610
                                self.dst_client.container,
611
                                dst_path),
612
                        'Use %s to transfer overwrite' % (
613
                                self.arguments['force'].lvalue)])
614
        return pairs
615

    
616
    def _run(self, source_path_or_url, destination_path_or_url=''):
617
        super(_source_destination, self)._run(source_path_or_url)
618
        dst_acc, dst_con, dst_path = self._resolve_pithos_url(
619
            destination_path_or_url)
620
        self.dst_client = PithosClient(
621
            base_url=self.client.base_url, token=self.client.token,
622
            container=self[
623
                'destination_container'] or dst_con or self.client.container,
624
            account=self[
625
                'destination_user_uuid'] or dst_acc or self.account)
626
        self.dst_path = dst_path or self.path
627

    
628

    
629
@command(file_cmds)
630
class file_copy(_source_destination):
631
    """Copy objects, even between different accounts or containers"""
632

    
633
    arguments = dict(
634
        public=ValueArgument('publish new object', '--public'),
635
        content_type=ValueArgument(
636
            'change object\'s content type', '--content-type'),
637
        source_version=ValueArgument(
638
            'The version of the source object', '--object-version')
639
    )
640

    
641
    @errors.generic.all
642
    @errors.pithos.connection
643
    @errors.pithos.container
644
    @errors.pithos.account
645
    def _run(self):
646
        for src, dst in self._src_dst(self['source_version']):
647
            self._report_transfer(src, dst, 'copy')
648
            if src and dst:
649
                self.dst_client.copy_object(
650
                    src_container=self.client.container,
651
                    src_object=src,
652
                    dst_container=self.dst_client.container,
653
                    dst_object=dst,
654
                    source_account=self.client.account,
655
                    source_version=self['source_version'],
656
                    public=self['public'],
657
                    content_type=self['content_type'])
658
            elif dst:
659
                self.dst_client.create_directory(dst)
660

    
661
    def main(self, source_path_or_url, destination_path_or_url=None):
662
        super(file_copy, self)._run(
663
            source_path_or_url, destination_path_or_url or '')
664
        self._run()
665

    
666

    
667
@command(file_cmds)
668
class file_move(_source_destination):
669
    """Move objects, even between different accounts or containers"""
670

    
671
    arguments = dict(
672
        public=ValueArgument('publish new object', '--public'),
673
        content_type=ValueArgument(
674
            'change object\'s content type', '--content-type')
675
    )
676

    
677
    @errors.generic.all
678
    @errors.pithos.connection
679
    @errors.pithos.container
680
    @errors.pithos.account
681
    def _run(self):
682
        for src, dst in self._src_dst():
683
            self._report_transfer(src, dst, 'move')
684
            if src and dst:
685
                self.dst_client.move_object(
686
                    src_container=self.client.container,
687
                    src_object=src,
688
                    dst_container=self.dst_client.container,
689
                    dst_object=dst,
690
                    source_account=self.account,
691
                    public=self['public'],
692
                    content_type=self['content_type'])
693
            elif dst:
694
                self.dst_client.create_directory(dst)
695
            else:
696
                self.client.del_object(src)
697

    
698
    def main(self, source_path_or_url, destination_path_or_url=None):
699
        super(file_move, self)._run(
700
            source_path_or_url, destination_path_or_url or '')
701
        self._run()
702

    
703

    
704
@command(file_cmds)
705
class file_append(_pithos_container, _optional_output_cmd):
706
    """Append local file to (existing) remote object
707
    The remote object should exist.
708
    If the remote object is a directory, it is transformed into a file.
709
    In the later case, objects under the directory remain intact.
710
    """
711

    
712
    arguments = dict(
713
        progress_bar=ProgressBarArgument(
714
            'do not show progress bar', ('-N', '--no-progress-bar'),
715
            default=False),
716
        max_threads=IntArgument('default: 1', '--threads'),
717
    )
718

    
719
    @errors.generic.all
720
    @errors.pithos.connection
721
    @errors.pithos.container
722
    @errors.pithos.object_path
723
    def _run(self, local_path):
724
        if self['max_threads'] > 0:
725
            self.client.MAX_THREADS = int(self['max_threads'])
726
        (progress_bar, upload_cb) = self._safe_progress_bar('Appending')
727
        try:
728
            with open(local_path, 'rb') as f:
729
                self._optional_output(
730
                    self.client.append_object(self.path, f, upload_cb))
731
        finally:
732
            self._safe_progress_bar_finish(progress_bar)
733

    
734
    def main(self, local_path, remote_path_or_url):
735
        super(self.__class__, self)._run(remote_path_or_url)
736
        self._run(local_path)
737

    
738

    
739
@command(file_cmds)
740
class file_truncate(_pithos_container, _optional_output_cmd):
741
    """Truncate remote file up to size"""
742

    
743
    arguments = dict(
744
        size_in_bytes=IntArgument('Length of file after truncation', '--size')
745
    )
746
    required = ('size_in_bytes', )
747

    
748
    @errors.generic.all
749
    @errors.pithos.connection
750
    @errors.pithos.container
751
    @errors.pithos.object_path
752
    @errors.pithos.object_size
753
    def _run(self, size):
754
        self._optional_output(self.client.truncate_object(self.path, size))
755

    
756
    def main(self, path_or_url):
757
        super(self.__class__, self)._run(path_or_url)
758
        self._run(size=self['size_in_bytes'])
759

    
760

    
761
@command(file_cmds)
762
class file_overwrite(_pithos_container, _optional_output_cmd):
763
    """Overwrite part of a remote file"""
764

    
765
    arguments = dict(
766
        progress_bar=ProgressBarArgument(
767
            'do not show progress bar', ('-N', '--no-progress-bar'),
768
            default=False),
769
        start_position=IntArgument('File position in bytes', '--from'),
770
        end_position=IntArgument('File position in bytes', '--to')
771
    )
772
    required = ('start_position', 'end_position')
773

    
774
    @errors.generic.all
775
    @errors.pithos.connection
776
    @errors.pithos.container
777
    @errors.pithos.object_path
778
    @errors.pithos.object_size
779
    def _run(self, local_path, start, end):
780
        start, end = int(start), int(end)
781
        (progress_bar, upload_cb) = self._safe_progress_bar(
782
            'Overwrite %s bytes' % (end - start))
783
        try:
784
            with open(path.abspath(local_path), 'rb') as f:
785
                self._optional_output(self.client.overwrite_object(
786
                    obj=self.path,
787
                    start=start,
788
                    end=end,
789
                    source_file=f,
790
                    upload_cb=upload_cb))
791
        finally:
792
            self._safe_progress_bar_finish(progress_bar)
793

    
794
    def main(self, local_path, path_or_url):
795
        super(self.__class__, self)._run(path_or_url)
796
        self.path = self.path or path.basename(local_path)
797
        self._run(
798
            local_path=local_path,
799
            start=self['start_position'],
800
            end=self['end_position'])
801

    
802

    
803
@command(file_cmds)
804
class file_upload(_pithos_container, _optional_output_cmd):
805
    """Upload a file"""
806

    
807
    arguments = dict(
808
        max_threads=IntArgument('default: 5', '--threads'),
809
        content_encoding=ValueArgument(
810
            'set MIME content type', '--content-encoding'),
811
        content_disposition=ValueArgument(
812
            'specify objects presentation style', '--content-disposition'),
813
        content_type=ValueArgument('specify content type', '--content-type'),
814
        uuid_for_read_permission=RepeatableArgument(
815
            'Give read access to a user or group (can be repeated) '
816
            'Use * for all users',
817
            '--read-permission'),
818
        uuid_for_write_permission=RepeatableArgument(
819
            'Give write access to a user or group (can be repeated) '
820
            'Use * for all users',
821
            '--write-permission'),
822
        public=FlagArgument('make object publicly accessible', '--public'),
823
        progress_bar=ProgressBarArgument(
824
            'do not show progress bar',
825
            ('-N', '--no-progress-bar'),
826
            default=False),
827
        overwrite=FlagArgument('Force (over)write', ('-f', '--force')),
828
        recursive=FlagArgument(
829
            'Recursively upload directory *contents* + subdirectories',
830
            ('-r', '--recursive')),
831
        unchunked=FlagArgument(
832
            'Upload file as one block (not recommended)', '--unchunked'),
833
        md5_checksum=ValueArgument(
834
            'Confirm upload with a custom checksum (MD5)', '--etag'),
835
        use_hashes=FlagArgument(
836
            'Source file contains hashmap not data', '--source-is-hashmap'),
837
    )
838

    
839
    def _sharing(self):
840
        sharing = dict()
841
        readlist = self['uuid_for_read_permission']
842
        if readlist:
843
            sharing['read'] = self['uuid_for_read_permission']
844
        writelist = self['uuid_for_write_permission']
845
        if writelist:
846
            sharing['write'] = self['uuid_for_write_permission']
847
        return sharing or None
848

    
849
    def _check_container_limit(self, path):
850
        cl_dict = self.client.get_container_limit()
851
        container_limit = int(cl_dict['x-container-policy-quota'])
852
        r = self.client.container_get()
853
        used_bytes = sum(int(o['bytes']) for o in r.json)
854
        path_size = get_path_size(path)
855
        if container_limit and path_size > (container_limit - used_bytes):
856
            raise CLIError(
857
                'Container %s (limit(%s) - used(%s)) < (size(%s) of %s)' % (
858
                    self.client.container,
859
                    format_size(container_limit),
860
                    format_size(used_bytes),
861
                    format_size(path_size),
862
                    path),
863
                details=[
864
                    'Check accound limit: /file quota',
865
                    'Check container limit:',
866
                    '\t/file containerlimit get %s' % self.client.container,
867
                    'Increase container limit:',
868
                    '\t/file containerlimit set <new limit> %s' % (
869
                        self.client.container)])
870

    
871
    def _src_dst(self, local_path, remote_path, objlist=None):
872
        lpath = path.abspath(local_path)
873
        short_path = path.basename(path.abspath(local_path))
874
        rpath = remote_path or short_path
875
        if path.isdir(lpath):
876
            if not self['recursive']:
877
                raise CLIError('%s is a directory' % lpath, details=[
878
                    'Use %s to upload directories & contents' % (
879
                        self.arguments['recursive'].lvalue)])
880
            robj = self.client.container_get(path=rpath)
881
            if not self['overwrite']:
882
                if robj.json:
883
                    raise CLIError(
884
                        'Objects/files prefixed as %s already exist' % rpath,
885
                        details=['Existing objects:'] + ['\t/%s/\t%s' % (
886
                            o['name'],
887
                            o['content_type'][12:]) for o in robj.json] + [
888
                            'Use -f to add, overwrite or resume'])
889
                else:
890
                    try:
891
                        topobj = self.client.get_object_info(rpath)
892
                        if not self._is_dir(topobj):
893
                            raise CLIError(
894
                                'Object /%s/%s exists but not a directory' % (
895
                                    self.container, rpath),
896
                                details=['Use -f to overwrite'])
897
                    except ClientError as ce:
898
                        if ce.status not in (404, ):
899
                            raise
900
            self._check_container_limit(lpath)
901
            prev = ''
902
            for top, subdirs, files in walk(lpath):
903
                if top != prev:
904
                    prev = top
905
                    try:
906
                        rel_path = rpath + top.split(lpath)[1]
907
                    except IndexError:
908
                        rel_path = rpath
909
                    self.error('mkdir /%s/%s' % (
910
                        self.client.container, rel_path))
911
                    self.client.create_directory(rel_path)
912
                for f in files:
913
                    fpath = path.join(top, f)
914
                    if path.isfile(fpath):
915
                        rel_path = rel_path.replace(path.sep, '/')
916
                        pathfix = f.replace(path.sep, '/')
917
                        yield open(fpath, 'rb'), '%s/%s' % (rel_path, pathfix)
918
                    else:
919
                        self.error('%s is not a regular file' % fpath)
920
        else:
921
            if not path.isfile(lpath):
922
                raise CLIError(('%s is not a regular file' % lpath) if (
923
                    path.exists(lpath)) else '%s does not exist' % lpath)
924
            try:
925
                robj = self.client.get_object_info(rpath)
926
                if remote_path and self._is_dir(robj):
927
                    rpath += '/%s' % (short_path.replace(path.sep, '/'))
928
                    self.client.get_object_info(rpath)
929
                if not self['overwrite']:
930
                    raise CLIError(
931
                        'Object /%s/%s already exists' % (
932
                            self.container, rpath),
933
                        details=['use -f to overwrite / resume'])
934
            except ClientError as ce:
935
                if ce.status not in (404, ):
936
                    raise
937
            self._check_container_limit(lpath)
938
            yield open(lpath, 'rb'), rpath
939

    
940
    def _run(self, local_path, remote_path):
941
        self.client.MAX_THREADS = int(self['max_threads'] or 5)
942
        params = dict(
943
            content_encoding=self['content_encoding'],
944
            content_type=self['content_type'],
945
            content_disposition=self['content_disposition'],
946
            sharing=self._sharing(),
947
            public=self['public'])
948
        uploaded, container_info_cache = list, dict()
949
        rpref = 'pithos://%s' if self['account'] else ''
950
        for f, rpath in self._src_dst(local_path, remote_path):
951
            self.error('%s --> %s/%s/%s' % (
952
                f.name, rpref, self.client.container, rpath))
953
            if not (self['content_type'] and self['content_encoding']):
954
                ctype, cenc = guess_mime_type(f.name)
955
                params['content_type'] = self['content_type'] or ctype
956
                params['content_encoding'] = self['content_encoding'] or cenc
957
            if self['unchunked']:
958
                r = self.client.upload_object_unchunked(
959
                    rpath, f,
960
                    etag=self['md5_checksum'], withHashFile=self['use_hashes'],
961
                    **params)
962
                if self['with_output'] or self['json_output']:
963
                    r['name'] = '/%s/%s' % (self.client.container, rpath)
964
                    uploaded.append(r)
965
            else:
966
                try:
967
                    (progress_bar, upload_cb) = self._safe_progress_bar(
968
                        'Uploading %s' % f.name.split(path.sep)[-1])
969
                    if progress_bar:
970
                        hash_bar = progress_bar.clone()
971
                        hash_cb = hash_bar.get_generator(
972
                            'Calculating block hashes')
973
                    else:
974
                        hash_cb = None
975
                    r = self.client.upload_object(
976
                        rpath, f,
977
                        hash_cb=hash_cb,
978
                        upload_cb=upload_cb,
979
                        container_info_cache=container_info_cache,
980
                        **params)
981
                    if self['with_output'] or self['json_output']:
982
                        r['name'] = '/%s/%s' % (self.client.container, rpath)
983
                        uploaded.append(r)
984
                except Exception:
985
                    self._safe_progress_bar_finish(progress_bar)
986
                    raise
987
                finally:
988
                    self._safe_progress_bar_finish(progress_bar)
989
        self._optional_output(uploaded)
990
        self.error('Upload completed')
991

    
992
    def main(self, local_path, remote_path_or_url):
993
        super(self.__class__, self)._run(remote_path_or_url)
994
        remote_path = self.path or path.basename(path.abspath(local_path))
995
        self._run(local_path=local_path, remote_path=remote_path)
996

    
997

    
998
class RangeArgument(ValueArgument):
999
    """
1000
    :value type: string of the form <start>-<end> where <start> and <end> are
1001
        integers
1002
    :value returns: the input string, after type checking <start> and <end>
1003
    """
1004

    
1005
    @property
1006
    def value(self):
1007
        return getattr(self, '_value', self.default)
1008

    
1009
    @value.setter
1010
    def value(self, newvalues):
1011
        if newvalues:
1012
            self._value = getattr(self, '_value', self.default)
1013
            for newvalue in newvalues.split(','):
1014
                self._value = ('%s,' % self._value) if self._value else ''
1015
                start, sep, end = newvalue.partition('-')
1016
                if sep:
1017
                    if start:
1018
                        start, end = (int(start), int(end))
1019
                        if start > end:
1020
                            raise CLIInvalidArgument(
1021
                                'Invalid range %s' % newvalue, details=[
1022
                                'Valid range formats',
1023
                                '  START-END', '  UP_TO', '  -FROM',
1024
                                'where all values are integers',
1025
                                'OR a compination (csv), e.g.,',
1026
                                '  %s=5,10-20,-5' % self.lvalue])
1027
                        self._value += '%s-%s' % (start, end)
1028
                    else:
1029
                        self._value += '-%s' % int(end)
1030
                else:
1031
                    self._value += '%s' % int(start)
1032

    
1033

    
1034
@command(file_cmds)
1035
class file_cat(_pithos_container):
1036
    """Fetch remote file contents"""
1037

    
1038
    arguments = dict(
1039
        range=RangeArgument('show range of data e.g., 5,10-20,-5', '--range'),
1040
        if_match=ValueArgument('show output if ETags match', '--if-match'),
1041
        if_none_match=ValueArgument(
1042
            'show output if ETags match', '--if-none-match'),
1043
        if_modified_since=DateArgument(
1044
            'show output modified since then', '--if-modified-since'),
1045
        if_unmodified_since=DateArgument(
1046
            'show output unmodified since then', '--if-unmodified-since'),
1047
        object_version=ValueArgument(
1048
            'Get contents of the chosen version', '--object-version')
1049
    )
1050

    
1051
    @errors.generic.all
1052
    @errors.pithos.connection
1053
    @errors.pithos.container
1054
    @errors.pithos.object_path
1055
    def _run(self):
1056
        r = self.client.download_object(
1057
            self.path, self._out,
1058
            range_str=self['range'],
1059
            version=self['object_version'],
1060
            if_match=self['if_match'],
1061
            if_none_match=self['if_none_match'],
1062
            if_modified_since=self['if_modified_since'],
1063
            if_unmodified_since=self['if_unmodified_since'])
1064
        print r
1065

    
1066
    def main(self, path_or_url):
1067
        super(self.__class__, self)._run(path_or_url)
1068
        self._run()
1069

    
1070

    
1071
@command(file_cmds)
1072
class file_download(_pithos_container):
1073
    """Download a remove file or directory object to local file system"""
1074

    
1075
    arguments = dict(
1076
        resume=FlagArgument(
1077
            'Resume/Overwrite (attempt resume, else overwrite)',
1078
            ('-f', '--resume')),
1079
        range=RangeArgument(
1080
            'Download only that range of data e.g., 5,10-20,-5', '--range'),
1081
        matching_etag=ValueArgument('download iff ETag match', '--if-match'),
1082
        non_matching_etag=ValueArgument(
1083
            'download iff ETags DO NOT match', '--if-none-match'),
1084
        modified_since_date=DateArgument(
1085
            'download iff remote file is modified since then',
1086
            '--if-modified-since'),
1087
        unmodified_since_date=DateArgument(
1088
            'show output iff remote file is unmodified since then',
1089
            '--if-unmodified-since'),
1090
        object_version=ValueArgument(
1091
            'download a file of a specific version', '--object-version'),
1092
        max_threads=IntArgument('default: 5', '--threads'),
1093
        progress_bar=ProgressBarArgument(
1094
            'do not show progress bar', ('-N', '--no-progress-bar'),
1095
            default=False),
1096
        recursive=FlagArgument(
1097
            'Download a remote directory object and its contents',
1098
            ('-r', '--recursive'))
1099
        )
1100

    
1101
    def _src_dst(self, local_path):
1102
        """Create a list of (src, dst) where src is a remote location and dst
1103
        is an open file descriptor. Directories are denoted as (None, dirpath)
1104
        and they are pretended to other objects in a very strict order (shorter
1105
        to longer path)."""
1106
        ret = []
1107
        try:
1108
            if self.path:
1109
                obj = self.client.get_object_info(
1110
                    self.path, version=self['object_version'])
1111
                obj.setdefault('name', self.path.strip('/'))
1112
            else:
1113
                obj = None
1114
        except ClientError as ce:
1115
            if ce.status in (404, ):
1116
                raiseCLIError(ce, details=[
1117
                    'To download an object, it must exist either as a file or'
1118
                    ' as a directory.',
1119
                    'For example, to download everything under prefix/ the '
1120
                    'directory "prefix" must exist.',
1121
                    'To see if an remote object is actually there:',
1122
                    '  /file info [/CONTAINER/]OBJECT',
1123
                    'To create a directory object:',
1124
                    '  /file mkdir [/CONTAINER/]OBJECT'])
1125
            if ce.status in (204, ):
1126
                raise CLIError(
1127
                    'No file or directory objects to download',
1128
                    details=[
1129
                        'To download a container (e.g., %s):' % self.container,
1130
                        '  [kamaki] container download %s [LOCAL_PATH]' % (
1131
                            self.container)])
1132
            raise
1133
        rpath = self.path.strip('/')
1134
        if local_path and self.path and local_path.endswith('/'):
1135
            local_path = local_path[-1:]
1136

    
1137
        if (not obj) or self._is_dir(obj):
1138
            if self['recursive']:
1139
                if not (self.path or local_path.endswith('/')):
1140
                    #  Download the whole container
1141
                    local_path = '' if local_path in ('.', ) else local_path
1142
                    local_path = '%s/' % (local_path or self.container)
1143
                obj = obj or dict(
1144
                    name='', content_type='application/directory')
1145
                dirs, files = [obj, ], []
1146
                objects = self.client.container_get(
1147
                    path=self.path,
1148
                    if_modified_since=self['modified_since_date'],
1149
                    if_unmodified_since=self['unmodified_since_date'])
1150
                for o in objects.json:
1151
                    (dirs if self._is_dir(o) else files).append(o)
1152

    
1153
                #  Put the directories on top of the list
1154
                for dpath in sorted(['%s%s' % (
1155
                        local_path, d['name'][len(rpath):]) for d in dirs]):
1156
                    if path.exists(dpath):
1157
                        if path.isdir(dpath):
1158
                            continue
1159
                        raise CLIError(
1160
                            'Cannot replace local file %s with a directory '
1161
                            'of the same name' % dpath,
1162
                            details=[
1163
                                'Either remove the file or specify a'
1164
                                'different target location'])
1165
                    ret.append((None, dpath, None))
1166

    
1167
                #  Append the file objects
1168
                for opath in [o['name'] for o in files]:
1169
                    lpath = '%s%s' % (local_path, opath[len(rpath):])
1170
                    if self['resume']:
1171
                        fxists = path.exists(lpath)
1172
                        if fxists and path.isdir(lpath):
1173
                            raise CLIError(
1174
                                'Cannot change local dir %s info file' % (
1175
                                    lpath),
1176
                                details=[
1177
                                    'Either remove the file or specify a'
1178
                                    'different target location'])
1179
                        ret.append((opath, lpath, fxists))
1180
                    elif path.exists(lpath):
1181
                        raise CLIError(
1182
                            'Cannot overwrite %s' % lpath,
1183
                            details=['To overwrite/resume, use  %s' % (
1184
                                self.arguments['resume'].lvalue)])
1185
                    else:
1186
                        ret.append((opath, lpath, None))
1187
            elif self.path:
1188
                raise CLIError(
1189
                    'Remote object /%s/%s is a directory' % (
1190
                        self.container, local_path),
1191
                    details=['Use %s to download directories' % (
1192
                        self.arguments['recursive'].lvalue)])
1193
            else:
1194
                parsed_name = self.arguments['recursive'].lvalue
1195
                raise CLIError(
1196
                    'Cannot download container %s' % self.container,
1197
                    details=[
1198
                        'Use %s to download containers' % parsed_name,
1199
                        '  [kamaki] file download %s /%s [LOCAL_PATH]' % (
1200
                            parsed_name, self.container)])
1201
        else:
1202
            #  Remote object is just a file
1203
            if path.exists(local_path) and not self['resume']:
1204
                raise CLIError(
1205
                    'Cannot overwrite local file %s' % (lpath),
1206
                    details=['To overwrite/resume, use  %s' % (
1207
                        self.arguments['resume'].lvalue)])
1208
            ret.append((rpath, local_path, self['resume']))
1209
        for r, l, resume in ret:
1210
            if r:
1211
                with open(l, 'rwb+' if resume else 'wb+') as f:
1212
                    yield (r, f)
1213
            else:
1214
                yield (r, l)
1215

    
1216
    @errors.generic.all
1217
    @errors.pithos.connection
1218
    @errors.pithos.container
1219
    @errors.pithos.object_path
1220
    @errors.pithos.local_path
1221
    @errors.pithos.local_path_download
1222
    def _run(self, local_path):
1223
        self.client.MAX_THREADS = int(self['max_threads'] or 5)
1224
        progress_bar = None
1225
        try:
1226
            for rpath, output_file in self._src_dst(local_path):
1227
                if not rpath:
1228
                    self.error('Create local directory %s' % output_file)
1229
                    makedirs(output_file)
1230
                    continue
1231
                self.error('/%s/%s --> %s' % (
1232
                    self.container, rpath, output_file.name))
1233
                progress_bar, download_cb = self._safe_progress_bar(
1234
                    '  download')
1235
                self.client.download_object(
1236
                    rpath, output_file,
1237
                    download_cb=download_cb,
1238
                    range_str=self['range'],
1239
                    version=self['object_version'],
1240
                    if_match=self['matching_etag'],
1241
                    resume=self['resume'],
1242
                    if_none_match=self['non_matching_etag'],
1243
                    if_modified_since=self['modified_since_date'],
1244
                    if_unmodified_since=self['unmodified_since_date'])
1245
        except KeyboardInterrupt:
1246
            from threading import activeCount, enumerate as activethreads
1247
            timeout = 0.5
1248
            while activeCount() > 1:
1249
                self._out.write('\nCancel %s threads: ' % (activeCount() - 1))
1250
                self._out.flush()
1251
                for thread in activethreads():
1252
                    try:
1253
                        thread.join(timeout)
1254
                        self._out.write('.' if thread.isAlive() else '*')
1255
                    except RuntimeError:
1256
                        continue
1257
                    finally:
1258
                        self._out.flush()
1259
                        timeout += 0.1
1260
            self.error('\nDownload canceled by user')
1261
            if local_path is not None:
1262
                self.error('to resume, re-run with --resume')
1263
        except Exception:
1264
            self._safe_progress_bar_finish(progress_bar)
1265
            raise
1266
        finally:
1267
            self._safe_progress_bar_finish(progress_bar)
1268

    
1269
    def main(self, remote_path_or_url, local_path=None):
1270
        super(self.__class__, self)._run(remote_path_or_url)
1271
        local_path = local_path or self.path or '.'
1272
        self._run(local_path=local_path)
1273

    
1274

    
1275
@command(container_cmds)
1276
class container_info(_pithos_account, _optional_json):
1277
    """Get information about a container"""
1278

    
1279
    arguments = dict(
1280
        until_date=DateArgument('show metadata until then', '--until'),
1281
        metadata=FlagArgument('Show only container metadata', '--metadata'),
1282
        sizelimit=FlagArgument(
1283
            'Show the maximum size limit for container', '--size-limit'),
1284
        in_bytes=FlagArgument('Show size limit in bytes', ('-b', '--bytes'))
1285
    )
1286

    
1287
    @errors.generic.all
1288
    @errors.pithos.connection
1289
    @errors.pithos.container
1290
    @errors.pithos.object_path
1291
    def _run(self):
1292
        if self['metadata']:
1293
            r, preflen = dict(), len('x-container-meta-')
1294
            for k, v in self.client.get_container_meta(
1295
                    until=self['until_date']).items():
1296
                r[k[preflen:]] = v
1297
        elif self['sizelimit']:
1298
            r = self.client.get_container_limit(
1299
                self.container)['x-container-policy-quota']
1300
            r = {'size limit': 'unlimited' if r in ('0', ) else (
1301
                int(r) if self['in_bytes'] else format_size(r))}
1302
        else:
1303
            r = self.client.get_container_info(self.container)
1304
        self._print(r, self.print_dict)
1305

    
1306
    def main(self, container):
1307
        super(self.__class__, self)._run()
1308
        self.container, self.client.container = container, container
1309
        self._run()
1310

    
1311

    
1312
class VersioningArgument(ValueArgument):
1313

    
1314
    schemes = ('auto', 'none')
1315

    
1316
    @property
1317
    def value(self):
1318
        return getattr(self, '_value', None)
1319

    
1320
    @value.setter
1321
    def value(self, new_scheme):
1322
        if new_scheme:
1323
            new_scheme = new_scheme.lower()
1324
            if new_scheme not in self.schemes:
1325
                raise CLIInvalidArgument('Invalid versioning value', details=[
1326
                    'Valid versioning values are %s' % ', '.join(
1327
                        self.schemes)])
1328
            self._value = new_scheme
1329

    
1330

    
1331
@command(container_cmds)
1332
class container_modify(_pithos_account, _optional_json):
1333
    """Modify the properties of a container"""
1334

    
1335
    arguments = dict(
1336
        metadata_to_add=KeyValueArgument(
1337
            'Add metadata in the form KEY=VALUE (can be repeated)',
1338
            '--metadata-add'),
1339
        metadata_to_delete=RepeatableArgument(
1340
            'Delete metadata by KEY (can be repeated)', '--metadata-del'),
1341
        sizelimit=DataSizeArgument(
1342
            'Set max size limit (0 for unlimited, '
1343
            'use units B, KiB, KB, etc.)', '--size-limit'),
1344
        versioning=VersioningArgument(
1345
            'Set a versioning scheme (%s)' % ', '.join(
1346
                VersioningArgument.schemes), '--versioning')
1347
    )
1348
    required = [
1349
        'metadata_to_add', 'metadata_to_delete', 'sizelimit', 'versioning']
1350

    
1351
    @errors.generic.all
1352
    @errors.pithos.connection
1353
    @errors.pithos.container
1354
    def _run(self, container):
1355
        metadata = self['metadata_to_add']
1356
        for k in (self['metadata_to_delete'] or []):
1357
            metadata[k] = ''
1358
        if metadata:
1359
            self.client.set_container_meta(metadata)
1360
            self._print(self.client.get_container_meta(), self.print_dict)
1361
        if self['sizelimit'] is not None:
1362
            self.client.set_container_limit(self['sizelimit'])
1363
            r = self.client.get_container_limit()['x-container-policy-quota']
1364
            r = 'unlimited' if r in ('0', ) else format_size(r)
1365
            self.writeln('new size limit: %s' % r)
1366
        if self['versioning']:
1367
            self.client.set_container_versioning(self['versioning'])
1368
            self.writeln('new versioning scheme: %s' % (
1369
                self.client.get_container_versioning(self.container)[
1370
                    'x-container-policy-versioning']))
1371

    
1372
    def main(self, container):
1373
        super(self.__class__, self)._run()
1374
        self.client.container, self.container = container, container
1375
        self._run(container=container)
1376

    
1377

    
1378
@command(container_cmds)
1379
class container_list(_pithos_account, _optional_json, _name_filter):
1380
    """List all containers, or their contents"""
1381

    
1382
    arguments = dict(
1383
        detail=FlagArgument('Containers with details', ('-l', '--list')),
1384
        limit=IntArgument('limit number of listed items', ('-n', '--number')),
1385
        marker=ValueArgument('output greater that marker', '--marker'),
1386
        modified_since_date=ValueArgument(
1387
            'show output modified since then', '--if-modified-since'),
1388
        unmodified_since_date=ValueArgument(
1389
            'show output not modified since then', '--if-unmodified-since'),
1390
        until_date=DateArgument('show metadata until then', '--until'),
1391
        shared=FlagArgument('show only shared', '--shared'),
1392
        more=FlagArgument('read long results', '--more'),
1393
        enum=FlagArgument('Enumerate results', '--enumerate'),
1394
        recursive=FlagArgument(
1395
            'Recursively list containers and their contents',
1396
            ('-r', '--recursive')),
1397
        shared_by_me=FlagArgument(
1398
            'show only files shared to other users', '--shared-by-me'),
1399
        public=FlagArgument('show only published objects', '--public'),
1400
    )
1401

    
1402
    def print_containers(self, container_list):
1403
        for index, container in enumerate(container_list):
1404
            if 'bytes' in container:
1405
                size = format_size(container['bytes'])
1406
            prfx = ('%s. ' % (index + 1)) if self['enum'] else ''
1407
            _cname = container['name'] if (
1408
                self['more']) else bold(container['name'])
1409
            cname = u'%s%s' % (prfx, _cname)
1410
            if self['detail']:
1411
                self.writeln(cname)
1412
                pretty_c = container.copy()
1413
                if 'bytes' in container:
1414
                    pretty_c['bytes'] = '%s (%s)' % (container['bytes'], size)
1415
                self.print_dict(pretty_c, exclude=('name'))
1416
                self.writeln()
1417
            else:
1418
                if 'count' in container and 'bytes' in container:
1419
                    self.writeln('%s (%s, %s objects)' % (
1420
                        cname, size, container['count']))
1421
                else:
1422
                    self.writeln(cname)
1423
            objects = container.get('objects', [])
1424
            if objects:
1425
                self.print_objects(objects)
1426
                self.writeln('')
1427

    
1428
    def _create_object_forest(self, container_list):
1429
        try:
1430
            for container in container_list:
1431
                self.client.container = container['name']
1432
                objects = self.client.container_get(
1433
                    limit=False if self['more'] else self['limit'],
1434
                    if_modified_since=self['modified_since_date'],
1435
                    if_unmodified_since=self['unmodified_since_date'],
1436
                    until=self['until_date'],
1437
                    show_only_shared=self['shared_by_me'],
1438
                    public=self['public'])
1439
                container['objects'] = objects.json
1440
        finally:
1441
            self.client.container = None
1442

    
1443
    @errors.generic.all
1444
    @errors.pithos.connection
1445
    @errors.pithos.object_path
1446
    @errors.pithos.container
1447
    def _run(self, container):
1448
        if container:
1449
            r = self.client.container_get(
1450
                limit=False if self['more'] else self['limit'],
1451
                marker=self['marker'],
1452
                if_modified_since=self['modified_since_date'],
1453
                if_unmodified_since=self['unmodified_since_date'],
1454
                until=self['until_date'],
1455
                show_only_shared=self['shared_by_me'],
1456
                public=self['public'])
1457
        else:
1458
            r = self.client.account_get(
1459
                limit=False if self['more'] else self['limit'],
1460
                marker=self['marker'],
1461
                if_modified_since=self['modified_since_date'],
1462
                if_unmodified_since=self['unmodified_since_date'],
1463
                until=self['until_date'],
1464
                show_only_shared=self['shared_by_me'],
1465
                public=self['public'])
1466
        files = self._filter_by_name(r.json)
1467
        if self['recursive'] and not container:
1468
            self._create_object_forest(files)
1469
        if self['more']:
1470
            outbu, self._out = self._out, StringIO()
1471
        try:
1472
            if self['json_output'] or self['output_format']:
1473
                self._print(files)
1474
            else:
1475
                (self.print_objects if container else self.print_containers)(
1476
                    files)
1477
        finally:
1478
            if self['more']:
1479
                pager(self._out.getvalue())
1480
                self._out = outbu
1481

    
1482
    def main(self, container=None):
1483
        super(self.__class__, self)._run()
1484
        self.client.container, self.container = container, container
1485
        self._run(container)
1486

    
1487

    
1488
@command(container_cmds)
1489
class container_create(_pithos_account):
1490
    """Create a new container"""
1491

    
1492
    arguments = dict(
1493
        versioning=ValueArgument(
1494
            'set container versioning (auto/none)', '--versioning'),
1495
        limit=IntArgument('set default container limit', '--limit'),
1496
        meta=KeyValueArgument(
1497
            'set container metadata (can be repeated)', '--meta')
1498
    )
1499

    
1500
    @errors.generic.all
1501
    @errors.pithos.connection
1502
    @errors.pithos.container
1503
    def _run(self, container):
1504
        try:
1505
            self.client.create_container(
1506
                container=container,
1507
                sizelimit=self['limit'],
1508
                versioning=self['versioning'],
1509
                metadata=self['meta'],
1510
                success=(201, ))
1511
        except ClientError as ce:
1512
            if ce.status in (202, ):
1513
                raise CLIError(
1514
                    'Container %s alread exists' % container, details=[
1515
                    'Either delete %s or choose another name' % (container)])
1516
            raise
1517

    
1518
    def main(self, new_container):
1519
        super(self.__class__, self)._run()
1520
        self._run(container=new_container)
1521

    
1522

    
1523
@command(container_cmds)
1524
class container_delete(_pithos_account):
1525
    """Delete a container"""
1526

    
1527
    arguments = dict(
1528
        yes=FlagArgument('Do not prompt for permission', '--yes'),
1529
        recursive=FlagArgument(
1530
            'delete container even if not empty', ('-r', '--recursive'))
1531
    )
1532

    
1533
    @errors.generic.all
1534
    @errors.pithos.connection
1535
    @errors.pithos.container
1536
    def _run(self, container):
1537
        num_of_contents = int(self.client.get_container_info(container)[
1538
            'x-container-object-count'])
1539
        delimiter, msg = None, 'Delete container %s ?' % container
1540
        if self['recursive']:
1541
            delimiter, msg = '/', 'Empty and d%s' % msg[1:]
1542
        elif num_of_contents:
1543
            raise CLIError('Container %s is not empty' % container, details=[
1544
                'Use %s to delete non-empty containers' % (
1545
                    self.arguments['recursive'].lvalue)])
1546
        if self['yes'] or self.ask_user(msg):
1547
            if num_of_contents:
1548
                self.client.del_container(delimiter=delimiter)
1549
            self.client.purge_container()
1550

    
1551
    def main(self, container):
1552
        super(self.__class__, self)._run()
1553
        self.container, self.client.container = container, container
1554
        self._run(container)
1555

    
1556

    
1557
@command(container_cmds)
1558
class container_empty(_pithos_account):
1559
    """Empty a container"""
1560

    
1561
    arguments = dict(yes=FlagArgument('Do not prompt for permission', '--yes'))
1562

    
1563
    @errors.generic.all
1564
    @errors.pithos.connection
1565
    @errors.pithos.container
1566
    def _run(self, container):
1567
        if self['yes'] or self.ask_user('Empty container %s ?' % container):
1568
            self.client.del_container(delimiter='/')
1569

    
1570
    def main(self, container):
1571
        super(self.__class__, self)._run()
1572
        self.container, self.client.container = container, container
1573
        self._run(container)
1574

    
1575

    
1576
@command(sharer_cmds)
1577
class sharer_list(_pithos_account, _optional_json):
1578
    """List accounts who share file objects with current user"""
1579

    
1580
    arguments = dict(
1581
        detail=FlagArgument('show detailed output', ('-l', '--details')),
1582
        marker=ValueArgument('show output greater then marker', '--marker')
1583
    )
1584

    
1585
    @errors.generic.all
1586
    @errors.pithos.connection
1587
    def _run(self):
1588
        accounts = self.client.get_sharing_accounts(marker=self['marker'])
1589
        if not (self['json_output'] or self['output_format']):
1590
            usernames = self._uuids2usernames(
1591
                [acc['name'] for acc in accounts])
1592
            for item in accounts:
1593
                uuid = item['name']
1594
                item['id'], item['name'] = uuid, usernames[uuid]
1595
                if not self['detail']:
1596
                    item.pop('last_modified')
1597
        self._print(accounts)
1598

    
1599
    def main(self):
1600
        super(self.__class__, self)._run()
1601
        self._run()
1602

    
1603

    
1604
@command(sharer_cmds)
1605
class sharer_info(_pithos_account, _optional_json):
1606
    """Details on a Pithos+ sharer account (default: current account)"""
1607

    
1608
    @errors.generic.all
1609
    @errors.pithos.connection
1610
    def _run(self):
1611
        self._print(self.client.get_account_info(), self.print_dict)
1612

    
1613
    def main(self, account_uuid=None):
1614
        super(self.__class__, self)._run()
1615
        if account_uuid:
1616
            self.client.account, self.account = account_uuid, account_uuid
1617
        self._run()
1618

    
1619

    
1620
class _pithos_group(_pithos_account):
1621
    prefix = 'x-account-group-'
1622
    preflen = len(prefix)
1623

    
1624
    def _groups(self):
1625
        groups = dict()
1626
        for k, v in self.client.get_account_group().items():
1627
            groups[k[self.preflen:]] = v
1628
        return groups
1629

    
1630

    
1631
@command(group_cmds)
1632
class group_list(_pithos_group, _optional_json):
1633
    """list all groups and group members"""
1634

    
1635
    @errors.generic.all
1636
    @errors.pithos.connection
1637
    def _run(self):
1638
        self._print(self._groups(), self.print_dict)
1639

    
1640
    def main(self):
1641
        super(self.__class__, self)._run()
1642
        self._run()
1643

    
1644

    
1645
@command(group_cmds)
1646
class group_create(_pithos_group, _optional_json):
1647
    """Create a group of users"""
1648

    
1649
    arguments = dict(
1650
        user_uuid=RepeatableArgument('Add a user to the group', '--uuid'),
1651
        username=RepeatableArgument('Add a user to the group', '--username')
1652
    )
1653
    required = ['user_uuid', 'user_name']
1654

    
1655
    @errors.generic.all
1656
    @errors.pithos.connection
1657
    def _run(self, groupname, *users):
1658
        if groupname in self._groups() and not self.ask_user(
1659
                'Group %s already exists, overwrite?' % groupname):
1660
            self.error('Aborted')
1661
            return
1662
        self.client.set_account_group(groupname, users)
1663
        self._print(self._groups(), self.print_dict)
1664

    
1665
    def main(self, groupname):
1666
        super(self.__class__, self)._run()
1667
        users = (self['user_uuid'] or []) + self._usernames2uuids(
1668
            self['username'] or []).values()
1669
        if users:
1670
            self._run(groupname, *users)
1671
        else:
1672
            raise CLISyntaxError(
1673
                'No valid users specified, use %s or %s' % (
1674
                    self.arguments['user_uuid'].lvalue,
1675
                    self.arguments['username'].lvalue),
1676
                details=[
1677
                    'Check if a username or uuid is valid with',
1678
                    '  user uuid2username', 'OR', '  user username2uuid'])
1679

    
1680

    
1681
@command(group_cmds)
1682
class group_delete(_pithos_group, _optional_json):
1683
    """Delete a user group"""
1684

    
1685
    @errors.generic.all
1686
    @errors.pithos.connection
1687
    def _run(self, groupname):
1688
        self.client.del_account_group(groupname)
1689
        self._print(self._groups(), self.print_dict)
1690

    
1691
    def main(self, groupname):
1692
        super(self.__class__, self)._run()
1693
        self._run(groupname)
1694

    
1695

    
1696
#  Deprecated commands
1697

    
1698
@command(file_cmds)
1699
class file_publish(_pithos_init):
1700
    """DEPRECATED, replaced by [kamaki] file modify OBJECT --publish"""
1701

    
1702
    def main(self, *args):
1703
        raise CLISyntaxError('DEPRECATED', details=[
1704
            'This command is replaced by:',
1705
            '  [kamaki] file modify OBJECT --publish'])
1706

    
1707

    
1708
@command(file_cmds)
1709
class file_unpublish(_pithos_init):
1710
    """DEPRECATED, replaced by [kamaki] file modify OBJECT --unpublish"""
1711

    
1712
    def main(self, *args):
1713
        raise CLISyntaxError('DEPRECATED', details=[
1714
            'This command is replaced by:',
1715
            '  [kamaki] file modify OBJECT --unpublish'])