Statistics
| Branch: | Tag: | Revision:

root / kamaki / cli / commands / pithos.py @ 4a5192d0

History | View | Annotate | Download (69.2 kB)

1
# Copyright 2011-2013 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.command
33

    
34
from time import localtime, strftime
35
from io import StringIO
36
from pydoc import pager
37
from os import path, walk, makedirs
38

    
39
from kamaki.clients.pithos import PithosClient, ClientError
40

    
41
from kamaki.cli import command
42
from kamaki.cli.command_tree import CommandTree
43
from kamaki.cli.commands import (
44
    _command_init, errors, addLogSettings, DontRaiseKeyError, _optional_json,
45
    _name_filter, _optional_output_cmd)
46
from kamaki.cli.errors import (
47
    CLIBaseUrlError, CLIError, CLIInvalidArgument, raiseCLIError,
48
    CLISyntaxError)
49
from kamaki.cli.argument import (
50
    FlagArgument, IntArgument, ValueArgument, DateArgument, KeyValueArgument,
51
    ProgressBarArgument, RepeatableArgument, DataSizeArgument)
52
from kamaki.cli.utils import (
53
    format_size, bold, get_path_size, guess_mime_type)
54

    
55
file_cmds = CommandTree('file', 'Pithos+/Storage object level API commands')
56
container_cmds = CommandTree(
57
    'container', 'Pithos+/Storage container level API commands')
58
sharer_cmds = CommandTree('sharer', 'Pithos+/Storage sharers')
59
group_cmds = CommandTree('group', 'Pithos+/Storage user groups')
60
_commands = [file_cmds, container_cmds, sharer_cmds, group_cmds]
61

    
62

    
63
class _pithos_init(_command_init):
64
    """Initilize a pithos+ client
65
    There is always a default account (current user uuid)
66
    There is always a default container (pithos)
67
    """
68

    
69
    @DontRaiseKeyError
70
    def _custom_container(self):
71
        return self.config.get_cloud(self.cloud, 'pithos_container')
72

    
73
    @DontRaiseKeyError
74
    def _custom_uuid(self):
75
        return self.config.get_cloud(self.cloud, 'pithos_uuid')
76

    
77
    def _set_account(self):
78
        self.account = self._custom_uuid()
79
        if self.account:
80
            return
81
        astakos = getattr(self, 'auth_base', None)
82
        if astakos:
83
            self.account = astakos.user_term('id', self.token)
84
        else:
85
            raise CLIBaseUrlError(service='astakos')
86

    
87
    @errors.generic.all
88
    @addLogSettings
89
    def _run(self):
90
        cloud = getattr(self, 'cloud', None)
91
        if cloud:
92
            self.base_url = self._custom_url('pithos')
93
        else:
94
            self.cloud = 'default'
95
        self.token = self._custom_token('pithos')
96
        self.container = self._custom_container() or 'pithos'
97

    
98
        astakos = getattr(self, 'auth_base', None)
99
        if astakos:
100
            self.token = self.token or astakos.token
101
            if not self.base_url:
102
                pithos_endpoints = astakos.get_service_endpoints(
103
                    self._custom_type('pithos') or 'object-store',
104
                    self._custom_version('pithos') or '')
105
                self.base_url = pithos_endpoints['publicURL']
106
        else:
107
            raise CLIBaseUrlError(service='astakos')
108

    
109
        self._set_account()
110
        self.client = PithosClient(
111
            self.base_url, self.token, self.account, self.container)
112

    
113
    def main(self):
114
        self._run()
115

    
116

    
117
class _pithos_account(_pithos_init):
118
    """Setup account"""
119

    
120
    def __init__(self, arguments={}, auth_base=None, cloud=None):
121
        super(_pithos_account, self).__init__(arguments, auth_base, cloud)
122
        self['account'] = ValueArgument(
123
            'Use (a different) user uuid', ('-A', '--account'))
124

    
125
    def print_objects(self, object_list):
126
        for index, obj in enumerate(object_list):
127
            pretty_obj = obj.copy()
128
            index += 1
129
            empty_space = ' ' * (len(str(len(object_list))) - len(str(index)))
130
            if 'subdir' in obj:
131
                continue
132
            if self._is_dir(obj):
133
                size = 'D'
134
            else:
135
                size = format_size(obj['bytes'])
136
                pretty_obj['bytes'] = '%s (%s)' % (obj['bytes'], size)
137
            oname = obj['name'] if self['more'] else bold(obj['name'])
138
            prfx = ('%s%s. ' % (empty_space, index)) if self['enum'] else ''
139
            if self['detail']:
140
                self.writeln('%s%s' % (prfx, oname))
141
                self.print_dict(pretty_obj, exclude=('name'))
142
                self.writeln()
143
            else:
144
                oname = '%s%9s %s' % (prfx, size, oname)
145
                oname += '/' if self._is_dir(obj) else u''
146
                self.writeln(oname)
147

    
148
    @staticmethod
149
    def _is_dir(remote_dict):
150
        return 'application/directory' == remote_dict.get(
151
            'content_type', remote_dict.get('content-type', ''))
152

    
153
    def _run(self):
154
        super(_pithos_account, self)._run()
155
        self.client.account = self['account'] or getattr(
156
            self, 'account', getattr(self.client, 'account', None))
157

    
158

    
159
class _pithos_container(_pithos_account):
160
    """Setup container"""
161

    
162
    def __init__(self, arguments={}, auth_base=None, cloud=None):
163
        super(_pithos_container, self).__init__(arguments, auth_base, cloud)
164
        self['container'] = ValueArgument(
165
            'Use this container (default: pithos)', ('-C', '--container'))
166

    
167
    @staticmethod
168
    def _resolve_pithos_url(url):
169
        """Match urls of one of the following formats:
170
        pithos://ACCOUNT/CONTAINER/OBJECT_PATH
171
        /CONTAINER/OBJECT_PATH
172
        return account, container, path
173
        """
174
        account, container, obj_path, prefix = '', '', url, 'pithos://'
175
        if url.startswith(prefix):
176
            account, sep, url = url[len(prefix):].partition('/')
177
            url = '/%s' % url
178
        if url.startswith('/'):
179
            container, sep, obj_path = url[1:].partition('/')
180
        return account, container, obj_path
181

    
182
    def _run(self, url=None):
183
        acc, con, self.path = self._resolve_pithos_url(url or '')
184
        #  self.account = acc or getattr(self, 'account', '')
185
        super(_pithos_container, self)._run()
186
        self.container = con or self['container'] or getattr(
187
            self, 'container', None) or getattr(self.client, 'container', '')
188
        self.client.account = acc or self.client.account
189
        self.client.container = self.container
190

    
191

    
192
@command(file_cmds)
193
class file_info(_pithos_container, _optional_json):
194
    """Get information/details about a file"""
195

    
196
    arguments = dict(
197
        object_version=ValueArgument(
198
            'download a file of a specific version', '--object-version'),
199
        hashmap=FlagArgument(
200
            'Get file hashmap instead of details', '--hashmap'),
201
        matching_etag=ValueArgument(
202
            'show output if ETags match', '--if-match'),
203
        non_matching_etag=ValueArgument(
204
            'show output if ETags DO NOT match', '--if-none-match'),
205
        modified_since_date=DateArgument(
206
            'show output modified since then', '--if-modified-since'),
207
        unmodified_since_date=DateArgument(
208
            'show output unmodified since then', '--if-unmodified-since'),
209
        sharing=FlagArgument(
210
            'show object permissions and sharing information', '--sharing'),
211
        metadata=FlagArgument('show only object metadata', '--metadata'),
212
        versions=FlagArgument(
213
            'show the list of versions for the file', '--object-versions')
214
    )
215

    
216
    def version_print(self, versions):
217
        return {'/%s/%s' % (self.container, self.path): [
218
            dict(version_id=vitem[0], created=strftime(
219
                '%d-%m-%Y %H:%M:%S',
220
                localtime(float(vitem[1])))) for vitem in versions]}
221

    
222
    @errors.generic.all
223
    @errors.pithos.connection
224
    @errors.pithos.container
225
    @errors.pithos.object_path
226
    def _run(self):
227
        if self['hashmap']:
228
            r = self.client.get_object_hashmap(
229
                self.path,
230
                version=self['object_version'],
231
                if_match=self['matching_etag'],
232
                if_none_match=self['non_matching_etag'],
233
                if_modified_since=self['modified_since_date'],
234
                if_unmodified_since=self['unmodified_since_date'])
235
        elif self['sharing']:
236
            r = self.client.get_object_sharing(self.path)
237
            r['public url'] = self.client.get_object_info(
238
                self.path, version=self['object_version']).get(
239
                    'x-object-public', None)
240
        elif self['metadata']:
241
            r, preflen = dict(), len('x-object-meta-')
242
            for k, v in self.client.get_object_meta(self.path).items():
243
                r[k[preflen:]] = v
244
        elif self['versions']:
245
            r = self.version_print(
246
                self.client.get_object_versionlist(self.path))
247
        else:
248
            r = self.client.get_object_info(
249
                self.path, version=self['object_version'])
250
        self._print(r, self.print_dict)
251

    
252
    def main(self, path_or_url):
253
        super(self.__class__, self)._run(path_or_url)
254
        self._run()
255

    
256

    
257
@command(file_cmds)
258
class file_list(_pithos_container, _optional_json, _name_filter):
259
    """List all objects in a container or a directory object"""
260

    
261
    arguments = dict(
262
        detail=FlagArgument('detailed output', ('-l', '--list')),
263
        limit=IntArgument('limit number of listed items', ('-n', '--number')),
264
        marker=ValueArgument('output greater that marker', '--marker'),
265
        delimiter=ValueArgument('show output up to delimiter', '--delimiter'),
266
        meta=ValueArgument(
267
            'show output with specified meta keys', '--meta',
268
            default=[]),
269
        if_modified_since=ValueArgument(
270
            'show output modified since then', '--if-modified-since'),
271
        if_unmodified_since=ValueArgument(
272
            'show output not modified since then', '--if-unmodified-since'),
273
        until=DateArgument('show metadata until then', '--until'),
274
        format=ValueArgument(
275
            'format to parse until data (default: d/m/Y H:M:S )', '--format'),
276
        shared_by_me=FlagArgument(
277
            'show only files shared to other users', '--shared-by-me'),
278
        public=FlagArgument('show only published objects', '--public'),
279
        more=FlagArgument('read long results', '--more'),
280
        enum=FlagArgument('Enumerate results', '--enumerate'),
281
        recursive=FlagArgument(
282
            'Recursively list containers and their contents',
283
            ('-R', '--recursive'))
284
    )
285

    
286
    @errors.generic.all
287
    @errors.pithos.connection
288
    @errors.pithos.container
289
    @errors.pithos.object_path
290
    def _run(self):
291
        r = self.client.container_get(
292
            limit=False if self['more'] else self['limit'],
293
            marker=self['marker'],
294
            prefix=self['name_pref'],
295
            delimiter=self['delimiter'],
296
            path=self.path or '',
297
            show_only_shared=self['shared_by_me'],
298
            public=self['public'],
299
            if_modified_since=self['if_modified_since'],
300
            if_unmodified_since=self['if_unmodified_since'],
301
            until=self['until'],
302
            meta=self['meta'])
303

    
304
        #  REMOVE THIS if version >> 0.12
305
        if not r.json:
306
            self.error('  NOTE: Since v0.12, use / for containers e.g.,')
307
            self.error('    [kamaki] file list /pithos')
308

    
309
        files = self._filter_by_name(r.json)
310
        if self['more']:
311
            outbu, self._out = self._out, StringIO()
312
        try:
313
            if self['json_output'] or self['output_format']:
314
                self._print(files)
315
            else:
316
                self.print_objects(files)
317
        finally:
318
            if self['more']:
319
                pager(self._out.getvalue())
320
                self._out = outbu
321

    
322
    def main(self, path_or_url=''):
323
        super(self.__class__, self)._run(path_or_url)
324
        self._run()
325

    
326

    
327
@command(file_cmds)
328
class file_modify(_pithos_container):
329
    """Modify the attributes of a file or directory object"""
330

    
331
    arguments = dict(
332
        publish=FlagArgument(
333
            'Make an object public (returns the public URL)', '--publish'),
334
        unpublish=FlagArgument(
335
            'Make an object unpublic', '--unpublish'),
336
        uuid_for_read_permission=RepeatableArgument(
337
            'Give read access to user/group (can be repeated, accumulative). '
338
            'Format for users: UUID . Format for groups: UUID:GROUP . '
339
            'Use * for all users/groups', '--read-permission'),
340
        uuid_for_write_permission=RepeatableArgument(
341
            'Give write access to user/group (can be repeated, accumulative). '
342
            'Format for users: UUID . Format for groups: UUID:GROUP . '
343
            'Use * for all users/groups', '--write-permission'),
344
        no_permissions=FlagArgument('Remove permissions', '--no-permissions'),
345
        metadata_to_set=KeyValueArgument(
346
            'Add metadata (KEY=VALUE) to an object (can be repeated)',
347
            '--metadata-add'),
348
        metadata_key_to_delete=RepeatableArgument(
349
            'Delete object metadata (can be repeated)', '--metadata-del'),
350
    )
351
    required = [
352
        'publish', 'unpublish', 'uuid_for_read_permission', 'metadata_to_set',
353
        'uuid_for_write_permission', 'no_permissions',
354
        'metadata_key_to_delete']
355

    
356
    @errors.generic.all
357
    @errors.pithos.connection
358
    @errors.pithos.container
359
    @errors.pithos.object_path
360
    def _run(self):
361
        if self['publish']:
362
            self.writeln(self.client.publish_object(self.path))
363
        if self['unpublish']:
364
            self.client.unpublish_object(self.path)
365
        if self['uuid_for_read_permission'] or self[
366
                'uuid_for_write_permission']:
367
            perms = self.client.get_object_sharing(self.path)
368
            read, write = perms.get('read', ''), perms.get('write', '')
369
            read = read.split(',') if read else []
370
            write = write.split(',') if write else []
371
            read += (self['uuid_for_read_permission'] or [])
372
            write += (self['uuid_for_write_permission'] or [])
373
            self.client.set_object_sharing(
374
                self.path, read_permission=read, write_permission=write)
375
            self.print_dict(self.client.get_object_sharing(self.path))
376
        if self['no_permissions']:
377
            self.client.del_object_sharing(self.path)
378
        metadata = self['metadata_to_set'] or dict()
379
        for k in (self['metadata_key_to_delete'] or []):
380
            metadata[k] = ''
381
        if metadata:
382
            self.client.set_object_meta(self.path, metadata)
383
            self.print_dict(self.client.get_object_meta(self.path))
384

    
385
    def main(self, path_or_url):
386
        super(self.__class__, self)._run(path_or_url)
387
        if self['publish'] and self['unpublish']:
388
            raise CLIInvalidArgument(
389
                'Arguments %s and %s cannot be used together' % (
390
                    self.arguments['publish'].lvalue,
391
                    self.arguments['publish'].lvalue))
392
        if self['no_permissions'] and (
393
                self['uuid_for_read_permission'] or self[
394
                    'uuid_for_write_permission']):
395
            raise CLIInvalidArgument(
396
                '%s cannot be used with other permission arguments' % (
397
                    self.arguments['no_permissions'].lvalue))
398
        self._run()
399

    
400

    
401
@command(file_cmds)
402
class file_create(_pithos_container, _optional_output_cmd):
403
    """Create an empty file"""
404

    
405
    arguments = dict(
406
        content_type=ValueArgument(
407
            'Set content type (default: application/octet-stream)',
408
            '--content-type',
409
            default='application/octet-stream')
410
    )
411

    
412
    @errors.generic.all
413
    @errors.pithos.connection
414
    @errors.pithos.container
415
    def _run(self):
416
        self._optional_output(
417
            self.client.create_object(self.path, self['content_type']))
418

    
419
    def main(self, path_or_url):
420
        super(self.__class__, self)._run(path_or_url)
421
        self._run()
422

    
423

    
424
@command(file_cmds)
425
class file_mkdir(_pithos_container, _optional_output_cmd):
426
    """Create a directory: /file create --content-type='applcation/directory'
427
    """
428

    
429
    @errors.generic.all
430
    @errors.pithos.connection
431
    @errors.pithos.container
432
    def _run(self):
433
        self._optional_output(self.client.create_directory(self.path))
434

    
435
    def main(self, path_or_url):
436
        super(self.__class__, self)._run(path_or_url)
437
        self._run()
438

    
439

    
440
@command(file_cmds)
441
class file_delete(_pithos_container):
442
    """Delete a file or directory object"""
443

    
444
    arguments = dict(
445
        until_date=DateArgument('remove history until then', '--until'),
446
        yes=FlagArgument('Do not prompt for permission', '--yes'),
447
        recursive=FlagArgument(
448
            'If a directory, empty first', ('-r', '--recursive')),
449
        delimiter=ValueArgument(
450
            'delete objects prefixed with <object><delimiter>', '--delimiter')
451
    )
452

    
453
    @errors.generic.all
454
    @errors.pithos.connection
455
    @errors.pithos.container
456
    @errors.pithos.object_path
457
    def _run(self):
458
        if self.path:
459
            if self['yes'] or self.ask_user(
460
                    'Delete /%s/%s ?' % (self.container, self.path)):
461
                self.client.del_object(
462
                    self.path,
463
                    until=self['until_date'],
464
                    delimiter='/' if self['recursive'] else self['delimiter'])
465
            else:
466
                self.error('Aborted')
467
        else:
468
            if self['yes'] or self.ask_user(
469
                    'Empty container /%s ?' % self.container):
470
                self.client.container_delete(self.container, delimiter='/')
471
            else:
472
                self.error('Aborted')
473

    
474
    def main(self, path_or_url):
475
        super(self.__class__, self)._run(path_or_url)
476
        self._run()
477

    
478

    
479
class _source_destination(_pithos_container, _optional_output_cmd):
480

    
481
    sd_arguments = dict(
482
        destination_user_uuid=ValueArgument(
483
            'default: current user uuid', '--to-account'),
484
        destination_container=ValueArgument(
485
            'default: pithos', '--to-container'),
486
        source_prefix=FlagArgument(
487
            'Transfer all files that are prefixed with SOURCE PATH If the '
488
            'destination path is specified, replace SOURCE_PATH with '
489
            'DESTINATION_PATH',
490
            ('-r', '--recursive')),
491
        force=FlagArgument(
492
            'Overwrite destination objects, if needed', ('-f', '--force')),
493
        source_version=ValueArgument(
494
            'The version of the source object', '--source-version')
495
    )
496

    
497
    def __init__(self, arguments={}, auth_base=None, cloud=None):
498
        self.arguments.update(arguments)
499
        self.arguments.update(self.sd_arguments)
500
        super(_source_destination, self).__init__(
501
            self.arguments, auth_base, cloud)
502

    
503
    def _report_transfer(self, src, dst, transfer_name):
504
        if not dst:
505
            if transfer_name in ('move', ):
506
                self.error('  delete source directory %s' % src)
507
            return
508
        dst_prf = '' if self.account == self.dst_client.account else (
509
                'pithos://%s' % self.dst_client.account)
510
        if src:
511
            src_prf = '' if self.account == self.dst_client.account else (
512
                    'pithos://%s' % self.account)
513
            self.error('  %s %s/%s/%s\n  -->  %s/%s/%s' % (
514
                transfer_name,
515
                src_prf, self.container, src,
516
                dst_prf, self.dst_client.container, dst))
517
        else:
518
            self.error('  mkdir %s/%s/%s' % (
519
                dst_prf, self.dst_client.container, dst))
520

    
521
    @errors.generic.all
522
    @errors.pithos.account
523
    def _src_dst(self, version=None):
524
        """Preconditions:
525
        self.account, self.container, self.path
526
        self.dst_acc, self.dst_con, self.dst_path
527
        They should all be configured properly
528
        :returns: [(src_path, dst_path), ...], if src_path is None, create
529
            destination directory
530
        """
531
        src_objects, dst_objects, pairs = dict(), dict(), []
532
        try:
533
            for obj in self.dst_client.list_objects(
534
                    prefix=self.dst_path or self.path or '/'):
535
                dst_objects[obj['name']] = obj
536
        except ClientError as ce:
537
            if ce.status in (404, ):
538
                raise CLIError(
539
                    'Destination container pithos://%s/%s not found' % (
540
                        self.dst_client.account, self.dst_client.container))
541
            raise ce
542
        if self['source_prefix']:
543
            #  Copy and replace prefixes
544
            for src_obj in self.client.list_objects(prefix=self.path):
545
                src_objects[src_obj['name']] = src_obj
546
            for src_path, src_obj in src_objects.items():
547
                dst_path = '%s%s' % (
548
                    self.dst_path or self.path, src_path[len(self.path):])
549
                dst_obj = dst_objects.get(dst_path, None)
550
                if self['force'] or not dst_obj:
551
                    #  Just do it
552
                    pairs.append((
553
                        None if self._is_dir(src_obj) else src_path, dst_path))
554
                    if self._is_dir(src_obj):
555
                        pairs.append((self.path or dst_path, None))
556
                elif not (self._is_dir(dst_obj) and self._is_dir(src_obj)):
557
                    raise CLIError(
558
                        'Destination object exists', importance=2, details=[
559
                            'Failed while transfering:',
560
                            '    pithos://%s/%s/%s' % (
561
                                    self.account,
562
                                    self.container,
563
                                    src_path),
564
                            '--> pithos://%s/%s/%s' % (
565
                                    self.dst_client.account,
566
                                    self.dst_client.container,
567
                                    dst_path),
568
                            'Use %s to transfer overwrite' % (
569
                                    self.arguments['force'].lvalue)])
570
        else:
571
            #  One object transfer
572
            try:
573
                src_version_arg = self.arguments.get('source_version', None)
574
                src_obj = self.client.get_object_info(
575
                    self.path,
576
                    version=src_version_arg.value if src_version_arg else None)
577
            except ClientError as ce:
578
                if ce.status in (204, ):
579
                    raise CLIError(
580
                        'Missing specific path container %s' % self.container,
581
                        importance=2, details=[
582
                            'To transfer container contents %s' % (
583
                                self.arguments['source_prefix'].lvalue)])
584
                raise
585
            dst_path = self.dst_path or self.path
586
            dst_obj = dst_objects.get(dst_path or self.path, None)
587
            if self['force'] or not dst_obj:
588
                pairs.append(
589
                    (None if self._is_dir(src_obj) else self.path, dst_path))
590
                if self._is_dir(src_obj):
591
                    pairs.append((self.path or dst_path, None))
592
            elif self._is_dir(src_obj):
593
                raise CLIError(
594
                    'Cannot transfer an application/directory object',
595
                    importance=2, details=[
596
                        'The object pithos://%s/%s/%s is a directory' % (
597
                            self.account,
598
                            self.container,
599
                            self.path),
600
                        'To recursively copy a directory, use',
601
                        '  %s' % self.arguments['source_prefix'].lvalue,
602
                        'To create a file, use',
603
                        '  /file create  (general purpose)',
604
                        '  /file mkdir   (a directory object)'])
605
            else:
606
                raise CLIError(
607
                    'Destination object exists',
608
                    importance=2, details=[
609
                        'Failed while transfering:',
610
                        '    pithos://%s/%s/%s' % (
611
                                self.account,
612
                                self.container,
613
                                self.path),
614
                        '--> pithos://%s/%s/%s' % (
615
                                self.dst_client.account,
616
                                self.dst_client.container,
617
                                dst_path),
618
                        'Use %s to transfer overwrite' % (
619
                                self.arguments['force'].lvalue)])
620
        return pairs
621

    
622
    def _run(self, source_path_or_url, destination_path_or_url=''):
623
        super(_source_destination, self)._run(source_path_or_url)
624
        dst_acc, dst_con, dst_path = self._resolve_pithos_url(
625
            destination_path_or_url)
626
        self.dst_client = PithosClient(
627
            base_url=self.client.base_url, token=self.client.token,
628
            container=self[
629
                'destination_container'] or dst_con or self.client.container,
630
            account=self[
631
                'destination_user_uuid'] or dst_acc or self.account)
632
        self.dst_path = dst_path or self.path
633

    
634

    
635
@command(file_cmds)
636
class file_copy(_source_destination):
637
    """Copy objects, even between different accounts or containers"""
638

    
639
    arguments = dict(
640
        public=ValueArgument('publish new object', '--public'),
641
        content_type=ValueArgument(
642
            'change object\'s content type', '--content-type'),
643
        source_version=ValueArgument(
644
            'The version of the source object', '--object-version')
645
    )
646

    
647
    @errors.generic.all
648
    @errors.pithos.connection
649
    @errors.pithos.container
650
    @errors.pithos.account
651
    def _run(self):
652
        for src, dst in self._src_dst(self['source_version']):
653
            self._report_transfer(src, dst, 'copy')
654
            if src and dst:
655
                self.dst_client.copy_object(
656
                    src_container=self.client.container,
657
                    src_object=src,
658
                    dst_container=self.dst_client.container,
659
                    dst_object=dst,
660
                    source_account=self.client.account,
661
                    source_version=self['source_version'],
662
                    public=self['public'],
663
                    content_type=self['content_type'])
664
            elif dst:
665
                self.dst_client.create_directory(dst)
666

    
667
    def main(self, source_path_or_url, destination_path_or_url=None):
668
        super(file_copy, self)._run(
669
            source_path_or_url, destination_path_or_url or '')
670
        self._run()
671

    
672

    
673
@command(file_cmds)
674
class file_move(_source_destination):
675
    """Move objects, even between different accounts or containers"""
676

    
677
    arguments = dict(
678
        public=ValueArgument('publish new object', '--public'),
679
        content_type=ValueArgument(
680
            'change object\'s content type', '--content-type')
681
    )
682

    
683
    @errors.generic.all
684
    @errors.pithos.connection
685
    @errors.pithos.container
686
    @errors.pithos.account
687
    def _run(self):
688
        for src, dst in self._src_dst():
689
            self._report_transfer(src, dst, 'move')
690
            if src and dst:
691
                self.dst_client.move_object(
692
                    src_container=self.client.container,
693
                    src_object=src,
694
                    dst_container=self.dst_client.container,
695
                    dst_object=dst,
696
                    source_account=self.account,
697
                    public=self['public'],
698
                    content_type=self['content_type'])
699
            elif dst:
700
                self.dst_client.create_directory(dst)
701
            else:
702
                self.client.del_object(src)
703

    
704
    def main(self, source_path_or_url, destination_path_or_url=None):
705
        super(file_move, self)._run(
706
            source_path_or_url, destination_path_or_url or '')
707
        self._run()
708

    
709

    
710
@command(file_cmds)
711
class file_append(_pithos_container, _optional_output_cmd):
712
    """Append local file to (existing) remote object
713
    The remote object should exist.
714
    If the remote object is a directory, it is transformed into a file.
715
    In the later case, objects under the directory remain intact.
716
    """
717

    
718
    arguments = dict(
719
        progress_bar=ProgressBarArgument(
720
            'do not show progress bar', ('-N', '--no-progress-bar'),
721
            default=False),
722
        max_threads=IntArgument('default: 1', '--threads'),
723
    )
724

    
725
    @errors.generic.all
726
    @errors.pithos.connection
727
    @errors.pithos.container
728
    @errors.pithos.object_path
729
    def _run(self, local_path):
730
        if self['max_threads'] > 0:
731
            self.client.MAX_THREADS = int(self['max_threads'])
732
        (progress_bar, upload_cb) = self._safe_progress_bar('Appending')
733
        try:
734
            with open(local_path, 'rb') as f:
735
                self._optional_output(
736
                    self.client.append_object(self.path, f, upload_cb))
737
        finally:
738
            self._safe_progress_bar_finish(progress_bar)
739

    
740
    def main(self, local_path, remote_path_or_url):
741
        super(self.__class__, self)._run(remote_path_or_url)
742
        self._run(local_path)
743

    
744

    
745
@command(file_cmds)
746
class file_truncate(_pithos_container, _optional_output_cmd):
747
    """Truncate remote file up to size"""
748

    
749
    arguments = dict(
750
        size_in_bytes=IntArgument('Length of file after truncation', '--size')
751
    )
752
    required = ('size_in_bytes', )
753

    
754
    @errors.generic.all
755
    @errors.pithos.connection
756
    @errors.pithos.container
757
    @errors.pithos.object_path
758
    @errors.pithos.object_size
759
    def _run(self, size):
760
        self._optional_output(self.client.truncate_object(self.path, size))
761

    
762
    def main(self, path_or_url):
763
        super(self.__class__, self)._run(path_or_url)
764
        self._run(size=self['size_in_bytes'])
765

    
766

    
767
@command(file_cmds)
768
class file_overwrite(_pithos_container, _optional_output_cmd):
769
    """Overwrite part of a remote file"""
770

    
771
    arguments = dict(
772
        progress_bar=ProgressBarArgument(
773
            'do not show progress bar', ('-N', '--no-progress-bar'),
774
            default=False),
775
        start_position=IntArgument('File position in bytes', '--from'),
776
        end_position=IntArgument('File position in bytes', '--to'),
777
    )
778
    required = ('start_position', 'end_position')
779

    
780
    @errors.generic.all
781
    @errors.pithos.connection
782
    @errors.pithos.container
783
    @errors.pithos.object_path
784
    @errors.pithos.object_size
785
    def _run(self, local_path, start, end):
786
        start, end = int(start), int(end)
787
        (progress_bar, upload_cb) = self._safe_progress_bar(
788
            'Overwrite %s bytes' % (end - start))
789
        try:
790
            with open(path.abspath(local_path), 'rb') as f:
791
                self._optional_output(self.client.overwrite_object(
792
                    obj=self.path,
793
                    start=start,
794
                    end=end,
795
                    source_file=f,
796
                    upload_cb=upload_cb))
797
        finally:
798
            self._safe_progress_bar_finish(progress_bar)
799

    
800
    def main(self, local_path, path_or_url):
801
        super(self.__class__, self)._run(path_or_url)
802
        self.path = self.path or path.basename(local_path)
803
        self._run(
804
            local_path=local_path,
805
            start=self['start_position'],
806
            end=self['end_position'])
807

    
808

    
809
@command(file_cmds)
810
class file_upload(_pithos_container, _optional_output_cmd):
811
    """Upload a file"""
812

    
813
    arguments = dict(
814
        max_threads=IntArgument('default: 5', '--threads'),
815
        content_encoding=ValueArgument(
816
            'set MIME content type', '--content-encoding'),
817
        content_disposition=ValueArgument(
818
            'specify objects presentation style', '--content-disposition'),
819
        content_type=ValueArgument('specify content type', '--content-type'),
820
        uuid_for_read_permission=RepeatableArgument(
821
            'Give read access to a user or group (can be repeated) '
822
            'Use * for all users',
823
            '--read-permission'),
824
        uuid_for_write_permission=RepeatableArgument(
825
            'Give write access to a user or group (can be repeated) '
826
            'Use * for all users',
827
            '--write-permission'),
828
        public=FlagArgument('make object publicly accessible', '--public'),
829
        progress_bar=ProgressBarArgument(
830
            'do not show progress bar',
831
            ('-N', '--no-progress-bar'),
832
            default=False),
833
        overwrite=FlagArgument('Force (over)write', ('-f', '--force')),
834
        recursive=FlagArgument(
835
            'Recursively upload directory *contents* + subdirectories',
836
            ('-r', '--recursive')),
837
        unchunked=FlagArgument(
838
            'Upload file as one block (not recommended)', '--unchunked'),
839
        md5_checksum=ValueArgument(
840
            'Confirm upload with a custom checksum (MD5)', '--etag'),
841
        use_hashes=FlagArgument(
842
            'Source file contains hashmap not data', '--source-is-hashmap'),
843
    )
844

    
845
    def _sharing(self):
846
        sharing = dict()
847
        readlist = self['uuid_for_read_permission']
848
        if readlist:
849
            sharing['read'] = self['uuid_for_read_permission']
850
        writelist = self['uuid_for_write_permission']
851
        if writelist:
852
            sharing['write'] = self['uuid_for_write_permission']
853
        return sharing or None
854

    
855
    def _check_container_limit(self, path):
856
        cl_dict = self.client.get_container_limit()
857
        container_limit = int(cl_dict['x-container-policy-quota'])
858
        r = self.client.container_get()
859
        used_bytes = sum(int(o['bytes']) for o in r.json)
860
        path_size = get_path_size(path)
861
        if container_limit and path_size > (container_limit - used_bytes):
862
            raise CLIError(
863
                'Container %s (limit(%s) - used(%s)) < (size(%s) of %s)' % (
864
                    self.client.container,
865
                    format_size(container_limit),
866
                    format_size(used_bytes),
867
                    format_size(path_size),
868
                    path),
869
                details=[
870
                    'Check accound limit: /file quota',
871
                    'Check container limit:',
872
                    '\t/file containerlimit get %s' % self.client.container,
873
                    'Increase container limit:',
874
                    '\t/file containerlimit set <new limit> %s' % (
875
                        self.client.container)])
876

    
877
    def _src_dst(self, local_path, remote_path, objlist=None):
878
        lpath = path.abspath(local_path)
879
        short_path = path.basename(path.abspath(local_path))
880
        rpath = remote_path or short_path
881
        if path.isdir(lpath):
882
            if not self['recursive']:
883
                raise CLIError('%s is a directory' % lpath, details=[
884
                    'Use %s to upload directories & contents' % (
885
                        self.arguments['recursive'].lvalue)])
886
            robj = self.client.container_get(path=rpath)
887
            if not self['overwrite']:
888
                if robj.json:
889
                    raise CLIError(
890
                        'Objects/files prefixed as %s already exist' % rpath,
891
                        details=['Existing objects:'] + ['\t/%s/\t%s' % (
892
                            o['name'],
893
                            o['content_type'][12:]) for o in robj.json] + [
894
                            'Use -f to add, overwrite or resume'])
895
                else:
896
                    try:
897
                        topobj = self.client.get_object_info(rpath)
898
                        if not self._is_dir(topobj):
899
                            raise CLIError(
900
                                'Object /%s/%s exists but not a directory' % (
901
                                    self.container, rpath),
902
                                details=['Use -f to overwrite'])
903
                    except ClientError as ce:
904
                        if ce.status not in (404, ):
905
                            raise
906
            self._check_container_limit(lpath)
907
            prev = ''
908
            for top, subdirs, files in walk(lpath):
909
                if top != prev:
910
                    prev = top
911
                    try:
912
                        rel_path = rpath + top.split(lpath)[1]
913
                    except IndexError:
914
                        rel_path = rpath
915
                    self.error('mkdir /%s/%s' % (
916
                        self.client.container, rel_path))
917
                    self.client.create_directory(rel_path)
918
                for f in files:
919
                    fpath = path.join(top, f)
920
                    if path.isfile(fpath):
921
                        rel_path = rel_path.replace(path.sep, '/')
922
                        pathfix = f.replace(path.sep, '/')
923
                        yield open(fpath, 'rb'), '%s/%s' % (rel_path, pathfix)
924
                    else:
925
                        self.error('%s is not a regular file' % fpath)
926
        else:
927
            if not path.isfile(lpath):
928
                raise CLIError(('%s is not a regular file' % lpath) if (
929
                    path.exists(lpath)) else '%s does not exist' % lpath)
930
            try:
931
                robj = self.client.get_object_info(rpath)
932
                if remote_path and self._is_dir(robj):
933
                    rpath += '/%s' % (short_path.replace(path.sep, '/'))
934
                    self.client.get_object_info(rpath)
935
                if not self['overwrite']:
936
                    raise CLIError(
937
                        'Object /%s/%s already exists' % (
938
                            self.container, rpath),
939
                        details=['use -f to overwrite / resume'])
940
            except ClientError as ce:
941
                if ce.status not in (404, ):
942
                    raise
943
            self._check_container_limit(lpath)
944
            yield open(lpath, 'rb'), rpath
945

    
946
    def _run(self, local_path, remote_path):
947
        self.client.MAX_THREADS = int(self['max_threads'] or 5)
948
        params = dict(
949
            content_encoding=self['content_encoding'],
950
            content_type=self['content_type'],
951
            content_disposition=self['content_disposition'],
952
            sharing=self._sharing(),
953
            public=self['public'])
954
        uploaded, container_info_cache = list, dict()
955
        rpref = 'pithos://%s' if self['account'] else ''
956
        for f, rpath in self._src_dst(local_path, remote_path):
957
            self.error('%s --> %s/%s/%s' % (
958
                f.name, rpref, self.client.container, rpath))
959
            if not (self['content_type'] and self['content_encoding']):
960
                ctype, cenc = guess_mime_type(f.name)
961
                params['content_type'] = self['content_type'] or ctype
962
                params['content_encoding'] = self['content_encoding'] or cenc
963
            if self['unchunked']:
964
                r = self.client.upload_object_unchunked(
965
                    rpath, f,
966
                    etag=self['md5_checksum'], withHashFile=self['use_hashes'],
967
                    **params)
968
                if self['with_output'] or self['json_output']:
969
                    r['name'] = '/%s/%s' % (self.client.container, rpath)
970
                    uploaded.append(r)
971
            else:
972
                try:
973
                    (progress_bar, upload_cb) = self._safe_progress_bar(
974
                        'Uploading %s' % f.name.split(path.sep)[-1])
975
                    if progress_bar:
976
                        hash_bar = progress_bar.clone()
977
                        hash_cb = hash_bar.get_generator(
978
                            'Calculating block hashes')
979
                    else:
980
                        hash_cb = None
981
                    r = self.client.upload_object(
982
                        rpath, f,
983
                        hash_cb=hash_cb,
984
                        upload_cb=upload_cb,
985
                        container_info_cache=container_info_cache,
986
                        **params)
987
                    if self['with_output'] or self['json_output']:
988
                        r['name'] = '/%s/%s' % (self.client.container, rpath)
989
                        uploaded.append(r)
990
                except Exception:
991
                    self._safe_progress_bar_finish(progress_bar)
992
                    raise
993
                finally:
994
                    self._safe_progress_bar_finish(progress_bar)
995
        self._optional_output(uploaded)
996
        self.error('Upload completed')
997

    
998
    def main(self, local_path, remote_path_or_url):
999
        super(self.__class__, self)._run(remote_path_or_url)
1000
        remote_path = self.path or path.basename(path.abspath(local_path))
1001
        self._run(local_path=local_path, remote_path=remote_path)
1002

    
1003

    
1004
class RangeArgument(ValueArgument):
1005
    """
1006
    :value type: string of the form <start>-<end> where <start> and <end> are
1007
        integers
1008
    :value returns: the input string, after type checking <start> and <end>
1009
    """
1010

    
1011
    @property
1012
    def value(self):
1013
        return getattr(self, '_value', self.default)
1014

    
1015
    @value.setter
1016
    def value(self, newvalues):
1017
        if newvalues:
1018
            self._value = getattr(self, '_value', self.default)
1019
            for newvalue in newvalues.split(','):
1020
                self._value = ('%s,' % self._value) if self._value else ''
1021
                start, sep, end = newvalue.partition('-')
1022
                if sep:
1023
                    if start:
1024
                        start, end = (int(start), int(end))
1025
                        if start > end:
1026
                            raise CLIInvalidArgument(
1027
                                'Invalid range %s' % newvalue, details=[
1028
                                'Valid range formats',
1029
                                '  START-END', '  UP_TO', '  -FROM',
1030
                                'where all values are integers',
1031
                                'OR a compination (csv), e.g.,',
1032
                                '  %s=5,10-20,-5' % self.lvalue])
1033
                        self._value += '%s-%s' % (start, end)
1034
                    else:
1035
                        self._value += '-%s' % int(end)
1036
                else:
1037
                    self._value += '%s' % int(start)
1038

    
1039

    
1040
@command(file_cmds)
1041
class file_cat(_pithos_container):
1042
    """Fetch remote file contents"""
1043

    
1044
    arguments = dict(
1045
        range=RangeArgument('show range of data e.g., 5,10-20,-5', '--range'),
1046
        if_match=ValueArgument('show output if ETags match', '--if-match'),
1047
        if_none_match=ValueArgument(
1048
            'show output if ETags match', '--if-none-match'),
1049
        if_modified_since=DateArgument(
1050
            'show output modified since then', '--if-modified-since'),
1051
        if_unmodified_since=DateArgument(
1052
            'show output unmodified since then', '--if-unmodified-since'),
1053
        object_version=ValueArgument(
1054
            'Get contents of the chosen version', '--object-version')
1055
    )
1056

    
1057
    @errors.generic.all
1058
    @errors.pithos.connection
1059
    @errors.pithos.container
1060
    @errors.pithos.object_path
1061
    def _run(self):
1062
        self.client.download_object(
1063
            self.path, self._out,
1064
            range_str=self['range'],
1065
            version=self['object_version'],
1066
            if_match=self['if_match'],
1067
            if_none_match=self['if_none_match'],
1068
            if_modified_since=self['if_modified_since'],
1069
            if_unmodified_since=self['if_unmodified_since'])
1070
        self._out.flush()
1071

    
1072
    def main(self, path_or_url):
1073
        super(self.__class__, self)._run(path_or_url)
1074
        self._run()
1075

    
1076

    
1077
@command(file_cmds)
1078
class file_download(_pithos_container):
1079
    """Download a remove file or directory object to local file system"""
1080

    
1081
    arguments = dict(
1082
        resume=FlagArgument(
1083
            'Resume/Overwrite (attempt resume, else overwrite)',
1084
            ('-f', '--resume')),
1085
        range=RangeArgument(
1086
            'Download only that range of data e.g., 5,10-20,-5', '--range'),
1087
        matching_etag=ValueArgument('download iff ETag match', '--if-match'),
1088
        non_matching_etag=ValueArgument(
1089
            'download iff ETags DO NOT match', '--if-none-match'),
1090
        modified_since_date=DateArgument(
1091
            'download iff remote file is modified since then',
1092
            '--if-modified-since'),
1093
        unmodified_since_date=DateArgument(
1094
            'show output iff remote file is unmodified since then',
1095
            '--if-unmodified-since'),
1096
        object_version=ValueArgument(
1097
            'download a file of a specific version', '--object-version'),
1098
        max_threads=IntArgument('default: 5', '--threads'),
1099
        progress_bar=ProgressBarArgument(
1100
            'do not show progress bar', ('-N', '--no-progress-bar'),
1101
            default=False),
1102
        recursive=FlagArgument(
1103
            'Download a remote directory object and its contents',
1104
            ('-r', '--recursive'))
1105
        )
1106

    
1107
    def _src_dst(self, local_path):
1108
        """Create a list of (src, dst) where src is a remote location and dst
1109
        is an open file descriptor. Directories are denoted as (None, dirpath)
1110
        and they are pretended to other objects in a very strict order (shorter
1111
        to longer path)."""
1112
        ret = []
1113
        try:
1114
            if self.path:
1115
                obj = self.client.get_object_info(
1116
                    self.path, version=self['object_version'])
1117
                obj.setdefault('name', self.path.strip('/'))
1118
            else:
1119
                obj = None
1120
        except ClientError as ce:
1121
            if ce.status in (404, ):
1122
                raiseCLIError(ce, details=[
1123
                    'To download an object, it must exist either as a file or'
1124
                    ' as a directory.',
1125
                    'For example, to download everything under prefix/ the '
1126
                    'directory "prefix" must exist.',
1127
                    'To see if an remote object is actually there:',
1128
                    '  /file info [/CONTAINER/]OBJECT',
1129
                    'To create a directory object:',
1130
                    '  /file mkdir [/CONTAINER/]OBJECT'])
1131
            if ce.status in (204, ):
1132
                raise CLIError(
1133
                    'No file or directory objects to download',
1134
                    details=[
1135
                        'To download a container (e.g., %s):' % self.container,
1136
                        '  [kamaki] container download %s [LOCAL_PATH]' % (
1137
                            self.container)])
1138
            raise
1139
        rpath = self.path.strip('/')
1140
        if local_path and self.path and local_path.endswith('/'):
1141
            local_path = local_path[-1:]
1142

    
1143
        if (not obj) or self._is_dir(obj):
1144
            if self['recursive']:
1145
                if not (self.path or local_path.endswith('/')):
1146
                    #  Download the whole container
1147
                    local_path = '' if local_path in ('.', ) else local_path
1148
                    local_path = '%s/' % (local_path or self.container)
1149
                obj = obj or dict(
1150
                    name='', content_type='application/directory')
1151
                dirs, files = [obj, ], []
1152
                objects = self.client.container_get(
1153
                    path=self.path,
1154
                    if_modified_since=self['modified_since_date'],
1155
                    if_unmodified_since=self['unmodified_since_date'])
1156
                for o in objects.json:
1157
                    (dirs if self._is_dir(o) else files).append(o)
1158

    
1159
                #  Put the directories on top of the list
1160
                for dpath in sorted(['%s%s' % (
1161
                        local_path, d['name'][len(rpath):]) for d in dirs]):
1162
                    if path.exists(dpath):
1163
                        if path.isdir(dpath):
1164
                            continue
1165
                        raise CLIError(
1166
                            'Cannot replace local file %s with a directory '
1167
                            'of the same name' % dpath,
1168
                            details=[
1169
                                'Either remove the file or specify a'
1170
                                'different target location'])
1171
                    ret.append((None, dpath, None))
1172

    
1173
                #  Append the file objects
1174
                for opath in [o['name'] for o in files]:
1175
                    lpath = '%s%s' % (local_path, opath[len(rpath):])
1176
                    if self['resume']:
1177
                        fxists = path.exists(lpath)
1178
                        if fxists and path.isdir(lpath):
1179
                            raise CLIError(
1180
                                'Cannot change local dir %s info file' % (
1181
                                    lpath),
1182
                                details=[
1183
                                    'Either remove the file or specify a'
1184
                                    'different target location'])
1185
                        ret.append((opath, lpath, fxists))
1186
                    elif path.exists(lpath):
1187
                        raise CLIError(
1188
                            'Cannot overwrite %s' % lpath,
1189
                            details=['To overwrite/resume, use  %s' % (
1190
                                self.arguments['resume'].lvalue)])
1191
                    else:
1192
                        ret.append((opath, lpath, None))
1193
            elif self.path:
1194
                raise CLIError(
1195
                    'Remote object /%s/%s is a directory' % (
1196
                        self.container, local_path),
1197
                    details=['Use %s to download directories' % (
1198
                        self.arguments['recursive'].lvalue)])
1199
            else:
1200
                parsed_name = self.arguments['recursive'].lvalue
1201
                raise CLIError(
1202
                    'Cannot download container %s' % self.container,
1203
                    details=[
1204
                        'Use %s to download containers' % parsed_name,
1205
                        '  [kamaki] file download %s /%s [LOCAL_PATH]' % (
1206
                            parsed_name, self.container)])
1207
        else:
1208
            #  Remote object is just a file
1209
            if path.exists(local_path):
1210
                if not self['resume']:
1211
                    raise CLIError(
1212
                        'Cannot overwrite local file %s' % (local_path),
1213
                        details=['To overwrite/resume, use  %s' % (
1214
                            self.arguments['resume'].lvalue)])
1215
            elif '/' in local_path[1:-1]:
1216
                dirs = [p for p in local_path.split('/') if p]
1217
                pref = '/' if local_path.startswith('/') else ''
1218
                for d in dirs[:-1]:
1219
                    pref += d
1220
                    if not path.exists(pref):
1221
                        ret.append((None, d, None))
1222
                    elif not path.isdir(pref):
1223
                        raise CLIError(
1224
                            'Failed to use %s as a destination' % local_path,
1225
                            importance=3,
1226
                            details=[
1227
                                'Local file %s is not a directory' % pref,
1228
                                'Destination prefix must consist of '
1229
                                'directories or non-existing names',
1230
                                'Either remove the file, or choose another '
1231
                                'destination'])
1232
            ret.append((rpath, local_path, self['resume']))
1233
        for r, l, resume in ret:
1234
            if r:
1235
                with open(l, 'rwb+' if resume else 'wb+') as f:
1236
                    yield (r, f)
1237
            else:
1238
                yield (r, l)
1239

    
1240
    @errors.generic.all
1241
    @errors.pithos.connection
1242
    @errors.pithos.container
1243
    @errors.pithos.object_path
1244
    @errors.pithos.local_path
1245
    @errors.pithos.local_path_download
1246
    def _run(self, local_path):
1247
        self.client.MAX_THREADS = int(self['max_threads'] or 5)
1248
        progress_bar = None
1249
        try:
1250
            for rpath, output_file in self._src_dst(local_path):
1251
                if not rpath:
1252
                    self.error('Create local directory %s' % output_file)
1253
                    makedirs(output_file)
1254
                    continue
1255
                self.error('/%s/%s --> %s' % (
1256
                    self.container, rpath, output_file.name))
1257
                progress_bar, download_cb = self._safe_progress_bar(
1258
                    '  download')
1259
                self.client.download_object(
1260
                    rpath, output_file,
1261
                    download_cb=download_cb,
1262
                    range_str=self['range'],
1263
                    version=self['object_version'],
1264
                    if_match=self['matching_etag'],
1265
                    resume=self['resume'],
1266
                    if_none_match=self['non_matching_etag'],
1267
                    if_modified_since=self['modified_since_date'],
1268
                    if_unmodified_since=self['unmodified_since_date'])
1269
        except KeyboardInterrupt:
1270
            from threading import activeCount, enumerate as activethreads
1271
            timeout = 0.5
1272
            while activeCount() > 1:
1273
                self._out.write('\nCancel %s threads: ' % (activeCount() - 1))
1274
                self._out.flush()
1275
                for thread in activethreads():
1276
                    try:
1277
                        thread.join(timeout)
1278
                        self._out.write('.' if thread.isAlive() else '*')
1279
                    except RuntimeError:
1280
                        continue
1281
                    finally:
1282
                        self._out.flush()
1283
                        timeout += 0.1
1284
            self.error('\nDownload canceled by user')
1285
            if local_path is not None:
1286
                self.error('to resume, re-run with --resume')
1287
        finally:
1288
            self._safe_progress_bar_finish(progress_bar)
1289

    
1290
    def main(self, remote_path_or_url, local_path=None):
1291
        super(self.__class__, self)._run(remote_path_or_url)
1292
        local_path = local_path or self.path or '.'
1293
        self._run(local_path=local_path)
1294

    
1295

    
1296
@command(container_cmds)
1297
class container_info(_pithos_account, _optional_json):
1298
    """Get information about a container"""
1299

    
1300
    arguments = dict(
1301
        until_date=DateArgument('show metadata until then', '--until'),
1302
        metadata=FlagArgument('Show only container metadata', '--metadata'),
1303
        sizelimit=FlagArgument(
1304
            'Show the maximum size limit for container', '--size-limit'),
1305
        in_bytes=FlagArgument('Show size limit in bytes', ('-b', '--bytes'))
1306
    )
1307

    
1308
    @errors.generic.all
1309
    @errors.pithos.connection
1310
    @errors.pithos.container
1311
    @errors.pithos.object_path
1312
    def _run(self):
1313
        if self['metadata']:
1314
            r, preflen = dict(), len('x-container-meta-')
1315
            for k, v in self.client.get_container_meta(
1316
                    until=self['until_date']).items():
1317
                r[k[preflen:]] = v
1318
        elif self['sizelimit']:
1319
            r = self.client.get_container_limit(
1320
                self.container)['x-container-policy-quota']
1321
            r = {'size limit': 'unlimited' if r in ('0', ) else (
1322
                int(r) if self['in_bytes'] else format_size(r))}
1323
        else:
1324
            r = self.client.get_container_info(self.container)
1325
        self._print(r, self.print_dict)
1326

    
1327
    def main(self, container):
1328
        super(self.__class__, self)._run()
1329
        self.container, self.client.container = container, container
1330
        self._run()
1331

    
1332

    
1333
class VersioningArgument(ValueArgument):
1334

    
1335
    schemes = ('auto', 'none')
1336

    
1337
    @property
1338
    def value(self):
1339
        return getattr(self, '_value', None)
1340

    
1341
    @value.setter
1342
    def value(self, new_scheme):
1343
        if new_scheme:
1344
            new_scheme = new_scheme.lower()
1345
            if new_scheme not in self.schemes:
1346
                raise CLIInvalidArgument('Invalid versioning value', details=[
1347
                    'Valid versioning values are %s' % ', '.join(
1348
                        self.schemes)])
1349
            self._value = new_scheme
1350

    
1351

    
1352
@command(container_cmds)
1353
class container_modify(_pithos_account, _optional_json):
1354
    """Modify the properties of a container"""
1355

    
1356
    arguments = dict(
1357
        metadata_to_add=KeyValueArgument(
1358
            'Add metadata in the form KEY=VALUE (can be repeated)',
1359
            '--metadata-add'),
1360
        metadata_to_delete=RepeatableArgument(
1361
            'Delete metadata by KEY (can be repeated)', '--metadata-del'),
1362
        sizelimit=DataSizeArgument(
1363
            'Set max size limit (0 for unlimited, '
1364
            'use units B, KiB, KB, etc.)', '--size-limit'),
1365
        versioning=VersioningArgument(
1366
            'Set a versioning scheme (%s)' % ', '.join(
1367
                VersioningArgument.schemes), '--versioning')
1368
    )
1369
    required = [
1370
        'metadata_to_add', 'metadata_to_delete', 'sizelimit', 'versioning']
1371

    
1372
    @errors.generic.all
1373
    @errors.pithos.connection
1374
    @errors.pithos.container
1375
    def _run(self, container):
1376
        metadata = self['metadata_to_add']
1377
        for k in (self['metadata_to_delete'] or []):
1378
            metadata[k] = ''
1379
        if metadata:
1380
            self.client.set_container_meta(metadata)
1381
            self._print(self.client.get_container_meta(), self.print_dict)
1382
        if self['sizelimit'] is not None:
1383
            self.client.set_container_limit(self['sizelimit'])
1384
            r = self.client.get_container_limit()['x-container-policy-quota']
1385
            r = 'unlimited' if r in ('0', ) else format_size(r)
1386
            self.writeln('new size limit: %s' % r)
1387
        if self['versioning']:
1388
            self.client.set_container_versioning(self['versioning'])
1389
            self.writeln('new versioning scheme: %s' % (
1390
                self.client.get_container_versioning(self.container)[
1391
                    'x-container-policy-versioning']))
1392

    
1393
    def main(self, container):
1394
        super(self.__class__, self)._run()
1395
        self.client.container, self.container = container, container
1396
        self._run(container=container)
1397

    
1398

    
1399
@command(container_cmds)
1400
class container_list(_pithos_account, _optional_json, _name_filter):
1401
    """List all containers, or their contents"""
1402

    
1403
    arguments = dict(
1404
        detail=FlagArgument('Containers with details', ('-l', '--list')),
1405
        limit=IntArgument('limit number of listed items', ('-n', '--number')),
1406
        marker=ValueArgument('output greater that marker', '--marker'),
1407
        modified_since_date=ValueArgument(
1408
            'show output modified since then', '--if-modified-since'),
1409
        unmodified_since_date=ValueArgument(
1410
            'show output not modified since then', '--if-unmodified-since'),
1411
        until_date=DateArgument('show metadata until then', '--until'),
1412
        shared=FlagArgument('show only shared', '--shared'),
1413
        more=FlagArgument('read long results', '--more'),
1414
        enum=FlagArgument('Enumerate results', '--enumerate'),
1415
        recursive=FlagArgument(
1416
            'Recursively list containers and their contents',
1417
            ('-r', '--recursive')),
1418
        shared_by_me=FlagArgument(
1419
            'show only files shared to other users', '--shared-by-me'),
1420
        public=FlagArgument('show only published objects', '--public'),
1421
    )
1422

    
1423
    def print_containers(self, container_list):
1424
        for index, container in enumerate(container_list):
1425
            if 'bytes' in container:
1426
                size = format_size(container['bytes'])
1427
            prfx = ('%s. ' % (index + 1)) if self['enum'] else ''
1428
            _cname = container['name'] if (
1429
                self['more']) else bold(container['name'])
1430
            cname = u'%s%s' % (prfx, _cname)
1431
            if self['detail']:
1432
                self.writeln(cname)
1433
                pretty_c = container.copy()
1434
                if 'bytes' in container:
1435
                    pretty_c['bytes'] = '%s (%s)' % (container['bytes'], size)
1436
                self.print_dict(pretty_c, exclude=('name'))
1437
                self.writeln()
1438
            else:
1439
                if 'count' in container and 'bytes' in container:
1440
                    self.writeln('%s (%s, %s objects)' % (
1441
                        cname, size, container['count']))
1442
                else:
1443
                    self.writeln(cname)
1444
            objects = container.get('objects', [])
1445
            if objects:
1446
                self.print_objects(objects)
1447
                self.writeln('')
1448

    
1449
    def _create_object_forest(self, container_list):
1450
        try:
1451
            for container in container_list:
1452
                self.client.container = container['name']
1453
                objects = self.client.container_get(
1454
                    limit=False if self['more'] else self['limit'],
1455
                    if_modified_since=self['modified_since_date'],
1456
                    if_unmodified_since=self['unmodified_since_date'],
1457
                    until=self['until_date'],
1458
                    show_only_shared=self['shared_by_me'],
1459
                    public=self['public'])
1460
                container['objects'] = objects.json
1461
        finally:
1462
            self.client.container = None
1463

    
1464
    @errors.generic.all
1465
    @errors.pithos.connection
1466
    @errors.pithos.object_path
1467
    @errors.pithos.container
1468
    def _run(self, container):
1469
        if container:
1470
            r = self.client.container_get(
1471
                limit=False if self['more'] else self['limit'],
1472
                marker=self['marker'],
1473
                if_modified_since=self['modified_since_date'],
1474
                if_unmodified_since=self['unmodified_since_date'],
1475
                until=self['until_date'],
1476
                show_only_shared=self['shared_by_me'],
1477
                public=self['public'])
1478
        else:
1479
            r = self.client.account_get(
1480
                limit=False if self['more'] else self['limit'],
1481
                marker=self['marker'],
1482
                if_modified_since=self['modified_since_date'],
1483
                if_unmodified_since=self['unmodified_since_date'],
1484
                until=self['until_date'],
1485
                show_only_shared=self['shared_by_me'],
1486
                public=self['public'])
1487
        files = self._filter_by_name(r.json)
1488
        if self['recursive'] and not container:
1489
            self._create_object_forest(files)
1490
        if self['more']:
1491
            outbu, self._out = self._out, StringIO()
1492
        try:
1493
            if self['json_output'] or self['output_format']:
1494
                self._print(files)
1495
            else:
1496
                (self.print_objects if container else self.print_containers)(
1497
                    files)
1498
        finally:
1499
            if self['more']:
1500
                pager(self._out.getvalue())
1501
                self._out = outbu
1502

    
1503
    def main(self, container=None):
1504
        super(self.__class__, self)._run()
1505
        self.client.container, self.container = container, container
1506
        self._run(container)
1507

    
1508

    
1509
@command(container_cmds)
1510
class container_create(_pithos_account):
1511
    """Create a new container"""
1512

    
1513
    arguments = dict(
1514
        versioning=ValueArgument(
1515
            'set container versioning (auto/none)', '--versioning'),
1516
        limit=IntArgument('set default container limit', '--limit'),
1517
        meta=KeyValueArgument(
1518
            'set container metadata (can be repeated)', '--meta')
1519
    )
1520

    
1521
    @errors.generic.all
1522
    @errors.pithos.connection
1523
    @errors.pithos.container
1524
    def _run(self, container):
1525
        try:
1526
            self.client.create_container(
1527
                container=container,
1528
                sizelimit=self['limit'],
1529
                versioning=self['versioning'],
1530
                metadata=self['meta'],
1531
                success=(201, ))
1532
        except ClientError as ce:
1533
            if ce.status in (202, ):
1534
                raise CLIError(
1535
                    'Container %s alread exists' % container, details=[
1536
                    'Either delete %s or choose another name' % (container)])
1537
            raise
1538

    
1539
    def main(self, new_container):
1540
        super(self.__class__, self)._run()
1541
        self._run(container=new_container)
1542

    
1543

    
1544
@command(container_cmds)
1545
class container_delete(_pithos_account):
1546
    """Delete a container"""
1547

    
1548
    arguments = dict(
1549
        yes=FlagArgument('Do not prompt for permission', '--yes'),
1550
        recursive=FlagArgument(
1551
            'delete container even if not empty', ('-r', '--recursive'))
1552
    )
1553

    
1554
    @errors.generic.all
1555
    @errors.pithos.connection
1556
    @errors.pithos.container
1557
    def _run(self, container):
1558
        num_of_contents = int(self.client.get_container_info(container)[
1559
            'x-container-object-count'])
1560
        delimiter, msg = None, 'Delete container %s ?' % container
1561
        if self['recursive']:
1562
            delimiter, msg = '/', 'Empty and d%s' % msg[1:]
1563
        elif num_of_contents:
1564
            raise CLIError('Container %s is not empty' % container, details=[
1565
                'Use %s to delete non-empty containers' % (
1566
                    self.arguments['recursive'].lvalue)])
1567
        if self['yes'] or self.ask_user(msg):
1568
            if num_of_contents:
1569
                self.client.del_container(delimiter=delimiter)
1570
            self.client.purge_container()
1571

    
1572
    def main(self, container):
1573
        super(self.__class__, self)._run()
1574
        self.container, self.client.container = container, container
1575
        self._run(container)
1576

    
1577

    
1578
@command(container_cmds)
1579
class container_empty(_pithos_account):
1580
    """Empty a container"""
1581

    
1582
    arguments = dict(yes=FlagArgument('Do not prompt for permission', '--yes'))
1583

    
1584
    @errors.generic.all
1585
    @errors.pithos.connection
1586
    @errors.pithos.container
1587
    def _run(self, container):
1588
        if self['yes'] or self.ask_user('Empty container %s ?' % container):
1589
            self.client.del_container(delimiter='/')
1590

    
1591
    def main(self, container):
1592
        super(self.__class__, self)._run()
1593
        self.container, self.client.container = container, container
1594
        self._run(container)
1595

    
1596

    
1597
@command(sharer_cmds)
1598
class sharer_list(_pithos_account, _optional_json):
1599
    """List accounts who share file objects with current user"""
1600

    
1601
    arguments = dict(
1602
        detail=FlagArgument('show detailed output', ('-l', '--details')),
1603
        marker=ValueArgument('show output greater then marker', '--marker')
1604
    )
1605

    
1606
    @errors.generic.all
1607
    @errors.pithos.connection
1608
    def _run(self):
1609
        accounts = self.client.get_sharing_accounts(marker=self['marker'])
1610
        if not (self['json_output'] or self['output_format']):
1611
            usernames = self._uuids2usernames(
1612
                [acc['name'] for acc in accounts])
1613
            for item in accounts:
1614
                uuid = item['name']
1615
                item['id'], item['name'] = uuid, usernames[uuid]
1616
                if not self['detail']:
1617
                    item.pop('last_modified')
1618
        self._print(accounts)
1619

    
1620
    def main(self):
1621
        super(self.__class__, self)._run()
1622
        self._run()
1623

    
1624

    
1625
@command(sharer_cmds)
1626
class sharer_info(_pithos_account, _optional_json):
1627
    """Details on a Pithos+ sharer account (default: current account)"""
1628

    
1629
    @errors.generic.all
1630
    @errors.pithos.connection
1631
    def _run(self):
1632
        self._print(self.client.get_account_info(), self.print_dict)
1633

    
1634
    def main(self, account_uuid=None):
1635
        super(self.__class__, self)._run()
1636
        if account_uuid:
1637
            self.client.account, self.account = account_uuid, account_uuid
1638
        self._run()
1639

    
1640

    
1641
class _pithos_group(_pithos_account):
1642
    prefix = 'x-account-group-'
1643
    preflen = len(prefix)
1644

    
1645
    def _groups(self):
1646
        groups = dict()
1647
        for k, v in self.client.get_account_group().items():
1648
            groups[k[self.preflen:]] = v
1649
        return groups
1650

    
1651

    
1652
@command(group_cmds)
1653
class group_list(_pithos_group, _optional_json):
1654
    """list all groups and group members"""
1655

    
1656
    @errors.generic.all
1657
    @errors.pithos.connection
1658
    def _run(self):
1659
        self._print(self._groups(), self.print_dict)
1660

    
1661
    def main(self):
1662
        super(self.__class__, self)._run()
1663
        self._run()
1664

    
1665

    
1666
@command(group_cmds)
1667
class group_create(_pithos_group, _optional_json):
1668
    """Create a group of users"""
1669

    
1670
    arguments = dict(
1671
        user_uuid=RepeatableArgument('Add a user to the group', '--uuid'),
1672
        username=RepeatableArgument('Add a user to the group', '--username')
1673
    )
1674
    required = ['user_uuid', 'username']
1675

    
1676
    @errors.generic.all
1677
    @errors.pithos.connection
1678
    def _run(self, groupname, *users):
1679
        if groupname in self._groups() and not self.ask_user(
1680
                'Group %s already exists, overwrite?' % groupname):
1681
            self.error('Aborted')
1682
            return
1683
        self.client.set_account_group(groupname, users)
1684
        self._print(self._groups(), self.print_dict)
1685

    
1686
    def main(self, groupname):
1687
        super(self.__class__, self)._run()
1688
        users = (self['user_uuid'] or []) + self._usernames2uuids(
1689
            self['username'] or []).values()
1690
        if users:
1691
            self._run(groupname, *users)
1692
        else:
1693
            raise CLISyntaxError(
1694
                'No valid users specified, use %s or %s' % (
1695
                    self.arguments['user_uuid'].lvalue,
1696
                    self.arguments['username'].lvalue),
1697
                details=[
1698
                    'Check if a username or uuid is valid with',
1699
                    '  user uuid2username', 'OR', '  user username2uuid'])
1700

    
1701

    
1702
@command(group_cmds)
1703
class group_delete(_pithos_group, _optional_json):
1704
    """Delete a user group"""
1705

    
1706
    @errors.generic.all
1707
    @errors.pithos.connection
1708
    def _run(self, groupname):
1709
        self.client.del_account_group(groupname)
1710
        self._print(self._groups(), self.print_dict)
1711

    
1712
    def main(self, groupname):
1713
        super(self.__class__, self)._run()
1714
        self._run(groupname)
1715

    
1716

    
1717
#  Deprecated commands
1718

    
1719
@command(file_cmds)
1720
class file_publish(_pithos_init):
1721
    """DEPRECATED, replaced by [kamaki] file modify OBJECT --publish"""
1722

    
1723
    def main(self, *args):
1724
        raise CLISyntaxError('DEPRECATED', details=[
1725
            'This command is replaced by:',
1726
            '  [kamaki] file modify OBJECT --publish'])
1727

    
1728

    
1729
@command(file_cmds)
1730
class file_unpublish(_pithos_init):
1731
    """DEPRECATED, replaced by [kamaki] file modify OBJECT --unpublish"""
1732

    
1733
    def main(self, *args):
1734
        raise CLISyntaxError('DEPRECATED', details=[
1735
            'This command is replaced by:',
1736
            '  [kamaki] file modify OBJECT --unpublish'])