Revision cec2dfcd kamaki/cli/commands/pithos.py

b/kamaki/cli/commands/pithos.py
31 31
# interpreted as representing official policies, either expressed
32 32
# or implied, of GRNET S.A.command
33 33

  
34
from time import localtime, strftime
35
from os import path, makedirs, walk
36 34
from io import StringIO
37 35
from pydoc import pager
38 36

  
39 37
from kamaki.cli import command
40 38
from kamaki.cli.command_tree import CommandTree
41
from kamaki.cli.errors import (
42
    raiseCLIError, CLISyntaxError, CLIBaseUrlError, CLIInvalidArgument)
43
from kamaki.cli.utils import (
44
    format_size, to_bytes, bold, get_path_size, guess_mime_type)
45
from kamaki.cli.argument import FlagArgument, ValueArgument, IntArgument
46
from kamaki.cli.argument import KeyValueArgument, DateArgument
47
from kamaki.cli.argument import ProgressBarArgument
48
from kamaki.cli.commands import _command_init, errors
49
from kamaki.cli.commands import addLogSettings, DontRaiseKeyError
50 39
from kamaki.cli.commands import (
51
    _optional_output_cmd, _optional_json, _name_filter)
52
from kamaki.clients.pithos import PithosClient, ClientError
53
from kamaki.clients.astakos import AstakosClient
54

  
55
pithos_cmds = CommandTree('file', 'Pithos+/Storage API commands')
56
_commands = [pithos_cmds]
57

  
58

  
59
# Argument functionality
60

  
61

  
62
class SharingArgument(ValueArgument):
63
    """Set sharing (read and/or write) groups
64
    .
65
    :value type: "read=term1,term2,... write=term1,term2,..."
66
    .
67
    :value returns: {'read':['term1', 'term2', ...],
68
    .   'write':['term1', 'term2', ...]}
69
    """
70

  
71
    @property
72
    def value(self):
73
        return getattr(self, '_value', self.default)
74

  
75
    @value.setter
76
    def value(self, newvalue):
77
        perms = {}
78
        try:
79
            permlist = newvalue.split(' ')
80
        except AttributeError:
81
            return
82
        for p in permlist:
83
            try:
84
                (key, val) = p.split('=')
85
            except ValueError as err:
86
                raiseCLIError(
87
                    err,
88
                    'Error in --sharing',
89
                    details='Incorrect format',
90
                    importance=1)
91
            if key.lower() not in ('read', 'write'):
92
                msg = 'Error in --sharing'
93
                raiseCLIError(err, msg, importance=1, details=[
94
                    'Invalid permission key %s' % key])
95
            val_list = val.split(',')
96
            if not key in perms:
97
                perms[key] = []
98
            for item in val_list:
99
                if item not in perms[key]:
100
                    perms[key].append(item)
101
        self._value = perms
102

  
103

  
104
class RangeArgument(ValueArgument):
105
    """
106
    :value type: string of the form <start>-<end> where <start> and <end> are
107
        integers
108
    :value returns: the input string, after type checking <start> and <end>
109
    """
110

  
111
    @property
112
    def value(self):
113
        return getattr(self, '_value', self.default)
114

  
115
    @value.setter
116
    def value(self, newvalues):
117
        if not newvalues:
118
            self._value = self.default
119
            return
120
        self._value = ''
121
        for newvalue in newvalues.split(','):
122
            self._value = ('%s,' % self._value) if self._value else ''
123
            start, sep, end = newvalue.partition('-')
124
            if sep:
125
                if start:
126
                    start, end = (int(start), int(end))
127
                    assert start <= end, 'Invalid range value %s' % newvalue
128
                    self._value += '%s-%s' % (int(start), int(end))
129
                else:
130
                    self._value += '-%s' % int(end)
131
            else:
132
                self._value += '%s' % int(start)
133

  
40
    _command_init, errors, addLogSettings, DontRaiseKeyError, _optional_json,
41
    _name_filter, _optional_output_cmd)
42
from kamaki.clients.pithos import PithosClient
43
from kamaki.cli.errors import (
44
    CLIBaseUrlError)
45
from kamaki.cli.argument import (
46
    FlagArgument, IntArgument, ValueArgument, DateArgument)
47
from kamaki.cli.utils import (format_size, bold)
134 48

  
135
# Command specs
49
file_cmds = CommandTree('file', 'Pithos+/Storage object level API commands')
50
container_cmds = CommandTree(
51
    'container', 'Pithos+/Storage container level API commands')
52
sharers_commands = CommandTree('sharers', 'Pithos+/Storage sharers')
53
_commands = [file_cmds, container_cmds, sharers_commands]
136 54

  
137 55

  
138 56
class _pithos_init(_command_init):
139
    """Initialize a pithos+ kamaki client"""
140

  
141
    @staticmethod
142
    def _is_dir(remote_dict):
143
        return 'application/directory' == remote_dict.get(
144
            'content_type', remote_dict.get('content-type', ''))
57
    """Initilize a pithos+ client
58
    There is always a default account (current user uuid)
59
    There is always a default container (pithos)
60
    """
145 61

  
146 62
    @DontRaiseKeyError
147 63
    def _custom_container(self):
......
155 71
        self.account = self._custom_uuid()
156 72
        if self.account:
157 73
            return
158
        if getattr(self, 'auth_base', False):
159
            self.account = self.auth_base.user_term('id', self.token)
74
        astakos = getattr(self, 'auth_base', None)
75
        if astakos:
76
            self.account = astakos.user_term('id', self.token)
160 77
        else:
161
            astakos_url = self._custom_url('astakos')
162
            astakos_token = self._custom_token('astakos') or self.token
163
            if not astakos_url:
164
                raise CLIBaseUrlError(service='astakos')
165
            astakos = AstakosClient(astakos_url, astakos_token)
166
            self.account = astakos.user_term('id')
78
            raise CLIBaseUrlError(service='astakos')
167 79

  
168 80
    @errors.generic.all
169 81
    @addLogSettings
170 82
    def _run(self):
171
        self.base_url = None
172
        if getattr(self, 'cloud', None):
83
        cloud = getattr(self, 'cloud', None)
84
        if cloud:
173 85
            self.base_url = self._custom_url('pithos')
174 86
        else:
175 87
            self.cloud = 'default'
176 88
        self.token = self._custom_token('pithos')
177
        self.container = self._custom_container()
89
        self.container = self._custom_container() or 'pithos'
178 90

  
179
        if getattr(self, 'auth_base', False):
180
            self.token = self.token or self.auth_base.token
91
        astakos = getattr(self, 'auth_base', None)
92
        if astakos:
93
            self.token = self.token or astakos.token
181 94
            if not self.base_url:
182
                pithos_endpoints = self.auth_base.get_service_endpoints(
95
                pithos_endpoints = astakos.get_service_endpoints(
183 96
                    self._custom_type('pithos') or 'object-store',
184 97
                    self._custom_version('pithos') or '')
185 98
                self.base_url = pithos_endpoints['publicURL']
186
        elif not self.base_url:
187
            raise CLIBaseUrlError(service='pithos')
99
        else:
100
            raise CLIBaseUrlError(service='astakos')
188 101

  
189 102
        self._set_account()
190 103
        self.client = PithosClient(
191
            base_url=self.base_url,
192
            token=self.token,
193
            account=self.account,
194
            container=self.container)
104
            self.base_url, self.token, self.account, self.container)
195 105

  
196 106
    def main(self):
197 107
        self._run()
198 108

  
199 109

  
200
class _file_account_command(_pithos_init):
201
    """Base class for account level storage commands"""
110
class _pithos_account(_pithos_init):
111
    """Setup account"""
202 112

  
203
    def __init__(self, arguments={}, auth_base=None, cloud=None):
204
        super(_file_account_command, self).__init__(
205
            arguments, auth_base, cloud)
113
    def __init__(self, *args, **kwargs):
114
        super(_pithos_account, self).__init__(*args, **kwargs)
206 115
        self['account'] = ValueArgument(
207
            'Set user account (not permanent)', ('-A', '--account'))
208

  
209
    def _run(self, custom_account=None):
210
        super(_file_account_command, self)._run()
211
        if custom_account:
212
            self.client.account = custom_account
213
        elif self['account']:
214
            self.client.account = self['account']
215

  
216
    @errors.generic.all
217
    def main(self):
218
        self._run()
116
            'Use (a different) user uuid', ('-A', '--account'))
219 117

  
118
    def _run(self):
119
        super(_pithos_account, self)._run()
120
        self.client.account = self['account'] or getattr(
121
            self, 'account', getattr(self.client, 'account', None))
220 122

  
221
class _file_container_command(_file_account_command):
222
    """Base class for container level storage commands"""
223 123

  
224
    container = None
225
    path = None
124
class _pithos_container(_pithos_account):
125
    """Setup container"""
226 126

  
227
    def __init__(self, arguments={}, auth_base=None, cloud=None):
228
        super(_file_container_command, self).__init__(
229
            arguments, auth_base, cloud)
127
    def __init__(self, *args, **kwargs):
128
        super(_pithos_container, self).__init__(*args, **kwargs)
230 129
        self['container'] = ValueArgument(
231
            'Set container to work with (temporary)', ('-C', '--container'))
130
            'Use this container (default: pithos)', ('-C', '--container'))
232 131

  
233
    def extract_container_and_path(
234
            self, container_with_path, path_is_optional=True):
235
        """Contains all heuristics for deciding what should be used as
236
        container or path. Options are:
237
        * user string of the form container:path
238
        * self.container, self.path variables set by super constructor, or
239
        explicitly by the caller application
240
        Error handling is explicit as these error cases happen only here
132
    def _resolve_pithos_url(self, url):
133
        """Match urls of one of the following formats:
134
        pithos://ACCOUNT/CONTAINER/OBJECT_PATH
135
        /CONTAINER/OBJECT_PATH
136
        Anything resolved, is set as self.<account|container|path>
241 137
        """
242
        try:
243
            assert isinstance(container_with_path, str)
244
        except AssertionError as err:
245
            if self['container'] and path_is_optional:
246
                self.container = self['container']
247
                self.client.container = self['container']
248
                return
249
            raiseCLIError(err)
138
        account, container, path, prefix = '', '', url, 'pithos://'
139
        if url.startswith(prefix):
140
            self.account, sep, url = url[len(prefix):].partition('/')
141
            url = '/%s' % url
142
        if url.startswith('/'):
143
            self.container, sep, path = url[1:].partition('/')
144
        self.path = path
250 145

  
251
        user_cont, sep, userpath = container_with_path.partition(':')
146
    def _run(self, url=None):
147
        super(_pithos_container, self)._run()
148
        self._resolve_pithos_url(url or '')
149
        self.client.container = self['container'] or getattr(
150
            self, 'container', None) or getattr(self.client, 'container', '')
252 151

  
253
        if sep:
254
            if not user_cont:
255
                raiseCLIError(CLISyntaxError(
256
                    'Container is missing\n',
257
                    details=errors.pithos.container_howto))
258
            alt_cont = self['container']
259
            if alt_cont and user_cont != alt_cont:
260
                raiseCLIError(CLISyntaxError(
261
                    'Conflict: 2 containers (%s, %s)' % (user_cont, alt_cont),
262
                    details=errors.pithos.container_howto)
263
                )
264
            self.container = user_cont
265
            if not userpath:
266
                raiseCLIError(CLISyntaxError(
267
                    'Path is missing for object in container %s' % user_cont,
268
                    details=errors.pithos.container_howto)
269
                )
270
            self.path = userpath
271
        else:
272
            alt_cont = self['container'] or self.client.container
273
            if alt_cont:
274
                self.container = alt_cont
275
                self.path = user_cont
276
            elif path_is_optional:
277
                self.container = user_cont
278
                self.path = None
279
            else:
280
                self.container = user_cont
281
                raiseCLIError(CLISyntaxError(
282
                    'Both container and path are required',
283
                    details=errors.pithos.container_howto)
284
                )
285

  
286
    @errors.generic.all
287
    def _run(self, container_with_path=None, path_is_optional=True):
288
        super(_file_container_command, self)._run()
289
        if self['container']:
290
            self.client.container = self['container']
291
            if container_with_path:
292
                self.path = container_with_path
293
            elif not path_is_optional:
294
                raise CLISyntaxError(
295
                    'Both container and path are required',
296
                    details=errors.pithos.container_howto)
297
        elif container_with_path:
298
            self.extract_container_and_path(
299
                container_with_path,
300
                path_is_optional)
301
            self.client.container = self.container
302
        self.container = self.client.container
303

  
304
    def main(self, container_with_path=None, path_is_optional=True):
305
        self._run(container_with_path, path_is_optional)
306 152

  
307

  
308
@command(pithos_cmds)
309
class file_list(_file_container_command, _optional_json, _name_filter):
310
    """List containers, object trees or objects in a directory
311
    Use with:
312
    1 no parameters : containers in current account
313
    2. one parameter (container) or --container : contents of container
314
    3. <container>:<prefix> or --container=<container> <prefix>: objects in
315
    .   container starting with prefix
316
    """
153
@command(file_cmds)
154
class file_list(_pithos_container, _optional_json, _name_filter):
155
    """List all objects in a container or a directory object"""
317 156

  
318 157
    arguments = dict(
319 158
        detail=FlagArgument('detailed output', ('-l', '--list')),
320 159
        limit=IntArgument('limit number of listed items', ('-n', '--number')),
321 160
        marker=ValueArgument('output greater that marker', '--marker'),
322 161
        delimiter=ValueArgument('show output up to delimiter', '--delimiter'),
323
        path=ValueArgument(
324
            'show output starting with prefix up to /', '--path'),
325 162
        meta=ValueArgument(
326 163
            'show output with specified meta keys', '--meta',
327 164
            default=[]),
......
334 171
            'format to parse until data (default: d/m/Y H:M:S )', '--format'),
335 172
        shared=FlagArgument('show only shared', '--shared'),
336 173
        more=FlagArgument('read long results', '--more'),
337
        exact_match=FlagArgument(
338
            'Show only objects that match exactly with path',
339
            '--exact-match'),
340 174
        enum=FlagArgument('Enumerate results', '--enumerate'),
341 175
        recursive=FlagArgument(
342 176
            'Recursively list containers and their contents',
......
345 179

  
346 180
    def print_objects(self, object_list):
347 181
        for index, obj in enumerate(object_list):
348
            if self['exact_match'] and self.path and not (
349
                    obj['name'] == self.path or 'content_type' in obj):
350
                continue
351 182
            pretty_obj = obj.copy()
352 183
            index += 1
353 184
            empty_space = ' ' * (len(str(len(object_list))) - len(str(index)))
354 185
            if 'subdir' in obj:
355 186
                continue
356 187
            if obj['content_type'] == 'application/directory':
357
                isDir = True
358
                size = 'D'
188
                isDir, size = True, 'D'
359 189
            else:
360
                isDir = False
361
                size = format_size(obj['bytes'])
190
                isDir, size = False, format_size(obj['bytes'])
362 191
                pretty_obj['bytes'] = '%s (%s)' % (obj['bytes'], size)
363 192
            oname = obj['name'] if self['more'] else bold(obj['name'])
364
            prfx = (
365
                '%s%s. ' % (empty_space, index)) if self['enum'] else ''
193
            prfx = ('%s%s. ' % (empty_space, index)) if self['enum'] else ''
366 194
            if self['detail']:
367 195
                self.writeln('%s%s' % (prfx, oname))
368 196
                self.print_dict(pretty_obj, exclude=('name'))
......
372 200
                oname += '/' if isDir else u''
373 201
                self.writeln(oname)
374 202

  
375
    def print_containers(self, container_list):
376
        for index, container in enumerate(container_list):
377
            if 'bytes' in container:
378
                size = format_size(container['bytes'])
379
            prfx = ('%s. ' % (index + 1)) if self['enum'] else ''
380
            _cname = container['name'] if (
381
                self['more']) else bold(container['name'])
382
            cname = u'%s%s' % (prfx, _cname)
383
            if self['detail']:
384
                self.writeln(cname)
385
                pretty_c = container.copy()
386
                if 'bytes' in container:
387
                    pretty_c['bytes'] = '%s (%s)' % (container['bytes'], size)
388
                self.print_dict(pretty_c, exclude=('name'))
389
                self.writeln()
390
            else:
391
                if 'count' in container and 'bytes' in container:
392
                    self.writeln('%s (%s, %s objects)' % (
393
                        cname, size, container['count']))
394
                else:
395
                    self.writeln(cname)
396
            objects = container.get('objects', [])
397
            if objects:
398
                self.print_objects(objects)
399
                self.writeln('')
400

  
401
    def _argument_context_check(self):
402
        container_level_only = ('recursive', )
403
        object_level_only = ('delimiter', 'path', 'exact_match')
404
        details, mistake = [], ''
405
        if self.container:
406
            for term in container_level_only:
407
                if self[term]:
408
                    details = [
409
                        'This is a container-level argument',
410
                        'Use it without a <container> parameter']
411
                    mistake = self.arguments[term]
412
        else:
413
            for term in object_level_only:
414
                if not self['recursive'] and self[term]:
415
                    details = [
416
                        'This is an opbject-level argument',
417
                        'Use it with a <container> parameter',
418
                        'or with the -R/--recursive argument']
419
                    mistake = self.arguments[term]
420
        if mistake and details:
421
            raise CLIInvalidArgument(
422
                'Invalid use of %s argument' % '/'.join(mistake.parsed_name),
423
                details=details + ['Try --help for more details'])
424

  
425
    def _create_object_forest(self, container_list):
426
        try:
427
            for container in container_list:
428
                self.client.container = container['name']
429
                objects = self.client.container_get(
430
                    limit=False if self['more'] else self['limit'],
431
                    marker=self['marker'],
432
                    delimiter=self['delimiter'],
433
                    path=self['path'],
434
                    if_modified_since=self['if_modified_since'],
435
                    if_unmodified_since=self['if_unmodified_since'],
436
                    until=self['until'],
437
                    meta=self['meta'],
438
                    show_only_shared=self['shared'])
439
                container['objects'] = objects.json
440
        finally:
441
            self.client.container = None
442

  
443 203
    @errors.generic.all
444 204
    @errors.pithos.connection
445
    @errors.pithos.object_path
446 205
    @errors.pithos.container
206
    @errors.pithos.object_path
447 207
    def _run(self):
448
        files, prnt = None, None
449
        self._argument_context_check()
450
        if not self.container:
451
            r = self.client.account_get(
452
                limit=False if self['more'] else self['limit'],
453
                marker=self['marker'],
454
                if_modified_since=self['if_modified_since'],
455
                if_unmodified_since=self['if_unmodified_since'],
456
                until=self['until'],
457
                show_only_shared=self['shared'])
458
            files, prnt = self._filter_by_name(r.json), self.print_containers
459
            if self['recursive']:
460
                self._create_object_forest(files)
461
        else:
462
            prefix = (
463
                self.path if not self['name'] else '') or self['name_pref']
464
            r = self.client.container_get(
465
                limit=False if self['more'] else self['limit'],
466
                marker=self['marker'],
467
                prefix=prefix,
468
                delimiter=self['delimiter'],
469
                path=self['path'],
470
                if_modified_since=self['if_modified_since'],
471
                if_unmodified_since=self['if_unmodified_since'],
472
                until=self['until'],
473
                meta=self['meta'],
474
                show_only_shared=self['shared'])
475
            files, prnt = self._filter_by_name(r.json), self.print_objects
208
        r = self.client.container_get(
209
            limit=False if self['more'] else self['limit'],
210
            marker=self['marker'],
211
            prefix=self['name_pref'] or '/',
212
            delimiter=self['delimiter'],
213
            path=self.path or '',
214
            if_modified_since=self['if_modified_since'],
215
            if_unmodified_since=self['if_unmodified_since'],
216
            until=self['until'],
217
            meta=self['meta'],
218
            show_only_shared=self['shared'])
219
        files = self._filter_by_name(r.json)
476 220
        if self['more']:
477 221
            outbu, self._out = self._out, StringIO()
478 222
        try:
479 223
            if self['json_output'] or self['output_format']:
480 224
                self._print(files)
481 225
            else:
482
                prnt(files)
226
                self.print_objects(files)
483 227
        finally:
484 228
            if self['more']:
485 229
                pager(self._out.getvalue())
486 230
                self._out = outbu
487 231

  
488
    def main(self, container____path__=None):
489
        super(self.__class__, self)._run(container____path__)
232
    def main(self, path_or_url='/'):
233
        super(self.__class__, self)._run(path_or_url)
490 234
        self._run()
491 235

  
492 236

  
493
@command(pithos_cmds)
494
class file_mkdir(_file_container_command, _optional_output_cmd):
495
    """Create a directory
496
    Kamaki hanldes directories the same way as OOS Storage and Pithos+:
497
    A directory  is   an  object  with  type  "application/directory"
498
    An object with path  dir/name can exist even if  dir does not exist
499
    or even if dir  is  a non  directory  object.  Users can modify dir '
500
    without affecting the dir/name object in any way.
501
    """
502

  
503
    @errors.generic.all
504
    @errors.pithos.connection
505
    @errors.pithos.container
506
    def _run(self):
507
        self._optional_output(self.client.create_directory(self.path))
508

  
509
    def main(self, container___directory):
510
        super(self.__class__, self)._run(
511
            container___directory, path_is_optional=False)
512
        self._run()
513

  
514

  
515
@command(pithos_cmds)
516
class file_touch(_file_container_command, _optional_output_cmd):
517
    """Create an empty object (file)
518
    If object exists, this command will reset it to 0 length
519
    """
237
@command(file_cmds)
238
class file_create(_pithos_container, _optional_output_cmd):
239
    """Create an empty remove file"""
520 240

  
521 241
    arguments = dict(
522 242
        content_type=ValueArgument(
......
525 245
            default='application/octet-stream')
526 246
    )
527 247

  
528
    @errors.generic.all
529
    @errors.pithos.connection
530
    @errors.pithos.container
531 248
    def _run(self):
532 249
        self._optional_output(
533 250
            self.client.create_object(self.path, self['content_type']))
534 251

  
535
    def main(self, container___path):
536
        super(file_touch, self)._run(container___path, path_is_optional=False)
537
        self._run()
538

  
539

  
540
@command(pithos_cmds)
541
class file_create(_file_container_command, _optional_output_cmd):
542
    """Create a container"""
543

  
544
    arguments = dict(
545
        versioning=ValueArgument(
546
            'set container versioning (auto/none)', '--versioning'),
547
        limit=IntArgument('set default container limit', '--limit'),
548
        meta=KeyValueArgument(
549
            'set container metadata (can be repeated)', '--meta')
550
    )
551

  
552
    @errors.generic.all
553
    @errors.pithos.connection
554
    @errors.pithos.container
555
    def _run(self, container):
556
        try:
557
            self._optional_output(self.client.create_container(
558
                container=container,
559
                sizelimit=self['limit'],
560
                versioning=self['versioning'],
561
                metadata=self['meta'],
562
                success=(201, )))
563
        except ClientError as ce:
564
            if ce.status in (202, ):
565
                raiseCLIError(ce, 'Container %s alread exists' % container)
566

  
567
    def main(self, container=None):
568
        super(self.__class__, self)._run(container)
569
        if container and self.container != container:
570
            raiseCLIError('Invalid container name %s' % container, details=[
571
                'Did you mean "%s" ?' % self.container,
572
                'Use --container for names containing :'])
573
        self._run(container)
574

  
575

  
576
class _source_destination_command(_file_container_command):
577

  
578
    arguments = dict(
579
        destination_account=ValueArgument('', ('-a', '--dst-account')),
580
        recursive=FlagArgument('', ('-R', '--recursive')),
581
        prefix=FlagArgument('', '--with-prefix', default=''),
582
        suffix=ValueArgument('', '--with-suffix', default=''),
583
        add_prefix=ValueArgument('', '--add-prefix', default=''),
584
        add_suffix=ValueArgument('', '--add-suffix', default=''),
585
        prefix_replace=ValueArgument('', '--prefix-to-replace', default=''),
586
        suffix_replace=ValueArgument('', '--suffix-to-replace', default=''),
587
    )
588

  
589
    def __init__(self, arguments={}, auth_base=None, cloud=None):
590
        self.arguments.update(arguments)
591
        super(_source_destination_command, self).__init__(
592
            self.arguments, auth_base, cloud)
593

  
594
    def _run(self, source_container___path, path_is_optional=False):
595
        super(_source_destination_command, self)._run(
596
            source_container___path, path_is_optional)
597
        self.dst_client = PithosClient(
598
            base_url=self.client.base_url,
599
            token=self.client.token,
600
            account=self['destination_account'] or self.client.account)
601

  
602
    @errors.generic.all
603
    @errors.pithos.account
604
    def _dest_container_path(self, dest_container_path):
605
        if self['destination_container']:
606
            self.dst_client.container = self['destination_container']
607
            return (self['destination_container'], dest_container_path)
608
        if dest_container_path:
609
            dst = dest_container_path.split(':')
610
            if len(dst) > 1:
611
                try:
612
                    self.dst_client.container = dst[0]
613
                    self.dst_client.get_container_info(dst[0])
614
                except ClientError as err:
615
                    if err.status in (404, 204):
616
                        raiseCLIError(
617
                            'Destination container %s not found' % dst[0])
618
                    raise
619
                else:
620
                    self.dst_client.container = dst[0]
621
                return (dst[0], dst[1])
622
            return(None, dst[0])
623
        raiseCLIError('No destination container:path provided')
624

  
625
    def _get_all(self, prefix):
626
        return self.client.container_get(prefix=prefix).json
627

  
628
    def _get_src_objects(self, src_path, source_version=None):
629
        """Get a list of the source objects to be called
630

  
631
        :param src_path: (str) source path
632

  
633
        :returns: (method, params) a method that returns a list when called
634
        or (object) if it is a single object
635
        """
636
        if src_path and src_path[-1] == '/':
637
            src_path = src_path[:-1]
638

  
639
        if self['prefix']:
640
            return (self._get_all, dict(prefix=src_path))
641
        try:
642
            srcobj = self.client.get_object_info(
643
                src_path, version=source_version)
644
        except ClientError as srcerr:
645
            if srcerr.status == 404:
646
                raiseCLIError(
647
                    'Source object %s not in source container %s' % (
648
                        src_path, self.client.container),
649
                    details=['Hint: --with-prefix to match multiple objects'])
650
            elif srcerr.status not in (204,):
651
                raise
652
            return (self.client.list_objects, {})
653

  
654
        if self._is_dir(srcobj):
655
            if not self['recursive']:
656
                raiseCLIError(
657
                    'Object %s of cont. %s is a dir' % (
658
                        src_path, self.client.container),
659
                    details=['Use --recursive to access directories'])
660
            return (self._get_all, dict(prefix=src_path))
661
        srcobj['name'] = src_path
662
        return srcobj
663

  
664
    def src_dst_pairs(self, dst_path, source_version=None):
665
        src_iter = self._get_src_objects(self.path, source_version)
666
        src_N = isinstance(src_iter, tuple)
667
        add_prefix = self['add_prefix'].strip('/')
668

  
669
        if dst_path and dst_path.endswith('/'):
670
            dst_path = dst_path[:-1]
671

  
672
        try:
673
            dstobj = self.dst_client.get_object_info(dst_path)
674
        except ClientError as trgerr:
675
            if trgerr.status in (404,):
676
                if src_N:
677
                    raiseCLIError(
678
                        'Cannot merge multiple paths to path %s' % dst_path,
679
                        details=[
680
                            'Try to use / or a directory as destination',
681
                            'or create the destination dir (/file mkdir)',
682
                            'or use a single object as source'])
683
            elif trgerr.status not in (204,):
684
                raise
685
        else:
686
            if self._is_dir(dstobj):
687
                add_prefix = '%s/%s' % (dst_path.strip('/'), add_prefix)
688
            elif src_N:
689
                raiseCLIError(
690
                    'Cannot merge multiple paths to path' % dst_path,
691
                    details=[
692
                        'Try to use / or a directory as destination',
693
                        'or create the destination dir (/file mkdir)',
694
                        'or use a single object as source'])
695

  
696
        if src_N:
697
            (method, kwargs) = src_iter
698
            for obj in method(**kwargs):
699
                name = obj['name']
700
                if name.endswith(self['suffix']):
701
                    yield (name, self._get_new_object(name, add_prefix))
702
        elif src_iter['name'].endswith(self['suffix']):
703
            name = src_iter['name']
704
            yield (name, self._get_new_object(dst_path or name, add_prefix))
705
        else:
706
            raiseCLIError('Source path %s conflicts with suffix %s' % (
707
                src_iter['name'], self['suffix']))
708

  
709
    def _get_new_object(self, obj, add_prefix):
710
        if self['prefix_replace'] and obj.startswith(self['prefix_replace']):
711
            obj = obj[len(self['prefix_replace']):]
712
        if self['suffix_replace'] and obj.endswith(self['suffix_replace']):
713
            obj = obj[:-len(self['suffix_replace'])]
714
        return add_prefix + obj + self['add_suffix']
715

  
716

  
717
@command(pithos_cmds)
718
class file_copy(_source_destination_command, _optional_output_cmd):
719
    """Copy objects from container to (another) container
720
    Semantics:
721
    copy cont:path dir
722
    .   transfer path as dir/path
723
    copy cont:path cont2:
724
    .   trasnfer all <obj> prefixed with path to container cont2
725
    copy cont:path [cont2:]path2
726
    .   transfer path to path2
727
    Use options:
728
    1. <container1>:<path1> [container2:]<path2> : if container2 is not given,
729
    destination is container1:path2
730
    2. <container>:<path1> <path2> : make a copy in the same container
731
    3. Can use --container= instead of <container1>
732
    """
733

  
734
    arguments = dict(
735
        destination_account=ValueArgument(
736
            'Account to copy to', ('-a', '--dst-account')),
737
        destination_container=ValueArgument(
738
            'use it if destination container name contains a : character',
739
            ('-D', '--dst-container')),
740
        public=ValueArgument('make object publicly accessible', '--public'),
741
        content_type=ValueArgument(
742
            'change object\'s content type', '--content-type'),
743
        recursive=FlagArgument(
744
            'copy directory and contents', ('-R', '--recursive')),
745
        prefix=FlagArgument(
746
            'Match objects prefixed with src path (feels like src_path*)',
747
            '--with-prefix',
748
            default=''),
749
        suffix=ValueArgument(
750
            'Suffix of source objects (feels like *suffix)', '--with-suffix',
751
            default=''),
752
        add_prefix=ValueArgument('Prefix targets', '--add-prefix', default=''),
753
        add_suffix=ValueArgument('Suffix targets', '--add-suffix', default=''),
754
        prefix_replace=ValueArgument(
755
            'Prefix of src to replace with dst path + add_prefix, if matched',
756
            '--prefix-to-replace',
757
            default=''),
758
        suffix_replace=ValueArgument(
759
            'Suffix of src to replace with add_suffix, if matched',
760
            '--suffix-to-replace',
761
            default=''),
762
        source_version=ValueArgument(
763
            'copy specific version', ('-S', '--source-version'))
764
    )
765

  
766
    @errors.generic.all
767
    @errors.pithos.connection
768
    @errors.pithos.container
769
    @errors.pithos.account
770
    def _run(self, dst_path):
771
        no_source_object = True
772
        src_account = self.client.account if (
773
            self['destination_account']) else None
774
        for src_obj, dst_obj in self.src_dst_pairs(
775
                dst_path, self['source_version']):
776
            no_source_object = False
777
            r = self.dst_client.copy_object(
778
                src_container=self.client.container,
779
                src_object=src_obj,
780
                dst_container=self.dst_client.container,
781
                dst_object=dst_obj,
782
                source_account=src_account,
783
                source_version=self['source_version'],
784
                public=self['public'],
785
                content_type=self['content_type'])
786
        if no_source_object:
787
            raiseCLIError('No object %s in container %s' % (
788
                self.path, self.container))
789
        self._optional_output(r)
790

  
791
    def main(
792
            self, source_container___path,
793
            destination_container___path=None):
794
        super(file_copy, self)._run(
795
            source_container___path, path_is_optional=False)
796
        (dst_cont, dst_path) = self._dest_container_path(
797
            destination_container___path)
798
        self.dst_client.container = dst_cont or self.container
799
        self._run(dst_path=dst_path or '')
800

  
801

  
802
@command(pithos_cmds)
803
class file_move(_source_destination_command, _optional_output_cmd):
804
    """Move/rename objects from container to (another) container
805
    Semantics:
806
    move cont:path dir
807
    .   rename path as dir/path
808
    move cont:path cont2:
809
    .   trasnfer all <obj> prefixed with path to container cont2
810
    move cont:path [cont2:]path2
811
    .   transfer path to path2
812
    Use options:
813
    1. <container1>:<path1> [container2:]<path2> : if container2 is not given,
814
    destination is container1:path2
815
    2. <container>:<path1> <path2> : move in the same container
816
    3. Can use --container= instead of <container1>
817
    """
818

  
819
    arguments = dict(
820
        destination_account=ValueArgument(
821
            'Account to move to', ('-a', '--dst-account')),
822
        destination_container=ValueArgument(
823
            'use it if destination container name contains a : character',
824
            ('-D', '--dst-container')),
825
        public=ValueArgument('make object publicly accessible', '--public'),
826
        content_type=ValueArgument(
827
            'change object\'s content type', '--content-type'),
828
        recursive=FlagArgument(
829
            'copy directory and contents', ('-R', '--recursive')),
830
        prefix=FlagArgument(
831
            'Match objects prefixed with src path (feels like src_path*)',
832
            '--with-prefix',
833
            default=''),
834
        suffix=ValueArgument(
835
            'Suffix of source objects (feels like *suffix)', '--with-suffix',
836
            default=''),
837
        add_prefix=ValueArgument('Prefix targets', '--add-prefix', default=''),
838
        add_suffix=ValueArgument('Suffix targets', '--add-suffix', default=''),
839
        prefix_replace=ValueArgument(
840
            'Prefix of src to replace with dst path + add_prefix, if matched',
841
            '--prefix-to-replace',
842
            default=''),
843
        suffix_replace=ValueArgument(
844
            'Suffix of src to replace with add_suffix, if matched',
845
            '--suffix-to-replace',
846
            default='')
847
    )
848

  
849
    @errors.generic.all
850
    @errors.pithos.connection
851
    @errors.pithos.container
852
    def _run(self, dst_path):
853
        no_source_object = True
854
        src_account = self.client.account if (
855
            self['destination_account']) else None
856
        for src_obj, dst_obj in self.src_dst_pairs(dst_path):
857
            no_source_object = False
858
            r = self.dst_client.move_object(
859
                src_container=self.container,
860
                src_object=src_obj,
861
                dst_container=self.dst_client.container,
862
                dst_object=dst_obj,
863
                source_account=src_account,
864
                public=self['public'],
865
                content_type=self['content_type'])
866
        if no_source_object:
867
            raiseCLIError('No object %s in container %s' % (
868
                self.path, self.container))
869
        self._optional_output(r)
870

  
871
    def main(
872
            self, source_container___path,
873
            destination_container___path=None):
874
        super(self.__class__, self)._run(
875
            source_container___path,
876
            path_is_optional=False)
877
        (dst_cont, dst_path) = self._dest_container_path(
878
            destination_container___path)
879
        (dst_cont, dst_path) = self._dest_container_path(
880
            destination_container___path)
881
        self.dst_client.container = dst_cont or self.container
882
        self._run(dst_path=dst_path or '')
883

  
884

  
885
@command(pithos_cmds)
886
class file_append(_file_container_command, _optional_output_cmd):
887
    """Append local file to (existing) remote object
888
    The remote object should exist.
889
    If the remote object is a directory, it is transformed into a file.
890
    In the later case, objects under the directory remain intact.
891
    """
892

  
893
    arguments = dict(
894
        progress_bar=ProgressBarArgument(
895
            'do not show progress bar', ('-N', '--no-progress-bar'),
896
            default=False)
897
    )
898

  
899
    @errors.generic.all
900
    @errors.pithos.connection
901
    @errors.pithos.container
902
    @errors.pithos.object_path
903
    def _run(self, local_path):
904
        (progress_bar, upload_cb) = self._safe_progress_bar('Appending')
905
        try:
906
            with open(local_path, 'rb') as f:
907
                self._optional_output(
908
                    self.client.append_object(self.path, f, upload_cb))
909
        finally:
910
            self._safe_progress_bar_finish(progress_bar)
911

  
912
    def main(self, local_path, container___path):
913
        super(self.__class__, self)._run(
914
            container___path, path_is_optional=False)
915
        self._run(local_path)
916

  
917

  
918
@command(pithos_cmds)
919
class file_truncate(_file_container_command, _optional_output_cmd):
920
    """Truncate remote file up to a size (default is 0)"""
921

  
922
    @errors.generic.all
923
    @errors.pithos.connection
924
    @errors.pithos.container
925
    @errors.pithos.object_path
926
    @errors.pithos.object_size
927
    def _run(self, size=0):
928
        self._optional_output(self.client.truncate_object(self.path, size))
929

  
930
    def main(self, container___path, size=0):
931
        super(self.__class__, self)._run(container___path)
932
        self._run(size=size)
933

  
934

  
935
@command(pithos_cmds)
936
class file_overwrite(_file_container_command, _optional_output_cmd):
937
    """Overwrite part (from start to end) of a remote file
938
    overwrite local-path container 10 20
939
    .   will overwrite bytes from 10 to 20 of a remote file with the same name
940
    .   as local-path basename
941
    overwrite local-path container:path 10 20
942
    .   will overwrite as above, but the remote file is named path
943
    """
944

  
945
    arguments = dict(
946
        progress_bar=ProgressBarArgument(
947
            'do not show progress bar', ('-N', '--no-progress-bar'),
948
            default=False)
949
    )
950

  
951
    @errors.generic.all
952
    @errors.pithos.connection
953
    @errors.pithos.container
954
    @errors.pithos.object_path
955
    @errors.pithos.object_size
956
    def _run(self, local_path, start, end):
957
        start, end = int(start), int(end)
958
        (progress_bar, upload_cb) = self._safe_progress_bar(
959
            'Overwrite %s bytes' % (end - start))
960
        try:
961
            with open(path.abspath(local_path), 'rb') as f:
962
                self._optional_output(self.client.overwrite_object(
963
                    obj=self.path,
964
                    start=start,
965
                    end=end,
966
                    source_file=f,
967
                    upload_cb=upload_cb))
968
        finally:
969
            self._safe_progress_bar_finish(progress_bar)
970

  
971
    def main(self, local_path, container___path, start, end):
972
        super(self.__class__, self)._run(
973
            container___path, path_is_optional=None)
974
        self.path = self.path or path.basename(local_path)
975
        self._run(local_path=local_path, start=start, end=end)
976

  
977

  
978
@command(pithos_cmds)
979
class file_manifest(_file_container_command, _optional_output_cmd):
980
    """Create a remote file of uploaded parts by manifestation
981
    Remains functional for compatibility with OOS Storage. Users are advised
982
    to use the upload command instead.
983
    Manifestation is a compliant process for uploading large files. The files
984
    have to be chunked in smalled files and uploaded as <prefix><increment>
985
    where increment is 1, 2, ...
986
    Finally, the manifest command glues partial files together in one file
987
    named <prefix>
988
    The upload command is faster, easier and more intuitive than manifest
989
    """
990

  
991
    arguments = dict(
992
        etag=ValueArgument('check written data', '--etag'),
993
        content_encoding=ValueArgument(
994
            'set MIME content type', '--content-encoding'),
995
        content_disposition=ValueArgument(
996
            'the presentation style of the object', '--content-disposition'),
997
        content_type=ValueArgument(
998
            'specify content type', '--content-type',
999
            default='application/octet-stream'),
1000
        sharing=SharingArgument(
1001
            '\n'.join([
1002
                'define object sharing policy',
1003
                '    ( "read=user1,grp1,user2,... write=user1,grp2,..." )']),
1004
            '--sharing'),
1005
        public=FlagArgument('make object publicly accessible', '--public')
1006
    )
1007

  
1008
    @errors.generic.all
1009
    @errors.pithos.connection
1010
    @errors.pithos.container
1011
    @errors.pithos.object_path
1012
    def _run(self):
1013
        ctype, cenc = guess_mime_type(self.path)
1014
        self._optional_output(self.client.create_object_by_manifestation(
1015
            self.path,
1016
            content_encoding=self['content_encoding'] or cenc,
1017
            content_disposition=self['content_disposition'],
1018
            content_type=self['content_type'] or ctype,
1019
            sharing=self['sharing'],
1020
            public=self['public']))
1021

  
1022
    def main(self, container___path):
1023
        super(self.__class__, self)._run(
1024
            container___path, path_is_optional=False)
1025
        self.run()
1026

  
1027

  
1028
@command(pithos_cmds)
1029
class file_upload(_file_container_command, _optional_output_cmd):
1030
    """Upload a file"""
1031

  
1032
    arguments = dict(
1033
        use_hashes=FlagArgument(
1034
            'provide hashmap file instead of data', '--use-hashes'),
1035
        etag=ValueArgument('check written data', '--etag'),
1036
        unchunked=FlagArgument('avoid chunked transfer mode', '--unchunked'),
1037
        content_encoding=ValueArgument(
1038
            'set MIME content type', '--content-encoding'),
1039
        content_disposition=ValueArgument(
1040
            'specify objects presentation style', '--content-disposition'),
1041
        content_type=ValueArgument('specify content type', '--content-type'),
1042
        sharing=SharingArgument(
1043
            help='\n'.join([
1044
                'define sharing object policy',
1045
                '( "read=user1,grp1,user2,... write=user1,grp2,... )']),
1046
            parsed_name='--sharing'),
1047
        public=FlagArgument('make object publicly accessible', '--public'),
1048
        poolsize=IntArgument('set pool size', '--with-pool-size'),
1049
        progress_bar=ProgressBarArgument(
1050
            'do not show progress bar',
1051
            ('-N', '--no-progress-bar'),
1052
            default=False),
1053
        overwrite=FlagArgument('Force (over)write', ('-f', '--force')),
1054
        recursive=FlagArgument(
1055
            'Recursively upload directory *contents* + subdirectories',
1056
            ('-R', '--recursive'))
1057
    )
1058

  
1059
    def _check_container_limit(self, path):
1060
        cl_dict = self.client.get_container_limit()
1061
        container_limit = int(cl_dict['x-container-policy-quota'])
1062
        r = self.client.container_get()
1063
        used_bytes = sum(int(o['bytes']) for o in r.json)
1064
        path_size = get_path_size(path)
1065
        if container_limit and path_size > (container_limit - used_bytes):
1066
            raiseCLIError(
1067
                'Container(%s) (limit(%s) - used(%s)) < size(%s) of %s' % (
1068
                    self.client.container,
1069
                    format_size(container_limit),
1070
                    format_size(used_bytes),
1071
                    format_size(path_size),
1072
                    path),
1073
                importance=1, details=[
1074
                    'Check accound limit: /file quota',
1075
                    'Check container limit:',
1076
                    '\t/file containerlimit get %s' % self.client.container,
1077
                    'Increase container limit:',
1078
                    '\t/file containerlimit set <new limit> %s' % (
1079
                        self.client.container)])
1080

  
1081
    def _path_pairs(self, local_path, remote_path):
1082
        """Get pairs of local and remote paths"""
1083
        lpath = path.abspath(local_path)
1084
        short_path = lpath.split(path.sep)[-1]
1085
        rpath = remote_path or short_path
1086
        if path.isdir(lpath):
1087
            if not self['recursive']:
1088
                raiseCLIError('%s is a directory' % lpath, details=[
1089
                    'Use -R to upload directory contents'])
1090
            robj = self.client.container_get(path=rpath)
1091
            if robj.json and not self['overwrite']:
1092
                raiseCLIError(
1093
                    'Objects prefixed with %s already exist' % rpath,
1094
                    importance=1,
1095
                    details=['Existing objects:'] + ['\t%s:\t%s' % (
1096
                        o['content_type'][12:],
1097
                        o['name']) for o in robj.json] + [
1098
                        'Use -f to add, overwrite or resume'])
1099
            if not self['overwrite']:
1100
                try:
1101
                    topobj = self.client.get_object_info(rpath)
1102
                    if not self._is_dir(topobj):
1103
                        raiseCLIError(
1104
                            'Object %s exists but it is not a dir' % rpath,
1105
                            importance=1, details=['Use -f to overwrite'])
1106
                except ClientError as ce:
1107
                    if ce.status not in (404, ):
1108
                        raise
1109
            self._check_container_limit(lpath)
1110
            prev = ''
1111
            for top, subdirs, files in walk(lpath):
1112
                if top != prev:
1113
                    prev = top
1114
                    try:
1115
                        rel_path = rpath + top.split(lpath)[1]
1116
                    except IndexError:
1117
                        rel_path = rpath
1118
                    self.error('mkdir %s:%s' % (
1119
                        self.client.container, rel_path))
1120
                    self.client.create_directory(rel_path)
1121
                for f in files:
1122
                    fpath = path.join(top, f)
1123
                    if path.isfile(fpath):
1124
                        rel_path = rel_path.replace(path.sep, '/')
1125
                        pathfix = f.replace(path.sep, '/')
1126
                        yield open(fpath, 'rb'), '%s/%s' % (rel_path, pathfix)
1127
                    else:
1128
                        self.error('%s is not a regular file' % fpath)
1129
        else:
1130
            if not path.isfile(lpath):
1131
                raiseCLIError(('%s is not a regular file' % lpath) if (
1132
                    path.exists(lpath)) else '%s does not exist' % lpath)
1133
            try:
1134
                robj = self.client.get_object_info(rpath)
1135
                if remote_path and self._is_dir(robj):
1136
                    rpath += '/%s' % (short_path.replace(path.sep, '/'))
1137
                    self.client.get_object_info(rpath)
1138
                if not self['overwrite']:
1139
                    raiseCLIError(
1140
                        'Object %s already exists' % rpath,
1141
                        importance=1,
1142
                        details=['use -f to overwrite or resume'])
1143
            except ClientError as ce:
1144
                if ce.status not in (404, ):
1145
                    raise
1146
            self._check_container_limit(lpath)
1147
            yield open(lpath, 'rb'), rpath
1148

  
1149
    @errors.generic.all
1150
    @errors.pithos.connection
1151
    @errors.pithos.container
1152
    @errors.pithos.object_path
1153
    @errors.pithos.local_path
1154
    def _run(self, local_path, remote_path):
1155
        if self['poolsize'] > 0:
1156
            self.client.MAX_THREADS = int(self['poolsize'])
1157
        params = dict(
1158
            content_encoding=self['content_encoding'],
1159
            content_type=self['content_type'],
1160
            content_disposition=self['content_disposition'],
1161
            sharing=self['sharing'],
1162
            public=self['public'])
1163
        uploaded = []
1164
        container_info_cache = dict()
1165
        for f, rpath in self._path_pairs(local_path, remote_path):
1166
            self.error('%s --> %s:%s' % (f.name, self.client.container, rpath))
1167
            if not (self['content_type'] and self['content_encoding']):
1168
                ctype, cenc = guess_mime_type(f.name)
1169
                params['content_type'] = self['content_type'] or ctype
1170
                params['content_encoding'] = self['content_encoding'] or cenc
1171
            if self['unchunked']:
1172
                r = self.client.upload_object_unchunked(
1173
                    rpath, f,
1174
                    etag=self['etag'], withHashFile=self['use_hashes'],
1175
                    **params)
1176
                if self['with_output'] or self['json_output']:
1177
                    r['name'] = '%s: %s' % (self.client.container, rpath)
1178
                    uploaded.append(r)
1179
            else:
1180
                try:
1181
                    (progress_bar, upload_cb) = self._safe_progress_bar(
1182
                        'Uploading %s' % f.name.split(path.sep)[-1])
1183
                    if progress_bar:
1184
                        hash_bar = progress_bar.clone()
1185
                        hash_cb = hash_bar.get_generator(
1186
                            'Calculating block hashes')
1187
                    else:
1188
                        hash_cb = None
1189
                    r = self.client.upload_object(
1190
                        rpath, f,
1191
                        hash_cb=hash_cb,
1192
                        upload_cb=upload_cb,
1193
                        container_info_cache=container_info_cache,
1194
                        **params)
1195
                    if self['with_output'] or self['json_output']:
1196
                        r['name'] = '%s: %s' % (self.client.container, rpath)
1197
                        uploaded.append(r)
1198
                except Exception:
1199
                    self._safe_progress_bar_finish(progress_bar)
1200
                    raise
1201
                finally:
1202
                    self._safe_progress_bar_finish(progress_bar)
1203
        self._optional_output(uploaded)
1204
        self.error('Upload completed')
1205

  
1206
    def main(self, local_path, container____path__=None):
1207
        super(self.__class__, self)._run(container____path__)
1208
        remote_path = self.path or path.basename(path.abspath(local_path))
1209
        self._run(local_path=local_path, remote_path=remote_path)
1210

  
1211

  
1212
@command(pithos_cmds)
1213
class file_cat(_file_container_command):
1214
    """Print remote file contents to console"""
1215

  
1216
    arguments = dict(
1217
        range=RangeArgument('show range of data', '--range'),
1218
        if_match=ValueArgument('show output if ETags match', '--if-match'),
1219
        if_none_match=ValueArgument(
1220
            'show output if ETags match', '--if-none-match'),
1221
        if_modified_since=DateArgument(
1222
            'show output modified since then', '--if-modified-since'),
1223
        if_unmodified_since=DateArgument(
1224
            'show output unmodified since then', '--if-unmodified-since'),
1225
        object_version=ValueArgument(
1226
            'get the specific version', ('-O', '--object-version'))
1227
    )
1228

  
1229
    @errors.generic.all
1230
    @errors.pithos.connection
1231
    @errors.pithos.container
1232
    @errors.pithos.object_path
1233
    def _run(self):
1234
        self.client.download_object(
1235
            self.path, self._out,
1236
            range_str=self['range'],
1237
            version=self['object_version'],
1238
            if_match=self['if_match'],
1239
            if_none_match=self['if_none_match'],
1240
            if_modified_since=self['if_modified_since'],
1241
            if_unmodified_since=self['if_unmodified_since'])
1242

  
1243
    def main(self, container___path):
1244
        super(self.__class__, self)._run(
1245
            container___path, path_is_optional=False)
1246
        self._run()
1247

  
1248

  
1249
@command(pithos_cmds)
1250
class file_download(_file_container_command):
1251
    """Download remote object as local file
1252
    If local destination is a directory:
1253
    *   download <container>:<path> <local dir> -R
1254
    will download all files on <container> prefixed as <path>,
1255
    to <local dir>/<full path> (or <local dir>\<full path> in windows)
1256
    *   download <container>:<path> <local dir>
1257
    will download only one file<path>
1258
    ATTENTION: to download cont:dir1/dir2/file there must exist objects
1259
    cont:dir1 and cont:dir1/dir2 of type application/directory
1260
    To create directory objects, use /file mkdir
1261
    """
1262

  
1263
    arguments = dict(
1264
        resume=FlagArgument('Resume instead of overwrite', ('-r', '--resume')),
1265
        range=RangeArgument('show range of data', '--range'),
1266
        if_match=ValueArgument('show output if ETags match', '--if-match'),
1267
        if_none_match=ValueArgument(
1268
            'show output if ETags match', '--if-none-match'),
1269
        if_modified_since=DateArgument(
1270
            'show output modified since then', '--if-modified-since'),
1271
        if_unmodified_since=DateArgument(
1272
            'show output unmodified since then', '--if-unmodified-since'),
1273
        object_version=ValueArgument(
1274
            'get the specific version', ('-O', '--object-version')),
1275
        poolsize=IntArgument('set pool size', '--with-pool-size'),
1276
        progress_bar=ProgressBarArgument(
1277
            'do not show progress bar', ('-N', '--no-progress-bar'),
1278
            default=False),
1279
        recursive=FlagArgument(
1280
            'Download a remote path and its contents', ('-R', '--recursive'))
1281
    )
1282

  
1283
    def _outputs(self, local_path):
1284
        """:returns: (local_file, remote_path)"""
1285
        remotes = []
1286
        if self['recursive']:
1287
            r = self.client.container_get(
1288
                prefix=self.path or '/',
1289
                if_modified_since=self['if_modified_since'],
1290
                if_unmodified_since=self['if_unmodified_since'])
1291
            dirlist = dict()
1292
            for remote in r.json:
1293
                rname = remote['name'].strip('/')
1294
                tmppath = ''
1295
                for newdir in rname.strip('/').split('/')[:-1]:
1296
                    tmppath = '/'.join([tmppath, newdir])
1297
                    dirlist.update({tmppath.strip('/'): True})
1298
                remotes.append((rname, file_download._is_dir(remote)))
1299
            dir_remotes = [r[0] for r in remotes if r[1]]
1300
            if not set(dirlist).issubset(dir_remotes):
1301
                badguys = [bg.strip('/') for bg in set(
1302
                    dirlist).difference(dir_remotes)]
1303
                raiseCLIError(
1304
                    'Some remote paths contain non existing directories',
1305
                    details=['Missing remote directories:'] + badguys)
1306
        elif self.path:
1307
            r = self.client.get_object_info(
1308
                self.path,
1309
                version=self['object_version'])
1310
            if file_download._is_dir(r):
1311
                raiseCLIError(
1312
                    'Illegal download: Remote object %s is a directory' % (
1313
                        self.path),
1314
                    details=['To download a directory, try --recursive or -R'])
1315
            if '/' in self.path.strip('/') and not local_path:
1316
                raiseCLIError(
1317
                    'Illegal download: remote object %s contains "/"' % (
1318
                        self.path),
1319
                    details=[
1320
                        'To download an object containing "/" characters',
1321
                        'either create the remote directories or',
1322
                        'specify a non-directory local path for this object'])
1323
            remotes = [(self.path, False)]
1324
        if not remotes:
1325
            if self.path:
1326
                raiseCLIError(
1327
                    'No matching path %s on container %s' % (
1328
                        self.path, self.container),
1329
                    details=[
1330
                        'To list the contents of %s, try:' % self.container,
1331
                        '   /file list %s' % self.container])
1332
            raiseCLIError(
1333
                'Illegal download of container %s' % self.container,
1334
                details=[
1335
                    'To download a whole container, try:',
1336
                    '   /file download --recursive <container>'])
1337

  
1338
        lprefix = path.abspath(local_path or path.curdir)
1339
        if path.isdir(lprefix):
1340
            for rpath, remote_is_dir in remotes:
1341
                lpath = path.sep.join([
1342
                    lprefix[:-1] if lprefix.endswith(path.sep) else lprefix,
1343
                    rpath.strip('/').replace('/', path.sep)])
1344
                if remote_is_dir:
1345
                    if path.exists(lpath) and path.isdir(lpath):
1346
                        continue
1347
                    makedirs(lpath)
1348
                elif path.exists(lpath):
1349
                    if not self['resume']:
1350
                        self.error('File %s exists, aborting...' % lpath)
1351
                        continue
1352
                    with open(lpath, 'rwb+') as f:
1353
                        yield (f, rpath)
1354
                else:
1355
                    with open(lpath, 'wb+') as f:
1356
                        yield (f, rpath)
1357
        elif path.exists(lprefix):
1358
            if len(remotes) > 1:
1359
                raiseCLIError(
1360
                    '%s remote objects cannot be merged in local file %s' % (
1361
                        len(remotes),
1362
                        local_path),
1363
                    details=[
1364
                        'To download multiple objects, local path should be',
1365
                        'a directory, or use download without a local path'])
1366
            (rpath, remote_is_dir) = remotes[0]
1367
            if remote_is_dir:
1368
                raiseCLIError(
1369
                    'Remote directory %s should not replace local file %s' % (
1370
                        rpath,
1371
                        local_path))
1372
            if self['resume']:
1373
                with open(lprefix, 'rwb+') as f:
1374
                    yield (f, rpath)
1375
            else:
1376
                raiseCLIError(
1377
                    'Local file %s already exist' % local_path,
1378
                    details=['Try --resume to overwrite it'])
1379
        else:
1380
            if len(remotes) > 1 or remotes[0][1]:
1381
                raiseCLIError(
1382
                    'Local directory %s does not exist' % local_path)
1383
            with open(lprefix, 'wb+') as f:
1384
                yield (f, remotes[0][0])
1385

  
1386
    @errors.generic.all
1387
    @errors.pithos.connection
1388
    @errors.pithos.container
1389
    @errors.pithos.object_path
1390
    @errors.pithos.local_path
1391
    def _run(self, local_path):
1392
        poolsize = self['poolsize']
1393
        if poolsize:
1394
            self.client.MAX_THREADS = int(poolsize)
1395
        progress_bar = None
1396
        try:
1397
            for f, rpath in self._outputs(local_path):
1398
                (
1399
                    progress_bar,
1400
                    download_cb) = self._safe_progress_bar(
1401
                        'Download %s' % rpath)
1402
                self.client.download_object(
1403
                    rpath, f,
1404
                    download_cb=download_cb,
1405
                    range_str=self['range'],
1406
                    version=self['object_version'],
1407
                    if_match=self['if_match'],
1408
                    resume=self['resume'],
1409
                    if_none_match=self['if_none_match'],
1410
                    if_modified_since=self['if_modified_since'],
1411
                    if_unmodified_since=self['if_unmodified_since'])
1412
        except KeyboardInterrupt:
1413
            from threading import activeCount, enumerate as activethreads
1414
            timeout = 0.5
1415
            while activeCount() > 1:
1416
                self._out.write('\nCancel %s threads: ' % (activeCount() - 1))
1417
                self._out.flush()
1418
                for thread in activethreads():
1419
                    try:
1420
                        thread.join(timeout)
1421
                        self._out.write('.' if thread.isAlive() else '*')
1422
                    except RuntimeError:
1423
                        continue
1424
                    finally:
1425
                        self._out.flush()
1426
                        timeout += 0.1
1427
            self.error('\nDownload canceled by user')
1428
            if local_path is not None:
1429
                self.error('to resume, re-run with --resume')
1430
        except Exception:
1431
            self._safe_progress_bar_finish(progress_bar)
1432
            raise
1433
        finally:
1434
            self._safe_progress_bar_finish(progress_bar)
1435

  
1436
    def main(self, container___path, local_path=None):
1437
        super(self.__class__, self)._run(container___path)
1438
        self._run(local_path=local_path)
1439

  
1440

  
1441
@command(pithos_cmds)
1442
class file_hashmap(_file_container_command, _optional_json):
1443
    """Get the hash-map of an object"""
1444

  
... This diff was truncated because it exceeds the maximum size that can be displayed.

Also available in: Unified diff