Statistics
| Branch: | Tag: | Revision:

root / tools / store @ a4c10cbc

History | View | Annotate | Download (23.1 kB)

1
#!/usr/bin/env python
2

    
3
# Copyright 2011 GRNET S.A. All rights reserved.
4
# 
5
# Redistribution and use in source and binary forms, with or
6
# without modification, are permitted provided that the following
7
# conditions are met:
8
# 
9
#   1. Redistributions of source code must retain the above
10
#      copyright notice, this list of conditions and the following
11
#      disclaimer.
12
# 
13
#   2. Redistributions in binary form must reproduce the above
14
#      copyright notice, this list of conditions and the following
15
#      disclaimer in the documentation and/or other materials
16
#      provided with the distribution.
17
# 
18
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29
# POSSIBILITY OF SUCH DAMAGE.
30
# 
31
# The views and conclusions contained in the software and
32
# documentation are those of the authors and should not be
33
# interpreted as representing official policies, either expressed
34
# or implied, of GRNET S.A.
35

    
36
from getpass import getuser
37
from optparse import OptionParser
38
from os.path import basename
39
from sys import argv, exit, stdin, stdout
40
from pithos.lib.client import Client, Fault
41
from datetime import datetime
42

    
43
import json
44
import logging
45
import types
46
import re
47
import time as _time
48

    
49
DEFAULT_HOST = 'pithos.dev.grnet.gr'
50
DEFAULT_API = 'v1'
51

    
52
_cli_commands = {}
53

    
54
def cli_command(*args):
55
    def decorator(cls):
56
        cls.commands = args
57
        for name in args:
58
            _cli_commands[name] = cls
59
        return cls
60
    return decorator
61

    
62
def class_for_cli_command(name):
63
    return _cli_commands[name]
64

    
65
class Command(object):
66
    def __init__(self, argv):
67
        parser = OptionParser()
68
        parser.add_option('--host', dest='host', metavar='HOST',
69
                          default=DEFAULT_HOST, help='use server HOST')
70
        parser.add_option('--user', dest='user', metavar='USERNAME',
71
                          default=getuser(), help='use account USERNAME')
72
        parser.add_option('--api', dest='api', metavar='API',
73
                          default=DEFAULT_API, help='use api API')
74
        parser.add_option('-v', action='store_true', dest='verbose',
75
                          default=False, help='use verbose output')
76
        parser.add_option('-d', action='store_true', dest='debug',
77
                          default=False, help='use debug output')
78
        self.add_options(parser)
79
        options, args = parser.parse_args(argv)
80
        
81
        # Add options to self
82
        for opt in parser.option_list:
83
            key = opt.dest
84
            if key:
85
                val = getattr(options, key)
86
                setattr(self, key, val)
87
        
88
        self.client = Client(self.host, self.user, self.api, self.verbose,
89
                             self.debug)
90
        
91
        self.parser = parser
92
        self.args = args
93
        
94
    def add_options(self, parser):
95
        pass
96
    
97
    def execute(self, *args):
98
        pass
99

    
100
@cli_command('list', 'ls')
101
class List(Command):
102
    syntax = '[<container>[/<object>]]'
103
    description = 'list containers or objects'
104
    
105
    def add_options(self, parser):
106
        parser.add_option('-l', action='store_true', dest='detail',
107
                          default=False, help='show detailed output')
108
        parser.add_option('-n', action='store', type='int', dest='limit',
109
                          default=1000, help='show limited output')
110
        parser.add_option('--marker', action='store', type='str',
111
                          dest='marker', default=None,
112
                          help='show output greater then marker')
113
        parser.add_option('--prefix', action='store', type='str',
114
                          dest='prefix', default=None,
115
                          help='show output starting with prefix')
116
        parser.add_option('--delimiter', action='store', type='str',
117
                          dest='delimiter', default=None,
118
                          help='show output up to the delimiter')
119
        parser.add_option('--path', action='store', type='str',
120
                          dest='path', default=None,
121
                          help='show output starting with prefix up to /')
122
        parser.add_option('--meta', action='store', type='str',
123
                          dest='meta', default=None,
124
                          help='show output having the specified meta keys')
125
        parser.add_option('--if-modified-since', action='store', type='str',
126
                          dest='if_modified_since', default=None,
127
                          help='show output if modified since then')
128
        parser.add_option('--if-unmodified-since', action='store', type='str',
129
                          dest='if_unmodified_since', default=None,
130
                          help='show output if not modified since then')
131
        parser.add_option('--until', action='store', dest='until',
132
                          default=False, help='show metadata until that date')
133
        parser.add_option('--format', action='store', dest='format',
134
                          default='%d/%m/%Y', help='format to parse until date')
135
    
136
    def execute(self, container=None):
137
        if container:
138
            self.list_objects(container)
139
        else:
140
            self.list_containers()
141
    
142
    def list_containers(self):
143
        params = {'limit':self.limit, 'marker':self.marker}
144
        headers = {'IF_MODIFIED_SINCE':self.if_modified_since,
145
                   'IF_UNMODIFIED_SINCE':self.if_unmodified_since}
146
        
147
        if self.until:
148
            t = _time.strptime(self.until, self.format)
149
            params['until'] = int(_time.mktime(t))
150
        
151
        l = self.client.list_containers(self.detail, params, headers)
152
        print_list(l)
153
    
154
    def list_objects(self, container):
155
        params = {'limit':self.limit, 'marker':self.marker,
156
                  'prefix':self.prefix, 'delimiter':self.delimiter,
157
                  'path':self.path, 'meta':self.meta}
158
        headers = {'IF_MODIFIED_SINCE':self.if_modified_since,
159
                   'IF_UNMODIFIED_SINCE':self.if_unmodified_since}
160
        container, sep, object = container.partition('/')
161
        if object:
162
            return
163
        
164
        if self.until:
165
            t = _time.strptime(self.until, self.format)
166
            params['until'] = int(_time.mktime(t))
167
        
168
        detail = 'json'
169
        l = self.client.list_objects(container, detail, params, headers)
170
        print_list(l, detail=self.detail)
171

    
172
@cli_command('meta')
173
class Meta(Command):
174
    syntax = '[<container>[/<object>]]'
175
    description = 'get the metadata of an account, a container or an object'
176
    
177
    def add_options(self, parser):
178
        parser.add_option('-r', action='store_true', dest='restricted',
179
                          default=False, help='show only user defined metadata')
180
        parser.add_option('--until', action='store', dest='until',
181
                          default=False, help='show metadata until that date')
182
        parser.add_option('--format', action='store', dest='format',
183
                          default='%d/%m/%Y', help='format to parse until date')
184
        parser.add_option('--version', action='store', dest='version',
185
                          default=None, help='show specific version \
186
                                  (applies only for objects)')
187
    
188
    def execute(self, path=''):
189
        container, sep, object = path.partition('/')
190
        if self.until:
191
            t = _time.strptime(self.until, self.format)
192
            self.until = int(_time.mktime(t))
193
        if object:
194
            meta = self.client.retrieve_object_metadata(container, object,
195
                                                        self.restricted,
196
                                                        self.version)
197
        elif container:
198
            meta = self.client.retrieve_container_metadata(container,
199
                                                           self.restricted,
200
                                                           self.until)
201
        else:
202
            meta = self.client.account_metadata(self.restricted, self.until)
203
        if meta == None:
204
            print 'Entity does not exist'
205
        else:
206
            print_dict(meta, header=None)
207

    
208
@cli_command('create')
209
class CreateContainer(Command):
210
    syntax = '<container> [key=val] [...]'
211
    description = 'create a container'
212
    
213
    def execute(self, container, *args):
214
        headers = {}
215
        for arg in args:
216
            key, sep, val = arg.partition('=')
217
            headers['X_CONTAINER_META_%s' %key.strip().upper()] = val.strip()
218
        ret = self.client.create_container(container, headers)
219
        if not ret:
220
            print 'Container already exists'
221

    
222
@cli_command('delete', 'rm')
223
class Delete(Command):
224
    syntax = '<container>[/<object>]'
225
    description = 'delete a container or an object'
226
    
227
    def execute(self, path):
228
        container, sep, object = path.partition('/')
229
        if object:
230
            self.client.delete_object(container, object)
231
        else:
232
            self.client.delete_container(container)
233

    
234
@cli_command('get')
235
class GetObject(Command):
236
    syntax = '<container>/<object>'
237
    description = 'get the data of an object'
238
    
239
    def add_options(self, parser):
240
        parser.add_option('-l', action='store_true', dest='detail',
241
                          default=False, help='show detailed output')
242
        parser.add_option('--range', action='store', dest='range',
243
                          default=None, help='show range of data')
244
        parser.add_option('--if-match', action='store', dest='if-match',
245
                          default=None, help='show output if ETags match')
246
        parser.add_option('--if-none-match', action='store',
247
                          dest='if-none-match', default=None,
248
                          help='show output if ETags don\'t match')
249
        parser.add_option('--if-modified-since', action='store', type='str',
250
                          dest='if-modified-since', default=None,
251
                          help='show output if modified since then')
252
        parser.add_option('--if-unmodified-since', action='store', type='str',
253
                          dest='if-unmodified-since', default=None,
254
                          help='show output if not modified since then')
255
        parser.add_option('-f', action='store', type='str',
256
                          dest='file', default=None,
257
                          help='save output in file')
258
        parser.add_option('--version', action='store', type='str',
259
                          dest='version', default=None,
260
                          help='get the specific \
261
                               version')
262
        parser.add_option('--versionlist', action='store_true',
263
                          dest='versionlist', default=False,
264
                          help='get the full object version list')
265
    
266
    def execute(self, path):
267
        headers = {}
268
        if self.range:
269
            headers['RANGE'] = 'bytes=%s' %self.range
270
        attrs = ['if-match', 'if-none-match', 'if-modified-since',
271
                 'if-unmodified-since']
272
        attrs = [a for a in attrs if getattr(self, a)]
273
        for a in attrs:
274
            headers[a.replace('-', '_').upper()] = getattr(self, a)
275
        container, sep, object = path.partition('/')
276
        if self.versionlist:
277
            self.version = 'list'
278
            self.detail = True
279
        data = self.client.retrieve_object(container, object, self.detail,
280
                                          headers, self.version)
281
        f = self.file and open(self.file, 'w') or stdout
282
        if self.detail:
283
            data = json.loads(data)
284
            if self.versionlist:
285
                print_versions(data, f=f)
286
            else:
287
                print_dict(data, f=f)
288
        else:
289
            f.write(data)
290
        f.close()
291

    
292
@cli_command('mkdir')
293
class PutMarker(Command):
294
    syntax = '<container>/<directory marker>'
295
    description = 'create a directory marker'
296
    
297
    def execute(self, path):
298
        container, sep, object = path.partition('/')
299
        self.client.create_directory_marker(container, object)
300

    
301
@cli_command('put')
302
class PutObject(Command):
303
    syntax = '<container>/<object> <path> [key=val] [...]'
304
    description = 'create/override object with path contents or standard input'
305
    
306
    def add_options(self, parser):
307
        parser.add_option('--chunked', action='store_true', dest='chunked',
308
                          default=False, help='set chunked transfer mode')
309
        parser.add_option('--etag', action='store', dest='etag',
310
                          default=None, help='check written data')
311
        parser.add_option('--content-encoding', action='store',
312
                          dest='content-encoding', default=None,
313
                          help='provide the object MIME content type')
314
        parser.add_option('--content-disposition', action='store', type='str',
315
                          dest='content-disposition', default=None,
316
                          help='provide the presentation style of the object')
317
        parser.add_option('--manifest', action='store', type='str',
318
                          dest='manifest', default=None,
319
                          help='use for large file support')
320
        parser.add_option('--type', action='store',
321
                          dest='content-type', default=False,
322
                          help='create object with specific content type')
323
        parser.add_option('--touch', action='store_true',
324
                          dest='touch', default=False,
325
                          help='create object with zero data')
326
    
327
    def execute(self, path, srcpath='-', *args):
328
        headers = {}
329
        if self.manifest:
330
            headers['X_OBJECT_MANIFEST'] = self.manifest
331
        
332
        attrs = ['etag', 'content-encoding', 'content-disposition',
333
                 'content-type']
334
        attrs = [a for a in attrs if getattr(self, a)]
335
        for a in attrs:
336
            headers[a.replace('-', '_').upper()] = getattr(self, a)
337
        
338
        #prepare user defined meta
339
        for arg in args:
340
            key, sep, val = arg.partition('=')
341
            headers['X_OBJECT_META_%s' %key.strip().upper()] = val.strip()
342
        
343
        container, sep, object = path.partition('/')
344
        
345
        f = None
346
        chunked = False
347
        if not self.touch:
348
            f = srcpath != '-' and open(srcpath) or stdin
349
            chunked = (self.chunked or f == stdin) and True or False
350
        self.client.create_object(container, object, f, chunked=chunked,
351
                                  headers=headers)
352
        if f:
353
            f.close()
354

    
355
@cli_command('copy', 'cp')
356
class CopyObject(Command):
357
    syntax = '<src container>/<src object> [<dst container>/]<dst object>'
358
    description = 'copies an object to a different location'
359
    
360
    def add_options(self, parser):
361
        parser.add_option('--version', action='store',
362
                          dest='version', default=False,
363
                          help='copy specific version')
364
    
365
    def execute(self, src, dst):
366
        src_container, sep, src_object = src.partition('/')
367
        dst_container, sep, dst_object = dst.partition('/')
368
        if not sep:
369
            dst_container = src_container
370
            dst_object = dst
371
        version = getattr(self, 'version')
372
        if version:
373
            headers = {}
374
            headers['X_SOURCE_VERSION'] = version
375
        self.client.copy_object(src_container, src_object, dst_container,
376
                                dst_object, headers)
377

    
378
@cli_command('set')
379
class SetMeta(Command):
380
    syntax = '[<container>[/<object>]] key=val [key=val] [...]'
381
    description = 'set metadata'
382
    
383
    def execute(self, path, *args):
384
        #in case of account fix the args
385
        if path.find('=') != -1:
386
            args = list(args)
387
            args.append(path)
388
            args = tuple(args)
389
            path = ''
390
        meta = {}
391
        for arg in args:
392
            key, sep, val = arg.partition('=')
393
            meta[key.strip()] = val.strip()
394
        container, sep, object = path.partition('/')
395
        if object:
396
            self.client.update_object_metadata(container, object, **meta)
397
        elif container:
398
            self.client.update_container_metadata(container, **meta)
399
        else:
400
            self.client.update_account_metadata(**meta)
401

    
402
@cli_command('update')
403
class UpdateObject(Command):
404
    syntax = '<container>/<object> path [key=val] [...]'
405
    description = 'update object metadata/data (default mode: append)'
406
    
407
    def add_options(self, parser):
408
        parser.add_option('-a', action='store_true', dest='append',
409
                          default=True, help='append data')
410
        parser.add_option('--start', action='store',
411
                          dest='start',
412
                          default=None, help='range of data to be updated')
413
        parser.add_option('--range', action='store', dest='content-range',
414
                          default=None, help='range of data to be updated')
415
        parser.add_option('--chunked', action='store_true', dest='chunked',
416
                          default=False, help='set chunked transfer mode')
417
        parser.add_option('--content-encoding', action='store',
418
                          dest='content-encoding', default=None,
419
                          help='provide the object MIME content type')
420
        parser.add_option('--content-disposition', action='store', type='str',
421
                          dest='content-disposition', default=None,
422
                          help='provide the presentation style of the object')
423
        parser.add_option('--manifest', action='store', type='str',
424
                          dest='manifest', default=None,
425
                          help='use for large file support')
426
    
427
    def execute(self, path, srcpath='-', *args):
428
        headers = {}
429
        if self.manifest:
430
            headers['X_OBJECT_MANIFEST'] = self.manifest
431
        
432
        if getattr(self, 'start'):
433
            headers['CONTENT_RANGE'] = 'bytes %s-/*' % getattr(self, 'start')
434
        elif self.append:
435
            headers['CONTENT_RANGE'] = 'bytes */*'
436
        
437
        attrs = ['content-encoding', 'content-disposition']
438
        attrs = [a for a in attrs if getattr(self, a)]
439
        for a in attrs:
440
            headers[a.replace('-', '_').upper()] = getattr(self, a)
441
        
442
        #prepare user defined meta
443
        for arg in args:
444
            key, sep, val = arg.partition('=')
445
            headers['X_OBJECT_META_%s' %key.strip().upper()] = val.strip()
446
        
447
        container, sep, object = path.partition('/')
448
        
449
        f = srcpath != '-' and open(srcpath) or stdin
450
        chunked = (self.chunked or f == stdin) and True or False
451
        self.client.update_object(container, object, f, chunked=chunked,
452
                                  headers=headers)
453
        f.close()
454

    
455
@cli_command('move', 'mv')
456
class MoveObject(Command):
457
    syntax = '<src container>/<src object> [<dst container>/]<dst object>'
458
    description = 'moves an object to a different location'
459
    
460
    def execute(self, src, dst):
461
        src_container, sep, src_object = src.partition('/')
462
        dst_container, sep, dst_object = dst.partition('/')
463
        if not sep:
464
            dst_container = src_container
465
            dst_object = dst
466
        
467
        self.client.move_object(src_container, src_object, dst_container,
468
                                dst_object, headers)
469

    
470
@cli_command('remove', 'rm')
471
class TrashObject(Command):
472
    syntax = '<container>/<object>'
473
    description = 'trashes an object'
474
    
475
    def execute(self, src):
476
        src_container, sep, src_object = src.partition('/')
477
        
478
        self.client.trash_object(src_container, src_object)
479

    
480
@cli_command('restore')
481
class TrashObject(Command):
482
    syntax = '<container>/<object>'
483
    description = 'trashes an object'
484
    
485
    def execute(self, src):
486
        src_container, sep, src_object = src.partition('/')
487
        
488
        self.client.restore_object(src_container, src_object)
489

    
490
@cli_command('unset')
491
class TrashObject(Command):
492
    syntax = '<container>/[<object>] key [key] [...]'
493
    description = 'deletes metadata info'
494
    
495
    def execute(self, path, *args):
496
        #in case of account fix the args
497
        if path.find('=') != -1:
498
            args = list(args)
499
            args.append(path)
500
            args = tuple(args)
501
            path = ''
502
        meta = []
503
        for key in args:
504
            meta.append(key)
505
        container, sep, object = path.partition('/')
506
        if object:
507
            self.client.delete_object_metadata(container, object, meta)
508
        elif container:
509
            self.client.delete_container_metadata(container, meta)
510
        else:
511
            self.client.delete_account_metadata(meta)
512

    
513
def print_usage():
514
    cmd = Command([])
515
    parser = cmd.parser
516
    parser.usage = '%prog <command> [options]'
517
    parser.print_help()
518
    
519
    commands = []
520
    for cls in set(_cli_commands.values()):
521
        name = ', '.join(cls.commands)
522
        description = getattr(cls, 'description', '')
523
        commands.append('  %s %s' % (name.ljust(12), description))
524
    print '\nCommands:\n' + '\n'.join(sorted(commands))
525

    
526
def print_dict(d, header='name', f=stdout, detail=True):
527
    header = header in d and header or 'subdir'
528
    if header and header in d:
529
        f.write('%s\n' %d.pop(header))
530
    if detail:
531
        patterns = ['^x_(account|container|object)_meta_(\w+)$']
532
        patterns.append(patterns[0].replace('_', '-'))
533
        for key, val in sorted(d.items()):
534
            for p in patterns:
535
                p = re.compile(p)
536
                m = p.match(key)
537
                if m:
538
                    key = m.group(2)
539
            f.write('%s: %s\n' % (key.rjust(30), val))
540

    
541
def print_list(l, verbose=False, f=stdout, detail=True):
542
    for elem in l:
543
        #if it's empty string continue
544
        if not elem:
545
            continue
546
        if type(elem) == types.DictionaryType:
547
            print_dict(elem, f=f, detail=detail)
548
        elif type(elem) == types.StringType:
549
            if not verbose:
550
                elem = elem.split('Traceback')[0]
551
            f.write('%s\n' % elem)
552
        else:
553
            f.write('%s\n' % elem)
554

    
555
def print_versions(data, f=stdout):
556
    if 'versions' not in data:
557
        f.write('%s\n' %data)
558
        return
559
    f.write('versions:\n')
560
    for id, t in data['versions']:
561
        f.write('%s @ %s\n' % (str(id).rjust(30), datetime.fromtimestamp(t)))
562

    
563
def main():
564
    try:
565
        name = argv[1]
566
        cls = class_for_cli_command(name)
567
    except (IndexError, KeyError):
568
        print_usage()
569
        exit(1)
570
    
571
    cmd = cls(argv[2:])
572
    
573
    try:
574
        cmd.execute(*cmd.args)
575
    except TypeError, e:
576
        print e
577
        cmd.parser.usage = '%%prog %s [options] %s' % (name, cmd.syntax)
578
        cmd.parser.print_help()
579
        exit(1)
580
    except Fault, f:
581
        print f.status, f.data
582
        status = f.status and '%s ' % f.status or ''
583
        print '%s%s' % (status, f.data)
584

    
585
if __name__ == '__main__':
586
    main()