Statistics
| Branch: | Tag: | Revision:

root / tools / store @ 961f2fbe

History | View | Annotate | Download (23.3 kB)

1
#!/usr/bin/env python
2

    
3
# Copyright 2011 GRNET S.A. All rights reserved.
4
# 
5
# Redistribution and use in source and binary forms, with or
6
# without modification, are permitted provided that the following
7
# conditions are met:
8
# 
9
#   1. Redistributions of source code must retain the above
10
#      copyright notice, this list of conditions and the following
11
#      disclaimer.
12
# 
13
#   2. Redistributions in binary form must reproduce the above
14
#      copyright notice, this list of conditions and the following
15
#      disclaimer in the documentation and/or other materials
16
#      provided with the distribution.
17
# 
18
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29
# POSSIBILITY OF SUCH DAMAGE.
30
# 
31
# The views and conclusions contained in the software and
32
# documentation are those of the authors and should not be
33
# interpreted as representing official policies, either expressed
34
# or implied, of GRNET S.A.
35

    
36
from getpass import getuser
37
from optparse import OptionParser
38
from os.path import basename
39
from sys import argv, exit, stdin, stdout
40
from pithos.lib.client import Client, Fault
41
from datetime import datetime
42

    
43
import json
44
import logging
45
import types
46
import re
47
import time as _time
48

    
49
DEFAULT_HOST = 'pithos.dev.grnet.gr'
50
DEFAULT_API = 'v1'
51

    
52
_cli_commands = {}
53

    
54
def cli_command(*args):
55
    def decorator(cls):
56
        cls.commands = args
57
        for name in args:
58
            _cli_commands[name] = cls
59
        return cls
60
    return decorator
61

    
62
def class_for_cli_command(name):
63
    return _cli_commands[name]
64

    
65
class Command(object):
66
    def __init__(self, argv):
67
        parser = OptionParser()
68
        parser.add_option('--host', dest='host', metavar='HOST',
69
                          default=DEFAULT_HOST, help='use server HOST')
70
        parser.add_option('--user', dest='user', metavar='USERNAME',
71
                          default=getuser(), help='use account USERNAME')
72
        parser.add_option('--api', dest='api', metavar='API',
73
                          default=DEFAULT_API, help='use api API')
74
        parser.add_option('-v', action='store_true', dest='verbose',
75
                          default=False, help='use verbose output')
76
        parser.add_option('-d', action='store_true', dest='debug',
77
                          default=False, help='use debug output')
78
        self.add_options(parser)
79
        options, args = parser.parse_args(argv)
80
        
81
        # Add options to self
82
        for opt in parser.option_list:
83
            key = opt.dest
84
            if key:
85
                val = getattr(options, key)
86
                setattr(self, key, val)
87
        
88
        self.client = Client(self.host, self.user, self.api, self.verbose,
89
                             self.debug)
90
        
91
        self.parser = parser
92
        self.args = args
93
        
94
    def add_options(self, parser):
95
        pass
96
    
97
    def execute(self, *args):
98
        pass
99

    
100
@cli_command('list', 'ls')
101
class List(Command):
102
    syntax = '[<container>[/<object>]]'
103
    description = 'list containers or objects'
104
    
105
    def add_options(self, parser):
106
        parser.add_option('-l', action='store_true', dest='detail',
107
                          default=False, help='show detailed output')
108
        parser.add_option('-n', action='store', type='int', dest='limit',
109
                          default=1000, help='show limited output')
110
        parser.add_option('--marker', action='store', type='str',
111
                          dest='marker', default=None,
112
                          help='show output greater then marker')
113
        parser.add_option('--prefix', action='store', type='str',
114
                          dest='prefix', default=None,
115
                          help='show output starting with prefix')
116
        parser.add_option('--delimiter', action='store', type='str',
117
                          dest='delimiter', default=None,
118
                          help='show output up to the delimiter')
119
        parser.add_option('--path', action='store', type='str',
120
                          dest='path', default=None,
121
                          help='show output starting with prefix up to /')
122
        parser.add_option('--meta', action='store', type='str',
123
                          dest='meta', default=None,
124
                          help='show output having the specified meta keys')
125
        parser.add_option('--if-modified-since', action='store', type='str',
126
                          dest='if_modified_since', default=None,
127
                          help='show output if modified since then')
128
        parser.add_option('--if-unmodified-since', action='store', type='str',
129
                          dest='if_unmodified_since', default=None,
130
                          help='show output if not modified since then')
131
        parser.add_option('--until', action='store', dest='until',
132
                          default=False, help='show metadata until that date')
133
        parser.add_option('--format', action='store', dest='format',
134
                          default='%d/%m/%Y', help='format to parse until date')
135
    
136
    def execute(self, container=None):
137
        if container:
138
            self.list_objects(container)
139
        else:
140
            self.list_containers()
141
    
142
    def list_containers(self):
143
        params = {'limit':self.limit, 'marker':self.marker}
144
        headers = {'IF_MODIFIED_SINCE':self.if_modified_since,
145
                   'IF_UNMODIFIED_SINCE':self.if_unmodified_since}
146
        
147
        if self.until:
148
            t = _time.strptime(self.until, self.format)
149
            params['until'] = int(_time.mktime(t))
150
        
151
        l = self.client.list_containers(self.detail, params, headers)
152
        print_list(l)
153
    
154
    def list_objects(self, container):
155
        params = {'limit':self.limit, 'marker':self.marker,
156
                  'prefix':self.prefix, 'delimiter':self.delimiter,
157
                  'path':self.path, 'meta':self.meta}
158
        headers = {'IF_MODIFIED_SINCE':self.if_modified_since,
159
                   'IF_UNMODIFIED_SINCE':self.if_unmodified_since}
160
        container, sep, object = container.partition('/')
161
        if object:
162
            return
163
        
164
        if self.until:
165
            t = _time.strptime(self.until, self.format)
166
            params['until'] = int(_time.mktime(t))
167
        
168
        detail = 'json'
169
        l = self.client.list_objects(container, detail, params, headers)
170
        print_list(l, detail=self.detail)
171

    
172
@cli_command('meta')
173
class Meta(Command):
174
    syntax = '[<container>[/<object>]]'
175
    description = 'get the metadata of an account, a container or an object'
176
    
177
    def add_options(self, parser):
178
        parser.add_option('-r', action='store_true', dest='restricted',
179
                          default=False, help='show only user defined metadata')
180
        parser.add_option('--until', action='store', dest='until',
181
                          default=False, help='show metadata until that date')
182
        parser.add_option('--format', action='store', dest='format',
183
                          default='%d/%m/%Y', help='format to parse until date')
184
        parser.add_option('--version', action='store', dest='version',
185
                          default=None, help='show specific version \
186
                                  (applies only for objects)')
187
    
188
    def execute(self, path=''):
189
        container, sep, object = path.partition('/')
190
        if self.until:
191
            t = _time.strptime(self.until, self.format)
192
            self.until = int(_time.mktime(t))
193
        if object:
194
            meta = self.client.retrieve_object_metadata(container, object,
195
                                                        self.restricted,
196
                                                        self.version)
197
        elif container:
198
            meta = self.client.retrieve_container_metadata(container,
199
                                                           self.restricted,
200
                                                           self.until)
201
        else:
202
            meta = self.client.account_metadata(self.restricted, self.until)
203
        if meta == None:
204
            print 'Entity does not exist'
205
        else:
206
            print_dict(meta, header=None)
207

    
208
@cli_command('create')
209
class CreateContainer(Command):
210
    syntax = '<container> [key=val] [...]'
211
    description = 'create a container'
212
    
213
    def execute(self, container, *args):
214
        headers = {}
215
        for arg in args:
216
            key, sep, val = arg.partition('=')
217
            headers['X_CONTAINER_META_%s' %key.strip().upper()] = val.strip()
218
        ret = self.client.create_container(container, headers)
219
        if not ret:
220
            print 'Container already exists'
221

    
222
@cli_command('delete', 'rm')
223
class Delete(Command):
224
    syntax = '<container>[/<object>]'
225
    description = 'delete a container or an object'
226
    
227
    def execute(self, path):
228
        container, sep, object = path.partition('/')
229
        if object:
230
            self.client.delete_object(container, object)
231
        else:
232
            self.client.delete_container(container)
233

    
234
@cli_command('get')
235
class GetObject(Command):
236
    syntax = '<container>/<object>'
237
    description = 'get the data of an object'
238
    
239
    def add_options(self, parser):
240
        parser.add_option('-l', action='store_true', dest='detail',
241
                          default=False, help='show detailed output')
242
        parser.add_option('--range', action='store', dest='range',
243
                          default=None, help='show range of data')
244
        parser.add_option('--if-match', action='store', dest='if-match',
245
                          default=None, help='show output if ETags match')
246
        parser.add_option('--if-none-match', action='store',
247
                          dest='if-none-match', default=None,
248
                          help='show output if ETags don\'t match')
249
        parser.add_option('--if-modified-since', action='store', type='str',
250
                          dest='if-modified-since', default=None,
251
                          help='show output if modified since then')
252
        parser.add_option('--if-unmodified-since', action='store', type='str',
253
                          dest='if-unmodified-since', default=None,
254
                          help='show output if not modified since then')
255
        parser.add_option('-f', action='store', type='str',
256
                          dest='file', default=None,
257
                          help='save output in file')
258
        parser.add_option('--version', action='store', type='str',
259
                          dest='version', default=None,
260
                          help='get the specific \
261
                               version')
262
        parser.add_option('--versionlist', action='store_true',
263
                          dest='versionlist', default=False,
264
                          help='get the full object version list')
265
    
266
    def execute(self, path):
267
        headers = {}
268
        if self.range:
269
            headers['RANGE'] = 'bytes=%s' %self.range
270
        attrs = ['if-match', 'if-none-match', 'if-modified-since',
271
                 'if-unmodified-since']
272
        attrs = [a for a in attrs if getattr(self, a)]
273
        for a in attrs:
274
            headers[a.replace('-', '_').upper()] = getattr(self, a)
275
        container, sep, object = path.partition('/')
276
        if self.versionlist:
277
            self.version = 'list'
278
            self.detail = True
279
        data = self.client.retrieve_object(container, object, self.detail,
280
                                          headers, self.version)
281
        f = self.file and open(self.file, 'w') or stdout
282
        if self.detail:
283
            data = json.loads(data)
284
            if self.versionlist:
285
                print_versions(data, f=f)
286
            else:
287
                print_dict(data, f=f)
288
        else:
289
            f.write(data)
290
        f.close()
291

    
292
@cli_command('mkdir')
293
class PutMarker(Command):
294
    syntax = '<container>/<directory marker>'
295
    description = 'create a directory marker'
296
    
297
    def execute(self, path):
298
        container, sep, object = path.partition('/')
299
        self.client.create_directory_marker(container, object)
300

    
301
@cli_command('put')
302
class PutObject(Command):
303
    syntax = '<container>/<object> <path> [key=val] [...]'
304
    description = 'create/override object with path contents or standard input'
305
    
306
    def add_options(self, parser):
307
        parser.add_option('--chunked', action='store_true', dest='chunked',
308
                          default=False, help='set chunked transfer mode')
309
        parser.add_option('--etag', action='store', dest='etag',
310
                          default=None, help='check written data')
311
        parser.add_option('--content-encoding', action='store',
312
                          dest='content-encoding', default=None,
313
                          help='provide the object MIME content type')
314
        parser.add_option('--content-disposition', action='store', type='str',
315
                          dest='content-disposition', default=None,
316
                          help='provide the presentation style of the object')
317
        parser.add_option('-S', action='store',
318
                          dest='segment-size', default=False,
319
                          help='use for large file support')
320
        parser.add_option('--manifest', action='store_true',
321
                          dest='manifest', default=None,
322
                          help='upload a manifestation file')
323
        parser.add_option('--type', action='store',
324
                          dest='content-type', default=False,
325
                          help='create object with specific content type')
326
        parser.add_option('--touch', action='store_true',
327
                          dest='touch', default=False,
328
                          help='create object with zero data')
329
    
330
    def execute(self, path, srcpath='-', *args):
331
        headers = {}
332
        if self.manifest:
333
            headers['X_OBJECT_MANIFEST'] = self.manifest
334
        
335
        attrs = ['etag', 'content-encoding', 'content-disposition',
336
                 'content-type']
337
        attrs = [a for a in attrs if getattr(self, a)]
338
        for a in attrs:
339
            headers[a.replace('-', '_').upper()] = getattr(self, a)
340
        
341
        #prepare user defined meta
342
        for arg in args:
343
            key, sep, val = arg.partition('=')
344
            headers['X_OBJECT_META_%s' %key.strip().upper()] = val.strip()
345
        
346
        container, sep, object = path.partition('/')
347
        
348
        f = None
349
        chunked = False
350
        if not self.touch:
351
            f = srcpath != '-' and open(srcpath) or stdin
352
            chunked = (self.chunked or f == stdin) and True or False
353
        self.client.create_object(container, object, f, chunked=chunked,
354
                                  headers=headers)
355
        if f:
356
            f.close()
357

    
358
@cli_command('copy', 'cp')
359
class CopyObject(Command):
360
    syntax = '<src container>/<src object> [<dst container>/]<dst object>'
361
    description = 'copies an object to a different location'
362
    
363
    def add_options(self, parser):
364
        parser.add_option('--version', action='store',
365
                          dest='version', default=False,
366
                          help='copy specific version')
367
    
368
    def execute(self, src, dst):
369
        src_container, sep, src_object = src.partition('/')
370
        dst_container, sep, dst_object = dst.partition('/')
371
        if not sep:
372
            dst_container = src_container
373
            dst_object = dst
374
        version = getattr(self, 'version')
375
        if version:
376
            headers = {}
377
            headers['X_SOURCE_VERSION'] = version
378
        self.client.copy_object(src_container, src_object, dst_container,
379
                                dst_object, headers)
380

    
381
@cli_command('set')
382
class SetMeta(Command):
383
    syntax = '[<container>[/<object>]] key=val [key=val] [...]'
384
    description = 'set metadata'
385
    
386
    def execute(self, path, *args):
387
        #in case of account fix the args
388
        if path.find('=') != -1:
389
            args = list(args)
390
            args.append(path)
391
            args = tuple(args)
392
            path = ''
393
        meta = {}
394
        for arg in args:
395
            key, sep, val = arg.partition('=')
396
            meta[key.strip()] = val.strip()
397
        container, sep, object = path.partition('/')
398
        if object:
399
            self.client.update_object_metadata(container, object, **meta)
400
        elif container:
401
            self.client.update_container_metadata(container, **meta)
402
        else:
403
            self.client.update_account_metadata(**meta)
404

    
405
@cli_command('update')
406
class UpdateObject(Command):
407
    syntax = '<container>/<object> path [key=val] [...]'
408
    description = 'update object metadata/data (default mode: append)'
409
    
410
    def add_options(self, parser):
411
        parser.add_option('-a', action='store_true', dest='append',
412
                          default=True, help='append data')
413
        parser.add_option('--start', action='store',
414
                          dest='start',
415
                          default=None, help='range of data to be updated')
416
        parser.add_option('--range', action='store', dest='content-range',
417
                          default=None, help='range of data to be updated')
418
        parser.add_option('--chunked', action='store_true', dest='chunked',
419
                          default=False, help='set chunked transfer mode')
420
        parser.add_option('--content-encoding', action='store',
421
                          dest='content-encoding', default=None,
422
                          help='provide the object MIME content type')
423
        parser.add_option('--content-disposition', action='store', type='str',
424
                          dest='content-disposition', default=None,
425
                          help='provide the presentation style of the object')
426
        parser.add_option('--manifest', action='store', type='str',
427
                          dest='manifest', default=None,
428
                          help='use for large file support')
429
    
430
    def execute(self, path, srcpath='-', *args):
431
        headers = {}
432
        if self.manifest:
433
            headers['X_OBJECT_MANIFEST'] = self.manifest
434
        
435
        if getattr(self, 'start'):
436
            headers['CONTENT_RANGE'] = 'bytes %s-/*' % getattr(self, 'start')
437
        elif self.append:
438
            headers['CONTENT_RANGE'] = 'bytes */*'
439
        
440
        attrs = ['content-encoding', 'content-disposition']
441
        attrs = [a for a in attrs if getattr(self, a)]
442
        for a in attrs:
443
            headers[a.replace('-', '_').upper()] = getattr(self, a)
444
        
445
        #prepare user defined meta
446
        for arg in args:
447
            key, sep, val = arg.partition('=')
448
            headers['X_OBJECT_META_%s' %key.strip().upper()] = val.strip()
449
        
450
        container, sep, object = path.partition('/')
451
        
452
        f = srcpath != '-' and open(srcpath) or stdin
453
        chunked = (self.chunked or f == stdin) and True or False
454
        self.client.update_object(container, object, f, chunked=chunked,
455
                                  headers=headers)
456
        f.close()
457

    
458
@cli_command('move', 'mv')
459
class MoveObject(Command):
460
    syntax = '<src container>/<src object> [<dst container>/]<dst object>'
461
    description = 'moves an object to a different location'
462
    
463
    def execute(self, src, dst):
464
        src_container, sep, src_object = src.partition('/')
465
        dst_container, sep, dst_object = dst.partition('/')
466
        if not sep:
467
            dst_container = src_container
468
            dst_object = dst
469
        
470
        self.client.move_object(src_container, src_object, dst_container,
471
                                dst_object, headers)
472

    
473
@cli_command('remove', 'rm')
474
class TrashObject(Command):
475
    syntax = '<container>/<object>'
476
    description = 'trashes an object'
477
    
478
    def execute(self, src):
479
        src_container, sep, src_object = src.partition('/')
480
        
481
        self.client.trash_object(src_container, src_object)
482

    
483
@cli_command('restore')
484
class TrashObject(Command):
485
    syntax = '<container>/<object>'
486
    description = 'trashes an object'
487
    
488
    def execute(self, src):
489
        src_container, sep, src_object = src.partition('/')
490
        
491
        self.client.restore_object(src_container, src_object)
492

    
493
@cli_command('unset')
494
class TrashObject(Command):
495
    syntax = '<container>/[<object>] key [key] [...]'
496
    description = 'deletes metadata info'
497
    
498
    def execute(self, path, *args):
499
        #in case of account fix the args
500
        if len(args) == 0:
501
            args = list(args)
502
            args.append(path)
503
            args = tuple(args)
504
            path = ''
505
        meta = []
506
        for key in args:
507
            meta.append(key)
508
        container, sep, object = path.partition('/')
509
        if object:
510
            self.client.delete_object_metadata(container, object, meta)
511
        elif container:
512
            self.client.delete_container_metadata(container, meta)
513
        else:
514
            self.client.delete_account_metadata(meta)
515

    
516
def print_usage():
517
    cmd = Command([])
518
    parser = cmd.parser
519
    parser.usage = '%prog <command> [options]'
520
    parser.print_help()
521
    
522
    commands = []
523
    for cls in set(_cli_commands.values()):
524
        name = ', '.join(cls.commands)
525
        description = getattr(cls, 'description', '')
526
        commands.append('  %s %s' % (name.ljust(12), description))
527
    print '\nCommands:\n' + '\n'.join(sorted(commands))
528

    
529
def print_dict(d, header='name', f=stdout, detail=True):
530
    header = header in d and header or 'subdir'
531
    if header and header in d:
532
        f.write('%s\n' %d.pop(header))
533
    if detail:
534
        patterns = ['^x_(account|container|object)_meta_(\w+)$']
535
        patterns.append(patterns[0].replace('_', '-'))
536
        for key, val in sorted(d.items()):
537
            for p in patterns:
538
                p = re.compile(p)
539
                m = p.match(key)
540
                if m:
541
                    key = m.group(2)
542
            f.write('%s: %s\n' % (key.rjust(30), val))
543

    
544
def print_list(l, verbose=False, f=stdout, detail=True):
545
    for elem in l:
546
        #if it's empty string continue
547
        if not elem:
548
            continue
549
        if type(elem) == types.DictionaryType:
550
            print_dict(elem, f=f, detail=detail)
551
        elif type(elem) == types.StringType:
552
            if not verbose:
553
                elem = elem.split('Traceback')[0]
554
            f.write('%s\n' % elem)
555
        else:
556
            f.write('%s\n' % elem)
557

    
558
def print_versions(data, f=stdout):
559
    if 'versions' not in data:
560
        f.write('%s\n' %data)
561
        return
562
    f.write('versions:\n')
563
    for id, t in data['versions']:
564
        f.write('%s @ %s\n' % (str(id).rjust(30), datetime.fromtimestamp(t)))
565

    
566
def main():
567
    try:
568
        name = argv[1]
569
        cls = class_for_cli_command(name)
570
    except (IndexError, KeyError):
571
        print_usage()
572
        exit(1)
573
    
574
    cmd = cls(argv[2:])
575
    
576
    try:
577
        cmd.execute(*cmd.args)
578
    except TypeError, e:
579
        print e
580
        cmd.parser.usage = '%%prog %s [options] %s' % (name, cmd.syntax)
581
        cmd.parser.print_help()
582
        exit(1)
583
    except Fault, f:
584
        print f.status, f.data
585
        status = f.status and '%s ' % f.status or ''
586
        print '%s%s' % (status, f.data)
587

    
588
if __name__ == '__main__':
589
    main()