Statistics
| Branch: | Tag: | Revision:

root / tools / store @ ad71a0ce

History | View | Annotate | Download (19.6 kB)

1
#!/usr/bin/env python
2

    
3
from getpass import getuser
4
from optparse import OptionParser
5
from os.path import basename
6
from sys import argv, exit, stdin, stdout
7
from pithos.lib.client import Client, Fault
8
from datetime import datetime
9

    
10
import json
11
import logging
12
import types
13
import re
14
import time as _time
15

    
16
DEFAULT_HOST = 'pithos.dev.grnet.gr'
17
DEFAULT_API = 'v1'
18

    
19
_cli_commands = {}
20

    
21
def cli_command(*args):
22
    def decorator(cls):
23
        cls.commands = args
24
        for name in args:
25
            _cli_commands[name] = cls
26
        return cls
27
    return decorator
28

    
29
def class_for_cli_command(name):
30
    return _cli_commands[name]
31

    
32
class Command(object):
33
    def __init__(self, argv):
34
        parser = OptionParser()
35
        parser.add_option('--host', dest='host', metavar='HOST',
36
                          default=DEFAULT_HOST, help='use server HOST')
37
        parser.add_option('--user', dest='user', metavar='USERNAME',
38
                          default=getuser(), help='use account USERNAME')
39
        parser.add_option('--api', dest='api', metavar='API',
40
                          default=DEFAULT_API, help='use api API')
41
        parser.add_option('-v', action='store_true', dest='verbose',
42
                          default=False, help='use verbose output')
43
        parser.add_option('-d', action='store_true', dest='debug',
44
                          default=False, help='use debug output')
45
        self.add_options(parser)
46
        options, args = parser.parse_args(argv)
47
        
48
        # Add options to self
49
        for opt in parser.option_list:
50
            key = opt.dest
51
            if key:
52
                val = getattr(options, key)
53
                setattr(self, key, val)
54
        
55
        self.client = Client(self.host, self.user, self.api, self.verbose,
56
                             self.debug)
57
        
58
        self.parser = parser
59
        self.args = args
60

    
61
    def add_options(self, parser):
62
        pass
63

    
64
    def execute(self, *args):
65
        pass
66

    
67
@cli_command('list', 'ls')
68
class List(Command):
69
    syntax = '[<container>[/<object>]]'
70
    description = 'list containers or objects'
71
    
72
    def add_options(self, parser):
73
        parser.add_option('-l', action='store_true', dest='detail',
74
                          default=False, help='show detailed output')
75
        parser.add_option('-n', action='store', type='int', dest='limit',
76
                          default=1000, help='show limited output')
77
        parser.add_option('--marker', action='store', type='str',
78
                          dest='marker', default=None,
79
                          help='show output greater then marker')
80
        parser.add_option('--prefix', action='store', type='str',
81
                          dest='prefix', default=None,
82
                          help='show output starting with prefix')
83
        parser.add_option('--delimiter', action='store', type='str',
84
                          dest='delimiter', default=None,
85
                          help='show output up to the delimiter')
86
        parser.add_option('--path', action='store', type='str',
87
                          dest='path', default=None,
88
                          help='show output starting with prefix up to /')
89
        parser.add_option('--meta', action='store', type='str',
90
                          dest='meta', default=None,
91
                          help='show output having the specified meta keys')
92
        parser.add_option('--if-modified-since', action='store', type='str',
93
                          dest='if_modified_since', default=None,
94
                          help='show output if modified since then')
95
        parser.add_option('--if-unmodified-since', action='store', type='str',
96
                          dest='if_unmodified_since', default=None,
97
                          help='show output if not modified since then')
98
        parser.add_option('--until', action='store', dest='until',
99
                          default=False, help='show metadata until that date')
100
        parser.add_option('--format', action='store', dest='format',
101
                          default='%d/%m/%Y', help='format to parse until date')
102

    
103
    def execute(self, container=None):
104
        if container:
105
            self.list_objects(container)
106
        else:
107
            self.list_containers()
108

    
109
    def list_containers(self):
110
        params = {'limit':self.limit, 'marker':self.marker}
111
        headers = {'IF_MODIFIED_SINCE':self.if_modified_since,
112
                   'IF_UNMODIFIED_SINCE':self.if_unmodified_since}
113
        
114
        if self.until:
115
            t = _time.strptime(self.until, self.format)
116
            params['until'] = int(_time.mktime(t))
117
        
118
        l = self.client.list_containers(self.detail, params, headers)
119
        print_list(l)
120

    
121
    def list_objects(self, container):
122
        params = {'limit':self.limit, 'marker':self.marker,
123
                  'prefix':self.prefix, 'delimiter':self.delimiter,
124
                  'path':self.path, 'meta':self.meta}
125
        headers = {'IF_MODIFIED_SINCE':self.if_modified_since,
126
                   'IF_UNMODIFIED_SINCE':self.if_unmodified_since}
127
        container, sep, object = container.partition('/')
128
        if object:
129
            print '%s/%s is an object' %(container, object)
130
            return
131
        
132
        if self.until:
133
            t = _time.strptime(self.until, self.format)
134
            params['until'] = int(_time.mktime(t))
135
        
136
        l = self.client.list_objects(container, self.detail, params, headers)
137
        print_list(l)
138

    
139
@cli_command('meta')
140
class Meta(Command):
141
    syntax = '[<container>[/<object>]]'
142
    description = 'get the metadata of an account, a container or an object'
143
    
144
    def add_options(self, parser):
145
        parser.add_option('-r', action='store_true', dest='restricted',
146
                          default=False, help='show only user defined metadata')
147
        parser.add_option('--until', action='store', dest='until',
148
                          default=False, help='show metadata until that date')
149
        parser.add_option('--format', action='store', dest='format',
150
                          default='%d/%m/%Y', help='format to parse until date')
151
        parser.add_option('--version', action='store', dest='version',
152
                          default=None, help='show specific version \
153
                                  (applies only for objects)')
154

    
155
    def execute(self, path=''):
156
        container, sep, object = path.partition('/')
157
        if self.until:
158
            t = _time.strptime(self.until, self.format)
159
            self.until = int(_time.mktime(t))
160
        if object:
161
            meta = self.client.retrieve_object_metadata(container, object,
162
                                                        self.restricted,
163
                                                        self.version)
164
        elif container:
165
            meta = self.client.retrieve_container_metadata(container,
166
                                                           self.restricted,
167
                                                           self.until)
168
        else:
169
            meta = self.client.account_metadata(self.restricted, self.until)
170
        if meta == None:
171
            print 'Entity does not exist'
172
        else:
173
            print_dict(meta, header=None)
174

    
175
@cli_command('create')
176
class CreateContainer(Command):
177
    syntax = '<container> [key=val] [...]'
178
    description = 'create a container'
179
    
180
    def execute(self, container, *args):
181
        headers = {}
182
        for arg in args:
183
            key, sep, val = arg.partition('=')
184
            headers['X_CONTAINER_META_%s' %key.strip().upper()] = val.strip()
185
        ret = self.client.create_container(container, headers)
186
        if not ret:
187
            print 'Container already exists'
188

    
189
@cli_command('delete', 'rm')
190
class Delete(Command):
191
    syntax = '<container>[/<object>]'
192
    description = 'delete a container or an object'
193
    
194
    def execute(self, path):
195
        container, sep, object = path.partition('/')
196
        if object:
197
            self.client.delete_object(container, object)
198
        else:
199
            self.client.delete_container(container)
200

    
201
@cli_command('get')
202
class GetObject(Command):
203
    syntax = '<container>/<object>'
204
    description = 'get the data of an object'
205
    
206
    def add_options(self, parser):
207
        parser.add_option('-l', action='store_true', dest='detail',
208
                          default=False, help='show detailed output')
209
        parser.add_option('--range', action='store', dest='range',
210
                          default=None, help='show range of data')
211
        parser.add_option('--if-match', action='store', dest='if-match',
212
                          default=None, help='show output if ETags match')
213
        parser.add_option('--if-none-match', action='store',
214
                          dest='if-none-match', default=None,
215
                          help='show output if ETags don\'t match')
216
        parser.add_option('--if-modified-since', action='store', type='str',
217
                          dest='if-modified-since', default=None,
218
                          help='show output if modified since then')
219
        parser.add_option('--if-unmodified-since', action='store', type='str',
220
                          dest='if-unmodified-since', default=None,
221
                          help='show output if not modified since then')
222
        parser.add_option('-f', action='store', type='str',
223
                          dest='file', default=None,
224
                          help='save output in file')
225
        parser.add_option('--version', action='store', type='str',
226
                          dest='version', default=None,
227
                          help='if \'list\' and in detailed mode get object\'s \
228
                               full version list otherwise get the specific \
229
                               version')
230

    
231
    def execute(self, path):
232
        headers = {}
233
        if self.range:
234
            headers['RANGE'] = 'bytes=%s' %self.range
235
        attrs = ['if-match', 'if-none-match', 'if-modified-since',
236
                 'if-unmodified-since']
237
        attrs = [a for a in attrs if getattr(self, a)]
238
        for a in attrs:
239
            headers[a.replace('-', '_').upper()] = getattr(self, a)
240
        container, sep, object = path.partition('/')
241
        data = self.client.retrieve_object(container, object, self.detail,
242
                                          headers, self.version)
243
        f = self.file and open(self.file, 'w') or stdout
244
        if self.detail:
245
            data = json.loads(data)
246
            if self.version == 'list':
247
                print_versions(data, f=f)
248
            else:
249
                print_dict(data, f=f)
250
        else:
251
            f.write(data)
252
        f.close()
253

    
254
@cli_command('put')
255
class PutObject(Command):
256
    syntax = '<container>/<object> <path> [key=val] [...]'
257
    description = 'create/override object with path contents or standard input'
258

    
259
    def add_options(self, parser):
260
        parser.add_option('--chunked', action='store_true', dest='chunked',
261
                          default=False, help='set chunked transfer mode')
262
        parser.add_option('--etag', action='store', dest='etag',
263
                          default=None, help='check written data')
264
        parser.add_option('--content-encoding', action='store',
265
                          dest='content-encoding', default=None,
266
                          help='provide the object MIME content type')
267
        parser.add_option('--content-disposition', action='store', type='str',
268
                          dest='content-disposition', default=None,
269
                          help='provide the presentation style of the object')
270
        parser.add_option('--manifest', action='store', type='str',
271
                          dest='manifest', default=None,
272
                          help='use for large file support')
273
        parser.add_option('--touch', action='store_true',
274
                          dest='touch', default=False,
275
                          help='create object with zero data')
276

    
277
    def execute(self, path, srcpath='-', *args):
278
        headers = {}
279
        if self.manifest:
280
            headers['X_OBJECT_MANIFEST'] = self.manifest
281
            
282
        attrs = ['etag', 'content-encoding', 'content-disposition']
283
        attrs = [a for a in attrs if getattr(self, a)]
284
        for a in attrs:
285
            headers[a.replace('-', '_').upper()] = getattr(self, a)
286
        
287
        #prepare user defined meta
288
        for arg in args:
289
            key, sep, val = arg.partition('=')
290
            headers['X_OBJECT_META_%s' %key.strip().upper()] = val.strip()
291
        
292
        container, sep, object = path.partition('/')
293
        
294
        f = None
295
        chunked = False
296
        if not self.touch:
297
            f = srcpath != '-' and open(srcpath) or stdin
298
            chunked = (self.chunked or f == stdin) and True or False
299
        self.client.create_object(container, object, f, chunked=chunked,
300
                                  headers=headers)
301
        if f:
302
            f.close()
303

    
304
@cli_command('copy', 'cp')
305
class CopyObject(Command):
306
    syntax = '<src container>/<src object> [<dst container>/]<dst object>'
307
    description = 'copies an object to a different location'
308
    
309
    def add_options(self, parser):
310
        parser.add_option('--version', action='store',
311
                          dest='version', default=False,
312
                          help='copy specific version')
313

    
314
    def execute(self, src, dst):
315
        src_container, sep, src_object = src.partition('/')
316
        dst_container, sep, dst_object = dst.partition('/')
317
        if not sep:
318
            dst_container = src_container
319
            dst_object = dst
320
        version = getattr(self, 'version')
321
        if version:
322
            headers = {}
323
            headers['SOURCE_VERSION'] = version 
324
        self.client.copy_object(src_container, src_object, dst_container,
325
                                dst_object)
326

    
327
@cli_command('set')
328
class SetMeta(Command):
329
    syntax = '[<container>[/<object>]] key=val [key=val] [...]'
330
    description = 'set metadata'
331
    
332
    def execute(self, path, *args):
333
        #in case of account fix the args
334
        if path.find('=') != -1:
335
            args = list(args)
336
            args.append(path)
337
            args = tuple(args)
338
            path = ''
339
        meta = {}
340
        for arg in args:
341
            key, sep, val = arg.partition('=')
342
            meta[key.strip()] = val.strip()
343
        container, sep, object = path.partition('/')
344
        if object:
345
            self.client.update_object_metadata(container, object, **meta)
346
        elif container:
347
            self.client.update_container_metadata(container, **meta)
348
        else:
349
            self.client.update_account_metadata(**meta)
350

    
351
@cli_command('update')
352
class UpdateObject(Command):
353
    syntax = '<container>/<object> path [key=val] [...]'
354
    description = 'update object metadata/data (default mode: append)'
355
    
356
    def add_options(self, parser):
357
        parser.add_option('-a', action='store_true', dest='append',
358
                          default=True, help='append data')
359
        parser.add_option('--start', action='store',
360
                          dest='start',
361
                          default=None, help='range of data to be updated')
362
        parser.add_option('--range', action='store', dest='content-range',
363
                          default=None, help='range of data to be updated')
364
        parser.add_option('--chunked', action='store_true', dest='chunked',
365
                          default=False, help='set chunked transfer mode')
366
        parser.add_option('--content-encoding', action='store',
367
                          dest='content-encoding', default=None,
368
                          help='provide the object MIME content type')
369
        parser.add_option('--content-disposition', action='store', type='str',
370
                          dest='content-disposition', default=None,
371
                          help='provide the presentation style of the object')
372
        parser.add_option('--manifest', action='store', type='str',
373
                          dest='manifest', default=None,
374
                          help='use for large file support')
375

    
376
    def execute(self, path, srcpath='-', *args):
377
        headers = {}
378
        if self.manifest:
379
            headers['X_OBJECT_MANIFEST'] = self.manifest
380
        
381
        if self.append:
382
            headers['CONTENT_RANGE'] = 'bytes */*'
383
        elif self.start:
384
            headers['CONTENT_RANGE'] = 'bytes %s-/*' % self.first-byte-pos
385
        
386
        attrs = ['content-encoding', 'content-disposition']
387
        attrs = [a for a in attrs if getattr(self, a)]
388
        for a in attrs:
389
            headers[a.replace('-', '_').upper()] = getattr(self, a)
390
        
391
        #prepare user defined meta
392
        for arg in args:
393
            key, sep, val = arg.partition('=')
394
            headers['X_OBJECT_META_%s' %key.strip().upper()] = val.strip()
395
        
396
        container, sep, object = path.partition('/')
397
        
398
        f = srcpath != '-' and open(srcpath) or stdin
399
        chunked = (self.chunked or f == stdin) and True or False
400
        self.client.update_object(container, object, f, chunked=chunked,
401
                                  headers=headers)
402
        f.close()
403

    
404
@cli_command('move', 'mv')
405
class MoveObject(Command):
406
    syntax = '<src container>/<src object> [<dst container>/]<dst object>'
407
    description = 'moves an object to a different location'
408
    
409
    def add_options(self, parser):
410
        parser.add_option('--version', action='store',
411
                          dest='version', default=False,
412
                          help='move specific version')
413

    
414
    def execute(self, src, dst):
415
        src_container, sep, src_object = src.partition('/')
416
        dst_container, sep, dst_object = dst.partition('/')
417
        if not sep:
418
            dst_container = src_container
419
            dst_object = dst
420
        self.client.move_object(src_container, src_object, dst_container,
421
                                dst_object)
422

    
423
def print_usage():
424
    cmd = Command([])
425
    parser = cmd.parser
426
    parser.usage = '%prog <command> [options]'
427
    parser.print_help()
428
    
429
    commands = []
430
    for cls in set(_cli_commands.values()):
431
        name = ', '.join(cls.commands)
432
        description = getattr(cls, 'description', '')
433
        commands.append('  %s %s' % (name.ljust(12), description))
434
    print '\nCommands:\n' + '\n'.join(sorted(commands))
435

    
436
def print_dict(d, header='name', f=stdout):
437
    header = header in d and header or 'subdir'
438
    if header and header in d:
439
        f.write('%s\n' %d.pop(header))
440
    patterns = ['^x_(account|container|object)_meta_(\w+)$']
441
    patterns.append(patterns[0].replace('_', '-'))
442
    for key, val in sorted(d.items()):
443
        for p in patterns:
444
            p = re.compile(p)
445
            m = p.match(key)
446
            if m:
447
                key = m.group(2)
448
        f.write('%s: %s\n' % (key.rjust(30), val))
449

    
450
def print_list(l, verbose=False, f=stdout):
451
    for elem in l:
452
        #if it's empty string continue
453
        if not elem:
454
            continue
455
        if type(elem) == types.DictionaryType:
456
            print_dict(elem, f=f)
457
        elif type(elem) == types.StringType:
458
            if not verbose:
459
                elem = elem.split('Traceback')[0]
460
            f.write('%s\n' % elem)
461
        else:
462
            f.write('%s\n' % elem)
463

    
464
def print_versions(data, f=stdout):
465
    if 'versions' not in data:
466
        f.write('%s\n' %data)
467
        return
468
    f.write('versions:\n')
469
    for id, t in data['versions']:
470
        f.write('%s @ %s\n' % (str(id).rjust(30), datetime.fromtimestamp(t)))
471

    
472
def main():
473
    try:
474
        name = argv[1]
475
        cls = class_for_cli_command(name)
476
    except (IndexError, KeyError):
477
        print_usage()
478
        exit(1)
479
    
480
    cmd = cls(argv[2:])
481
    
482
    try:
483
        cmd.execute(*cmd.args)
484
    except TypeError, e:
485
        print e
486
        cmd.parser.usage = '%%prog %s [options] %s' % (name, cmd.syntax)
487
        cmd.parser.print_help()
488
        exit(1)
489
    except Fault, f:
490
        print f.status, f.data
491

    
492
if __name__ == '__main__':
493
    main()