Revision bcb7c5a8

b/pithos/lib/client.py
59 59
        self.status = status
60 60

  
61 61
class Client(object):
62
    def __init__(self, host, account, api='v1', verbose=False, debug=False):
62
    def __init__(self, host, token, account, api='v1', verbose=False, debug=False):
63 63
        """`host` can also include a port, e.g '127.0.0.1:8000'."""
64 64
        
65 65
        self.host = host
......
67 67
        self.api = api
68 68
        self.verbose = verbose or debug
69 69
        self.debug = debug
70
        self.token = token
70 71
    
71 72
    def _chunked_transfer(self, path, method='PUT', f=stdin, headers=None,
72 73
                          blocksize=1024):
......
75 76
        # write header
76 77
        path = '/%s/%s%s' % (self.api, self.account, path)
77 78
        http.putrequest(method, path)
79
        http.putheader('X-Auth-Token', self.token)
78 80
        http.putheader('Content-Type', 'application/octet-stream')
79 81
        http.putheader('Transfer-Encoding', 'chunked')
80 82
        if headers:
......
141 143
        
142 144
        kwargs = {}
143 145
        kwargs['headers'] = headers or {}
146
        kwargs['headers']['X-Auth-Token'] = self.token
144 147
        if not headers or \
145 148
        'Transfer-Encoding' not in headers \
146 149
        or headers['Transfer-Encoding'] != 'chunked':
147 150
            kwargs['headers']['Content-Length'] = len(body) if body else 0
148 151
        if body:
149 152
            kwargs['body'] = body
150
            kwargs['headers']['Content-Type'] = 'application/octet-stream'
151
        #print '****', method, full_path, kwargs
153
        kwargs['headers'].setdefault('Content-Type', 'application/octet-stream')
152 154
        try:
153 155
            conn.request(method, full_path, **kwargs)
154 156
        except socket.error, e:
......
196 198
        status, headers, data = self.get(path, format=format, headers=headers,
197 199
                                         params=params)
198 200
        if detail:
199
            data = json.loads(data)
201
            data = json.loads(data) if data else ''
200 202
        else:
201 203
            data = data.strip().split('\n')
202 204
        return data
......
258 260
    def delete_account_metadata(self, meta=[]):
259 261
        self._delete_metadata('', 'account', meta)
260 262
    
261
    def set_account_groups(self, groups):
263
    def set_account_groups(self, **groups):
262 264
        headers = {}
263 265
        for key, val in groups.items():
264 266
            headers['X-Account-Group-%s' % key.capitalize()] = val
......
283 285
    def _filter_trashed(self, l):
284 286
        return self._filter(l, {'trash':'true'})
285 287
    
286
    def list_objects(self, container, detail=False, params=None, headers=None,
287
                     include_trashed=False):
288
    def list_objects(self, container, detail=False, headers=None,
289
                     include_trashed=False, **params):
288 290
        l = self._list('/' + container, detail, params, headers)
289 291
        if not include_trashed:
290 292
            l = self._filter_trashed(l)
291 293
        return l
292 294
    
293
    def create_container(self, container, headers=None):
295
    def create_container(self, container, headers=None, **meta):
296
        for k,v in meta.items():
297
            headers['X_CONTAINER_META_%s' %k.strip().upper()] = v.strip()
294 298
        status, header, data = self.put('/' + container, headers=headers)
295 299
        if status == 202:
296 300
            return False
......
314 318
        path = '/%s' % (container)
315 319
        self._delete_metadata(path, 'container', meta)
316 320
    
321
    def set_container_policies(self, container, **policies):
322
        path = '/%s' % (container)
323
        headers = {}
324
        print ''
325
        for key, val in policies.items():
326
            headers['X-Container-Policy-%s' % key.capitalize()] = val
327
        self.post(path, headers=headers)
328
    
317 329
    # Storage Object Services
318 330
    
319 331
    def retrieve_object(self, container, object, detail=False, headers=None,
......
331 343
        self.create_object(container, object, f=None, headers=h)
332 344
    
333 345
    def create_object(self, container, object, f=stdin, chunked=False,
334
                      blocksize=1024, headers=None):
346
                      blocksize=1024, headers=None, use_hashes=False, **meta):
335 347
        """
336 348
        creates an object
337 349
        if f is None then creates a zero length object
338 350
        if f is stdin or chunked is set then performs chunked transfer 
339 351
        """
340 352
        path = '/%s/%s' % (container, object)
341
        if not chunked and f != stdin:
353
        for k,v in meta.items():
354
            headers['X_OBJECT_META_%s' %k.strip().upper()] = v.strip()
355
        if not chunked:
356
            format = 'json' if use_hashes else 'text'
342 357
            data = f.read() if f else None
343
            return self.put(path, data, headers=headers)
358
            if data:
359
                if format == 'json':
360
                    data = eval(data)
361
                    data = json.dumps(data)
362
            return self.put(path, data, headers=headers, format=format)
344 363
        else:
345 364
            return self._chunked_transfer(path, 'PUT', f, headers=headers,
346 365
                                   blocksize=1024)
347 366
    
348 367
    def update_object(self, container, object, f=stdin, chunked=False,
349
                      blocksize=1024, headers=None):
368
                      blocksize=1024, headers=None, offset=None, **meta):
350 369
        path = '/%s/%s' % (container, object)
370
        for k,v in meta.items():
371
            headers['X_OBJECT_META_%s' %k.strip().upper()] = v.strip()
372
        if offset:
373
            headers['Content-Range'] = 'bytes %s-/*' % offset
374
        else:
375
            headers['Content-Range'] = 'bytes */*'
376
        
351 377
        if not chunked and f != stdin:
352 378
            data = f.read() if f else None
353 379
            self.post(path, data, headers=headers)
b/tools/store
35 35

  
36 36
from getpass import getuser
37 37
from optparse import OptionParser
38
from os.path import basename
38
from os import environ
39 39
from sys import argv, exit, stdin, stdout
40 40
from pithos.lib.client import Client, Fault
41 41
from datetime import datetime
......
45 45
import types
46 46
import re
47 47
import time as _time
48
import os
48 49

  
49 50
DEFAULT_HOST = 'pithos.dev.grnet.gr'
50 51
DEFAULT_API = 'v1'
......
70 71
        parser.add_option('--host', dest='host', metavar='HOST',
71 72
                          default=DEFAULT_HOST, help='use server HOST')
72 73
        parser.add_option('--user', dest='user', metavar='USERNAME',
73
                          default=getuser(), help='use account USERNAME')
74
                          default=_get_user(),
75
                          help='use account USERNAME')
76
        parser.add_option('--token', dest='token', metavar='AUTH',
77
                          default=_get_auth(),
78
                          help='use account AUTH')
74 79
        parser.add_option('--api', dest='api', metavar='API',
75 80
                          default=DEFAULT_API, help='use api API')
76 81
        parser.add_option('-v', action='store_true', dest='verbose',
......
87 92
                val = getattr(options, key)
88 93
                setattr(self, key, val)
89 94
        
90
        self.client = Client(self.host, self.user, self.api, self.verbose,
95
        self.client = Client(self.host, self.token, self.user, self.api, self.verbose,
91 96
                             self.debug)
92 97
        
93 98
        self.parser = parser
......
154 159
        print_list(l)
155 160
    
156 161
    def list_objects(self, container):
157
        params = {'limit':self.limit, 'marker':self.marker,
158
                  'prefix':self.prefix, 'delimiter':self.delimiter,
159
                  'path':self.path, 'meta':self.meta}
162
        #prepate params
163
        params = {}
164
        attrs = ['limit', 'marker', 'prefix', 'delimiter', 'path', 'meta']
165
        for a in [a for a in attrs if getattr(self, a)]:
166
            params[a] = getattr(self, a)
167
        
168
        if self.until:
169
            t = _time.strptime(self.until, self.format)
170
            params['until'] = int(_time.mktime(t))
171
        
160 172
        headers = {'IF_MODIFIED_SINCE':self.if_modified_since,
161 173
                   'IF_UNMODIFIED_SINCE':self.if_unmodified_since}
162 174
        container, sep, object = container.partition('/')
163 175
        if object:
164 176
            return
165 177
        
166
        if self.until:
167
            t = _time.strptime(self.until, self.format)
168
            params['until'] = int(_time.mktime(t))
169
        
170 178
        detail = 'json'
171 179
        #if request with meta quering disable trash filtering
172 180
        show_trashed = True if self.meta else False
173
        l = self.client.list_objects(container, detail, params, headers,
174
                                     include_trashed = show_trashed)
181
        l = self.client.list_objects(container, detail, headers,
182
                                     include_trashed = show_trashed, **params)
175 183
        print_list(l, detail=self.detail)
176 184

  
177 185
@cli_command('meta')
......
217 225
    
218 226
    def execute(self, container, *args):
219 227
        headers = {}
228
        meta = {}
220 229
        for arg in args:
221 230
            key, sep, val = arg.partition('=')
222
            headers['X_CONTAINER_META_%s' %key.strip().upper()] = val.strip()
223
        ret = self.client.create_container(container, headers)
231
            meta[key] = val
232
        ret = self.client.create_container(container, headers, **meta)
224 233
        if not ret:
225 234
            print 'Container already exists'
226 235

  
......
257 266
        parser.add_option('--if-unmodified-since', action='store', type='str',
258 267
                          dest='if-unmodified-since', default=None,
259 268
                          help='show output if not modified since then')
260
        parser.add_option('-f', action='store', type='str',
269
        parser.add_option('-o', action='store', type='str',
261 270
                          dest='file', default=None,
262 271
                          help='save output in file')
263 272
        parser.add_option('--version', action='store', type='str',
......
309 318
    description = 'create/override object with path contents or standard input'
310 319
    
311 320
    def add_options(self, parser):
321
        parser.add_option('--use_hashes', action='store_true', dest='use_hashes',
322
                          default=False, help='provide hashmap instead of data')
312 323
        parser.add_option('--chunked', action='store_true', dest='chunked',
313 324
                          default=False, help='set chunked transfer mode')
314 325
        parser.add_option('--etag', action='store', dest='etag',
......
328 339
        parser.add_option('--type', action='store',
329 340
                          dest='content-type', default=False,
330 341
                          help='create object with specific content type')
331
        parser.add_option('--touch', action='store_true',
332
                          dest='touch', default=False,
333
                          help='create object with zero data')
342
        #parser.add_option('--touch', action='store_true',
343
        #                  dest='touch', default=False,
344
        #                  help='create object with zero data')
334 345
        parser.add_option('--sharing', action='store',
335 346
                          dest='sharing', default=None,
336 347
                          help='define sharing object policy')
348
        parser.add_option('-f', action='store',
349
                          dest='srcpath', default=None,
350
                          help='file descriptor to read from: pass - for standard input')
351
        
352
    def execute(self, path, *args):
353
        if path.find('=') != -1:
354
            raise Fault('Missing path argument')
355
        
356
        #prepare user defined meta
357
        meta = {}
358
        for arg in args:
359
            key, sep, val = arg.partition('=')
360
            meta[key] = val
337 361
        
338
    def execute(self, path, srcpath='-', *args):
339 362
        headers = {}
340
        if self.manifest:
341
            headers['X_OBJECT_MANIFEST'] = self.manifest
363
        manifest = getattr(self, 'manifest')
364
        if manifest:
365
            # if it's manifestation file
366
            # send zero-byte data with X-Object-Manifest header
367
            self.touch = True
368
            headers['X_OBJECT_MANIFEST'] = manifest
342 369
        if self.sharing:
343 370
            headers['X_OBJECT_SHARING'] = self.sharing
344 371
        
......
348 375
        for a in attrs:
349 376
            headers[a.replace('-', '_').upper()] = getattr(self, a)
350 377
        
351
        #prepare user defined meta
352
        for arg in args:
353
            key, sep, val = arg.partition('=')
354
            headers['X_OBJECT_META_%s' %key.strip().upper()] = val.strip()
355
        
356 378
        container, sep, object = path.partition('/')
357 379
        
358 380
        f = None
359
        chunked = False
360
        if not self.touch:
361
            f = srcpath != '-' and open(srcpath) or stdin
362
            chunked = (self.chunked or f == stdin) and True or False
363
        self.client.create_object(container, object, f, chunked=chunked,
364
                                  headers=headers)
381
        if self.srcpath:
382
            f = open(self.srcpath) if self.srcpath != '-' else stdin
383
        
384
        if self.use_hashes and not f:
385
            raise Fault('Illegal option combination')
386
        
387
        self.client.create_object(container, object, f, chunked=self.chunked,
388
                                  headers=headers, use_hashes=self.use_hashes,
389
                                  **meta)
365 390
        if f:
366 391
            f.close()
367 392

  
......
420 445
    def add_options(self, parser):
421 446
        parser.add_option('-a', action='store_true', dest='append',
422 447
                          default=True, help='append data')
423
        parser.add_option('--start', action='store',
424
                          dest='start',
448
        parser.add_option('--offset', action='store',
449
                          dest='offset',
425 450
                          default=None, help='starting offest to be updated')
426 451
        parser.add_option('--range', action='store', dest='content-range',
427 452
                          default=None, help='range of data to be updated')
......
439 464
        parser.add_option('--sharing', action='store',
440 465
                          dest='sharing', default=None,
441 466
                          help='define sharing object policy')
442
        parser.add_option('--touch', action='store_true',
443
                          dest='touch', default=False,
444
                          help='change object properties')
467
        parser.add_option('--nosharing', action='store_true',
468
                          dest='no_sharing', default=None,
469
                          help='clear object sharing policy')
470
        parser.add_option('-f', action='store',
471
                          dest='srcpath', default=None,
472
                          help='file descriptor to read from: pass - for standard input')
445 473
    
446
    def execute(self, path, srcpath='-', *args):
474
    def execute(self, path, *args):
475
        if path.find('=') != -1:
476
            raise Fault('Missing path argument')
477
        
447 478
        headers = {}
448 479
        if self.manifest:
449 480
            headers['X_OBJECT_MANIFEST'] = self.manifest
450 481
        if self.sharing:
451 482
            headers['X_OBJECT_SHARING'] = self.sharing
452 483
        
453
        if getattr(self, 'start'):
454
            headers['CONTENT_RANGE'] = 'bytes %s-/*' % getattr(self, 'start')
455
        elif self.append:
456
            headers['CONTENT_RANGE'] = 'bytes */*'
484
        if self.no_sharing:
485
            headers['X_OBJECT_SHARING'] = ''
457 486
        
458 487
        attrs = ['content-encoding', 'content-disposition']
459 488
        attrs = [a for a in attrs if getattr(self, a)]
......
461 490
            headers[a.replace('-', '_').upper()] = getattr(self, a)
462 491
        
463 492
        #prepare user defined meta
493
        meta = {}
464 494
        for arg in args:
465 495
            key, sep, val = arg.partition('=')
466
            headers['X_OBJECT_META_%s' %key.strip().upper()] = val.strip()
496
            meta[key] = val
467 497
        
468 498
        container, sep, object = path.partition('/')
469 499
        
470 500
        f = None
471 501
        chunked = False
472
        if not self.touch:
473
            f = srcpath != '-' and open(srcpath) or stdin
474
            chunked = (self.chunked or f == stdin) and True or False
502
        if self.srcpath:
503
            f = self.srcpath != '-' and open(self.srcpath) or stdin
504
        if f:
505
            chunked = True if (self.chunked or f == stdin) else False
475 506
        self.client.update_object(container, object, f, chunked=chunked,
476
                                  headers=headers)
507
                                  headers=headers, offset=self.offset, **meta)
477 508
        if f:
478 509
            f.close()
479 510

  
......
513 544
        self.client.restore_object(src_container, src_object)
514 545

  
515 546
@cli_command('unset')
516
class TrashObject(Command):
547
class UnsetObject(Command):
517 548
    syntax = '<container>/[<object>] key [key] [...]'
518 549
    description = 'deletes metadata info'
519 550
    
......
538 569
@cli_command('group')
539 570
class SetGroup(Command):
540 571
    syntax = 'key=val [key=val] [...]'
541
    description = 'sets group account info'
572
    description = 'set group account info'
542 573
    
543
    def execute(self, path='', **args):
544
        if len(args) == 0:
545
            args = list(args)
546
            args.append(path)
547
            args = tuple(args)
548
            path = ''
574
    def execute(self, *args):
549 575
        groups = {}
550 576
        for arg in args:
551 577
            key, sep, val = arg.partition('=')
552 578
            groups[key] = val
553
        self.client.set_account_groups(groups)
579
        self.client.set_account_groups(**groups)
580

  
581
@cli_command('policy')
582
class SetPolicy(Command):
583
    syntax = 'container key=val [key=val] [...]'
584
    description = 'set contianer policies'
585
    
586
    def execute(self, path, *args):
587
        if path.find('=') != -1:
588
            raise Fault('Missing container argument')
589
        
590
        container, sep, object = path.partition('/')
591
        
592
        if object:
593
            raise Fault('Only containers have policies')
594
        
595
        policies = {}
596
        for arg in args:
597
            key, sep, val = arg.partition('=')
598
            policies[key] = val
599
        
600
        self.client.set_container_policies(container, **policies)
554 601

  
555 602
def print_usage():
556 603
    cmd = Command('', [])
......
602 649
    for id, t in data['versions']:
603 650
        f.write('%s @ %s\n' % (str(id).rjust(30), datetime.fromtimestamp(t)))
604 651

  
652
def _get_user():
653
        try:
654
            return os.environ['PITHOS_USER']
655
        except KeyError:
656
            return getuser()
657

  
658
def _get_auth():
659
        try:
660
            return os.environ['PITHOS_AUTH']
661
        except KeyError:
662
            return '0000'
663
    
664

  
605 665
def main():
606 666
    try:
607 667
        name = argv[1]

Also available in: Unified diff