Statistics
| Branch: | Tag: | Revision:

root / pithos / lib / client.py @ 96738a78

History | View | Annotate | Download (40.8 kB)

1
# Copyright 2011-2012 GRNET S.A. All rights reserved.
2
# 
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
# 
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
# 
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
# 
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
# 
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
from httplib import HTTPConnection, HTTPSConnection, HTTP
35
from sys import stdin
36
from xml.dom import minidom
37
from StringIO import StringIO
38
from urllib import quote, unquote
39
from urlparse import urlparse
40

    
41
import json
42
import types
43
import socket
44
import urllib
45
import datetime
46

    
47
ERROR_CODES = {304:'Not Modified',
48
               400:'Bad Request',
49
               401:'Unauthorized',
50
               403:'Forbidden',
51
               404:'Not Found',
52
               409:'Conflict',
53
               411:'Length Required',
54
               412:'Precondition Failed',
55
               413:'Request Entity Too Large',
56
               416:'Range Not Satisfiable',
57
               422:'Unprocessable Entity',
58
               500:'Internal Server Error',
59
               501:'Not Implemented'}
60

    
61
class Fault(Exception):
62
    def __init__(self, data='', status=None):
63
        if data == '' and status in ERROR_CODES.keys():
64
            data = ERROR_CODES[status]
65
        Exception.__init__(self, data)
66
        self.data = data
67
        self.status = status
68

    
69
class Client(object):
70
    def __init__(self, url, token, account, verbose=False, debug=False):
71
        """`url` can also include a port, e.g '127.0.0.1:8000'."""
72
        
73
        self.url = url
74
        self.account = account
75
        self.verbose = verbose or debug
76
        self.debug = debug
77
        self.token = token
78
    
79
    def _req(self, method, path, body=None, headers={}, format='text', params={}):
80
        p = urlparse(self.url)
81
        if p.scheme == 'http':
82
            conn = HTTPConnection(p.netloc)
83
        elif p.scheme == 'https':
84
            conn = HTTPSConnection(p.netloc)
85
        else:
86
            raise Exception('Unknown URL scheme')
87
        
88
        full_path = _prepare_path(p.path + path, format, params)
89
        
90
        kwargs = {}
91
        kwargs['headers'] = _prepare_headers(headers)
92
        kwargs['headers']['X-Auth-Token'] = self.token
93
        if body:
94
            kwargs['body'] = body
95
            kwargs['headers'].setdefault('content-type', 'application/octet-stream')
96
        kwargs['headers'].setdefault('content-length', len(body) if body else 0)
97
        
98
        #print '#', method, full_path, kwargs
99
        #t1 = datetime.datetime.utcnow()
100
        conn.request(method, full_path, **kwargs)
101
        
102
        resp = conn.getresponse()
103
        #t2 = datetime.datetime.utcnow()
104
        #print 'response time:', str(t2-t1)
105
        return _handle_response(resp, self.verbose, self.debug)
106
    
107
    def _chunked_transfer(self, path, method='PUT', f=stdin, headers=None,
108
                          blocksize=1024, params={}):
109
        """perfomrs a chunked request"""
110
        p = urlparse(self.url)
111
        if p.scheme == 'http':
112
            conn = HTTPConnection(p.netloc)
113
        elif p.scheme == 'https':
114
            conn = HTTPSConnection(p.netloc)
115
        else:
116
            raise Exception('Unknown URL scheme')
117
        
118
        full_path = _prepare_path(p.path + path, params)
119
        
120
        conn.putrequest(method, full_path)
121
        conn.putheader('x-auth-token', self.token)
122
        conn.putheader('content-type', 'application/octet-stream')
123
        conn.putheader('transfer-encoding', 'chunked')
124
        for k,v in _prepare_headers(headers).items():
125
            conn.putheader(k, v)
126
        conn.endheaders()
127
        
128
        # write body
129
        data = ''
130
        while True:
131
            if f.closed:
132
                break
133
            block = f.read(blocksize)
134
            if block == '':
135
                break
136
            data = '%x\r\n%s\r\n' % (len(block), block)
137
            try:
138
                conn.send(data)
139
            except:
140
                #retry
141
                conn.send(data)
142
        data = '0\r\n\r\n'
143
        try:
144
            conn.send(data)
145
        except:
146
            #retry
147
            conn.send(data)
148
        
149
        resp = conn.getresponse()
150
        return _handle_response(resp, self.verbose, self.debug)
151
    
152
    def delete(self, path, format='text', params={}):
153
        return self._req('DELETE', path, format=format, params=params)
154
    
155
    def get(self, path, format='text', headers={}, params={}):
156
        return self._req('GET', path, headers=headers, format=format,
157
                        params=params)
158
    
159
    def head(self, path, format='text', params={}):
160
         return self._req('HEAD', path, format=format, params=params)
161
    
162
    def post(self, path, body=None, format='text', headers=None, params={}):
163
        return self._req('POST', path, body, headers=headers, format=format,
164
                        params=params)
165
    
166
    def put(self, path, body=None, format='text', headers=None, params={}):
167
        return self._req('PUT', path, body, headers=headers, format=format,
168
                         params=params)
169
    
170
    def _list(self, path, format='text', params={}, **headers):
171
        status, headers, data = self.get(path, format=format, headers=headers,
172
                                         params=params)
173
        if format == 'json':
174
            data = json.loads(data) if data else ''
175
        elif format == 'xml':
176
            data = minidom.parseString(data)
177
        else:
178
            data = data.split('\n')[:-1] if data else ''
179
        return data
180
    
181
    def _get_metadata(self, path, prefix=None, params={}):
182
        status, headers, data = self.head(path, params=params)
183
        prefixlen = len(prefix) if prefix else 0
184
        meta = {}
185
        for key, val in headers.items():
186
            if prefix and not key.startswith(prefix):
187
                continue
188
            elif prefix and key.startswith(prefix):
189
                key = key[prefixlen:]
190
            meta[key] = val
191
        return meta
192
    
193
    def _filter(self, l, d):
194
        """
195
        filter out from l elements having the metadata values provided
196
        """
197
        ll = l
198
        for elem in l:
199
            if type(elem) == types.DictionaryType:
200
                for key in d.keys():
201
                    k = 'x_object_meta_%s' % key
202
                    if k in elem.keys() and elem[k] == d[key]:
203
                        ll.remove(elem)
204
                        break
205
        return ll
206
    
207
class OOS_Client(Client):
208
    """Openstack Object Storage Client"""
209
    
210
    def _update_metadata(self, path, entity, **meta):
211
        """adds new and updates the values of previously set metadata"""
212
        ex_meta = self.retrieve_account_metadata(restricted=True)
213
        ex_meta.update(meta)
214
        headers = {}
215
        prefix = 'x-%s-meta-' % entity
216
        for k,v in ex_meta.items():
217
            k = '%s%s' % (prefix, k)
218
            headers[k] = v
219
        return self.post(path, headers=headers)
220
    
221
    def _reset_metadata(self, path, entity, **meta):
222
        """
223
        overwrites all user defined metadata
224
        """
225
        headers = {}
226
        prefix = 'x-%s-meta-' % entity
227
        for k,v in meta.items():
228
            k = '%s%s' % (prefix, k)
229
            headers[k] = v
230
        return self.post(path, headers=headers)
231
    
232
    def _delete_metadata(self, path, entity, meta=[]):
233
        """delete previously set metadata"""
234
        ex_meta = self.retrieve_account_metadata(restricted=True)
235
        headers = {}
236
        prefix = 'x-%s-meta-' % entity
237
        for k in ex_meta.keys():
238
            if k in meta:
239
                headers['%s%s' % (prefix, k)] = ex_meta[k]
240
        return self.post(path, headers=headers)
241
    
242
    # Storage Account Services
243
    
244
    def list_containers(self, format='text', limit=None,
245
                        marker=None, params={}, account=None, **headers):
246
        """lists containers"""
247
        account = account or self.account
248
        path = '/%s' % account
249
        params.update({'limit':limit, 'marker':marker})
250
        return self._list(path, format, params, **headers)
251
    
252
    def retrieve_account_metadata(self, restricted=False, account=None, **params):
253
        """returns the account metadata"""
254
        account = account or self.account
255
        path = '/%s' % account
256
        prefix = 'x-account-meta-' if restricted else None
257
        return self._get_metadata(path, prefix, params)
258
    
259
    def update_account_metadata(self, account=None, **meta):
260
        """updates the account metadata"""
261
        account = account or self.account
262
        path = '/%s' % account
263
        return self._update_metadata(path, 'account', **meta)
264
        
265
    def delete_account_metadata(self, meta=[], account=None):
266
        """deletes the account metadata"""
267
        account = account or self.account
268
        path = '/%s' % account
269
        return self._delete_metadata(path, 'account', meta)
270
    
271
    def reset_account_metadata(self, account=None, **meta):
272
        """resets account metadata"""
273
        account = account or self.account
274
        path = '/%s' % account
275
        return self._reset_metadata(path, 'account', **meta)
276
    
277
    # Storage Container Services
278
    
279
    def _filter_trashed(self, l):
280
        return self._filter(l, {'trash':'true'})
281
    
282
    def list_objects(self, container, format='text',
283
                     limit=None, marker=None, prefix=None, delimiter=None,
284
                     path=None, include_trashed=False, params={}, account=None,
285
                     **headers):
286
        """returns a list with the container objects"""
287
        account = account or self.account
288
        params.update({'limit':limit, 'marker':marker, 'prefix':prefix,
289
                       'delimiter':delimiter, 'path':path})
290
        l = self._list('/%s/%s' % (account, container), format, params,
291
                       **headers)
292
        #TODO support filter trashed with xml also
293
        if format != 'xml' and not include_trashed:
294
            l = self._filter_trashed(l)
295
        return l
296
    
297
    def create_container(self, container, account=None, **meta):
298
        """creates a container"""
299
        account = account or self.account
300
        headers = {}
301
        for k,v in meta.items():
302
            headers['x-container-meta-%s' %k.strip().upper()] = v.strip()
303
        status, header, data = self.put('/%s/%s' % (account, container),
304
                                        headers=headers)
305
        if status == 202:
306
            return False
307
        elif status != 201:
308
            raise Fault(data, int(status))
309
        return True
310
    
311
    def delete_container(self, container, params={}, account=None):
312
        """deletes a container"""
313
        account = account or self.account
314
        return self.delete('/%s/%s' % (account, container), params=params)
315
    
316
    def retrieve_container_metadata(self, container, restricted=False,
317
                                    account=None, **params):
318
        """returns the container metadata"""
319
        account = account or self.account
320
        prefix = 'x-container-meta-' if restricted else None
321
        return self._get_metadata('/%s/%s' % (account, container), prefix,
322
                                  params)
323
    
324
    def update_container_metadata(self, container, account=None, **meta):
325
        """unpdates the container metadata"""
326
        account = account or self.account
327
        return self._update_metadata('/%s/%s' % (account, container),
328
                                     'container', **meta)
329
        
330
    def delete_container_metadata(self, container, meta=[], account=None):
331
        """deletes the container metadata"""
332
        account = account or self.account
333
        path = '/%s/%s' % (account, container)
334
        return self._delete_metadata(path, 'container', meta)
335
    
336
    # Storage Object Services
337
    
338
    def request_object(self, container, object, format='text', params={},
339
                       account=None, **headers):
340
        """returns tuple containing the status, headers and data response for an object request"""
341
        account = account or self.account
342
        path = '/%s/%s/%s' % (account, container, object)
343
        status, headers, data = self.get(path, format, headers, params)
344
        return status, headers, data
345
    
346
    def retrieve_object(self, container, object, format='text', params={},
347
                        account=None, **headers):
348
        """returns an object's data"""
349
        account = account or self.account
350
        t = self.request_object(container, object, format, params, account,
351
                                **headers)
352
        data = t[2]
353
        if format == 'json':
354
            data = json.loads(data) if data else ''
355
        elif format == 'xml':
356
            data = minidom.parseString(data)
357
        return data
358
    
359
    def retrieve_object_hashmap(self, container, object, params={},
360
                        account=None, **headers):
361
        """returns the hashmap representing object's data"""
362
        args = locals().copy()
363
        for elem in ['self', 'container', 'object']:
364
            args.pop(elem)
365
        return self.retrieve_object(container, object, format='json', **args)
366
    
367
    def create_directory_marker(self, container, object, account=None):
368
        """creates a dierectory marker"""
369
        account = account or self.account
370
        if not object:
371
            raise Fault('Directory markers have to be nested in a container')
372
        h = {'content_type':'application/directory'}
373
        return self.create_zero_length_object(container, object, account=account,
374
                                              **h)
375
    
376
    def create_object(self, container, object, f=stdin, format='text', meta={},
377
                      params={}, etag=None, content_type=None, content_encoding=None,
378
                      content_disposition=None, account=None, **headers):
379
        """creates a zero-length object"""
380
        account = account or self.account
381
        path = '/%s/%s/%s' % (account, container, object)
382
        for k, v  in headers.items():
383
            if v == None:
384
                headers.pop(k)
385
        
386
        l = ['etag', 'content_encoding', 'content_disposition', 'content_type']
387
        l = [elem for elem in l if eval(elem)]
388
        for elem in l:
389
            headers.update({elem:eval(elem)})
390
        headers.setdefault('content-type', 'application/octet-stream')
391
        
392
        for k,v in meta.items():
393
            headers['x-object-meta-%s' %k.strip()] = v.strip()
394
        data = f.read() if f else None
395
        return self.put(path, data, format, headers=headers, params=params)
396
    
397
    def create_zero_length_object(self, container, object, meta={}, etag=None,
398
                                  content_type=None, content_encoding=None,
399
                                  content_disposition=None, account=None,
400
                                  **headers):
401
        account = account or self.account
402
        args = locals().copy()
403
        for elem in ['self', 'container', 'headers', 'account']:
404
            args.pop(elem)
405
        args.update(headers)
406
        return self.create_object(container, account=account, f=None, **args)
407
    
408
    def update_object(self, container, object, f=stdin,
409
                      offset=None, meta={}, params={}, content_length=None,
410
                      content_type=None, content_encoding=None,
411
                      content_disposition=None,  account=None, **headers):
412
        account = account or self.account
413
        path = '/%s/%s/%s' % (account, container, object)
414
        for k, v  in headers.items():
415
            if v == None:
416
                headers.pop(k)
417
        
418
        l = ['content_encoding', 'content_disposition', 'content_type',
419
             'content_length']
420
        l = [elem for elem in l if eval(elem)]
421
        for elem in l:
422
            headers.update({elem:eval(elem)})
423
        
424
        if 'content_range' not in headers.keys():
425
            if offset != None:
426
                headers['content_range'] = 'bytes %s-/*' % offset
427
            else:
428
                headers['content_range'] = 'bytes */*'
429
            
430
        for k,v in meta.items():
431
            headers['x-object-meta-%s' %k.strip()] = v.strip()
432
        data = f.read() if f else None
433
        return self.post(path, data, headers=headers, params=params)
434
    
435
    def update_object_using_chunks(self, container, object, f=stdin,
436
                                   blocksize=1024, offset=None, meta={},
437
                                   params={}, content_type=None, content_encoding=None,
438
                                   content_disposition=None, account=None, **headers):
439
        """updates an object (incremental upload)"""
440
        account = account or self.account
441
        path = '/%s/%s/%s' % (account, container, object)
442
        headers = headers if not headers else {}
443
        l = ['content_type', 'content_encoding', 'content_disposition']
444
        l = [elem for elem in l if eval(elem)]
445
        for elem in l:
446
            headers.update({elem:eval(elem)})
447
        
448
        if offset != None:
449
            headers['content_range'] = 'bytes %s-/*' % offset
450
        else:
451
            headers['content_range'] = 'bytes */*'
452
        
453
        for k,v in meta.items():
454
            v = v.strip()
455
            headers['x-object-meta-%s' %k.strip()] = v
456
        return self._chunked_transfer(path, 'POST', f, headers=headers,
457
                                      blocksize=blocksize, params=params)
458
    
459
    def _change_obj_location(self, src_container, src_object, dst_container,
460
                             dst_object, remove=False, meta={}, account=None,
461
                             content_type=None, **headers):
462
        account = account or self.account
463
        path = '/%s/%s/%s' % (account, dst_container, dst_object)
464
        headers = {} if not headers else headers
465
        params = {}
466
        for k, v in meta.items():
467
            headers['x-object-meta-%s' % k] = v
468
        if remove:
469
            headers['x-move-from'] = '/%s/%s' % (src_container, src_object)
470
        else:
471
            headers['x-copy-from'] = '/%s/%s' % (src_container, src_object)
472
        headers['content_length'] = 0
473
        if content_type:
474
            headers['content_type'] = content_type
475
        else:
476
            params['ignore_content_type'] = ''
477
        return self.put(path, headers=headers, params=params)
478
    
479
    def copy_object(self, src_container, src_object, dst_container, dst_object,
480
                   meta={}, account=None, content_type=None, **headers):
481
        """copies an object"""
482
        account = account or self.account
483
        return self._change_obj_location(src_container, src_object,
484
                                   dst_container, dst_object, account=account,
485
                                   remove=False, meta=meta,
486
                                   content_type=content_type, **headers)
487
    
488
    def move_object(self, src_container, src_object, dst_container,
489
                             dst_object, meta={}, account=None,
490
                             content_type=None, **headers):
491
        """moves an object"""
492
        account = account or self.account
493
        return self._change_obj_location(src_container, src_object,
494
                                         dst_container, dst_object,
495
                                         account=account, remove=True,
496
                                         meta=meta, content_type=content_type,
497
                                         **headers)
498
    
499
    def delete_object(self, container, object, params={}, account=None):
500
        """deletes an object"""
501
        account = account or self.account
502
        return self.delete('/%s/%s/%s' % (account, container, object),
503
                           params=params)
504
    
505
    def retrieve_object_metadata(self, container, object, restricted=False,
506
                                 version=None, account=None):
507
        """
508
        set restricted to True to get only user defined metadata
509
        """
510
        account = account or self.account
511
        path = '/%s/%s/%s' % (account, container, object)
512
        prefix = 'x-object-meta-' if restricted else None
513
        params = {'version':version} if version else {}
514
        return self._get_metadata(path, prefix, params=params)
515
    
516
    def update_object_metadata(self, container, object, account=None,
517
                               **meta):
518
        """
519
        updates object's metadata
520
        """
521
        account = account or self.account
522
        path = '/%s/%s/%s' % (account, container, object)
523
        return self._update_metadata(path, 'object', **meta)
524
    
525
    def delete_object_metadata(self, container, object, meta=[], account=None):
526
        """
527
        deletes object's metadata
528
        """
529
        account = account or self.account
530
        path = '/%s/%s' % (account, container, object)
531
        return self._delete_metadata(path, 'object', meta)
532
    
533
class Pithos_Client(OOS_Client):
534
    """Pithos Storage Client. Extends OOS_Client"""
535
    
536
    def _update_metadata(self, path, entity, **meta):
537
        """
538
        adds new and updates the values of previously set metadata
539
        """
540
        params = {'update':None}
541
        headers = {}
542
        prefix = 'x-%s-meta-' % entity
543
        for k,v in meta.items():
544
            k = '%s%s' % (prefix, k)
545
            headers[k] = v
546
        return self.post(path, headers=headers, params=params)
547
    
548
    def _delete_metadata(self, path, entity, meta=[]):
549
        """
550
        delete previously set metadata
551
        """
552
        params = {'update':None}
553
        headers = {}
554
        prefix = 'x-%s-meta-' % entity
555
        for m in meta:
556
            headers['%s%s' % (prefix, m)] = ''
557
        return self.post(path, headers=headers, params=params)
558
    
559
    # Storage Account Services
560
    
561
    def list_containers(self, format='text', if_modified_since=None,
562
                        if_unmodified_since=None, limit=None, marker=None,
563
                        shared=False, until=None, account=None):
564
        """returns a list with the account containers"""
565
        account = account or self.account
566
        params = {'until':until} if until else {}
567
        if shared:
568
            params['shared'] = None
569
        headers = {'if-modified-since':if_modified_since,
570
                   'if-unmodified-since':if_unmodified_since}
571
        return OOS_Client.list_containers(self, account=account, format=format,
572
                                          limit=limit, marker=marker,
573
                                          params=params, **headers)
574
    
575
    def retrieve_account_metadata(self, restricted=False, until=None,
576
                                  account=None):
577
        """returns the account metadata"""
578
        account = account or self.account
579
        params = {'until':until} if until else {}
580
        return OOS_Client.retrieve_account_metadata(self, account=account,
581
                                                    restricted=restricted,
582
                                                    **params)
583
    
584
    def set_account_groups(self, account=None, **groups):
585
        """create account groups"""
586
        account = account or self.account
587
        path = '/%s' % account
588
        headers = {}
589
        for k, v in groups.items():
590
            headers['x-account-group-%s' % k] = v
591
        params = {'update':None}
592
        return self.post(path, headers=headers, params=params)
593
    
594
    def retrieve_account_groups(self, account=None):
595
        """returns the account groups"""
596
        account = account or self.account
597
        meta = self.retrieve_account_metadata(account=account)
598
        prefix = 'x-account-group-'
599
        prefixlen = len(prefix)
600
        groups = {}
601
        for key, val in meta.items():
602
            if prefix and not key.startswith(prefix):
603
                continue
604
            elif prefix and key.startswith(prefix):
605
                key = key[prefixlen:]
606
            groups[key] = val
607
        return groups
608
    
609
    def unset_account_groups(self, groups=[], account=None):
610
        """delete account groups"""
611
        account = account or self.account
612
        path = '/%s' % account
613
        headers = {}
614
        for elem in groups:
615
            headers['x-account-group-%s' % elem] = ''
616
        params = {'update':None}
617
        return self.post(path, headers=headers, params=params)
618
    
619
    def reset_account_groups(self, account=None, **groups):
620
        """overrides account groups"""
621
        account = account or self.account
622
        path = '/%s' % account
623
        headers = {}
624
        for k, v in groups.items():
625
            v = v.strip()
626
            headers['x-account-group-%s' % k] = v
627
        meta = self.retrieve_account_metadata(restricted=True)
628
        prefix = 'x-account-meta-'
629
        for k,v in meta.items():
630
            k = '%s%s' % (prefix, k)
631
            headers[k] = v
632
        return self.post(path, headers=headers)
633
    
634
    # Storage Container Services
635
    
636
    def list_objects(self, container, format='text',
637
                     limit=None, marker=None, prefix=None, delimiter=None,
638
                     path=None, shared=False, include_trashed=False, params={},
639
                     if_modified_since=None, if_unmodified_since=None, meta='',
640
                     until=None, account=None):
641
        """returns a list with the container objects"""
642
        account = account or self.account
643
        params = {'until':until, 'meta':meta}
644
        if shared:
645
            params['shared'] = None
646
        args = locals().copy()
647
        for elem in ['self', 'container', 'params', 'until', 'meta']:
648
            args.pop(elem)
649
        return OOS_Client.list_objects(self, container, params=params, **args)
650
    
651
    def retrieve_container_metadata(self, container, restricted=False,
652
                                    until=None, account=None):
653
        """returns container's metadata"""
654
        account = account or self.account
655
        params = {'until':until} if until else {}
656
        return OOS_Client.retrieve_container_metadata(self, container,
657
                                                      account=account,
658
                                                      restricted=restricted,
659
                                                      **params)
660
    
661
    def set_container_policies(self, container, account=None,
662
                               **policies):
663
        """sets containers policies"""
664
        account = account or self.account
665
        path = '/%s/%s' % (account, container)
666
        headers = {}
667
        for key, val in policies.items():
668
            headers['x-container-policy-%s' % key] = val
669
        return self.post(path, headers=headers)
670
    
671
    def update_container_data(self, container, f=stdin):
672
        """adds blocks of data to the container"""
673
        account = self.account
674
        path = '/%s/%s' % (account, container)
675
        params = {'update': None}
676
        headers = {'content_type': 'application/octet-stream'}
677
        data = f.read() if f else None
678
        headers['content_length'] = len(data)
679
        return self.post(path, data, headers=headers, params=params)
680
    
681
    def delete_container(self, container, until=None, account=None):
682
        """deletes a container or the container history until the date provided"""
683
        account = account or self.account
684
        params = {'until':until} if until else {}
685
        return OOS_Client.delete_container(self, container, account=account,
686
                                           params=params)
687
    
688
    # Storage Object Services
689
    
690
    def retrieve_object(self, container, object, params={}, format='text',
691
                        range=None, if_range=None,
692
                        if_match=None, if_none_match=None,
693
                        if_modified_since=None, if_unmodified_since=None,
694
                        account=None, **headers):
695
        """returns an object"""
696
        account = account or self.account
697
        headers={}
698
        l = ['range', 'if_range', 'if_match', 'if_none_match',
699
             'if_modified_since', 'if_unmodified_since']
700
        l = [elem for elem in l if eval(elem)]
701
        for elem in l:
702
            headers.update({elem:eval(elem)})
703
        if format != 'text':
704
            params['hashmap'] = None
705
        return OOS_Client.retrieve_object(self, container, object,
706
                                          account=account, format=format,
707
                                          params=params, **headers)
708
    
709
    def retrieve_object_version(self, container, object, version,
710
                                format='text', range=None, if_range=None,
711
                                if_match=None, if_none_match=None,
712
                                if_modified_since=None, if_unmodified_since=None,
713
                                account=None):
714
        """returns a specific object version"""
715
        account = account or self.account
716
        args = locals().copy()
717
        l = ['self', 'container', 'object']
718
        for elem in l:
719
            args.pop(elem)
720
        params = {'version':version}
721
        return self.retrieve_object(container, object, params=params, **args)
722
    
723
    def retrieve_object_versionlist(self, container, object, range=None,
724
                                    if_range=None, if_match=None,
725
                                    if_none_match=None, if_modified_since=None,
726
                                    if_unmodified_since=None, account=None):
727
        """returns the object version list"""
728
        account = account or self.account
729
        args = locals().copy()
730
        l = ['self', 'container', 'object']
731
        for elem in l:
732
            args.pop(elem)
733
        
734
        return self.retrieve_object_version(container, object, version='list',
735
                                            format='json', **args)
736
    
737
    def create_zero_length_object(self, container, object,
738
                                  meta={}, etag=None, content_type=None,
739
                                  content_encoding=None,
740
                                  content_disposition=None,
741
                                  x_object_manifest=None, x_object_sharing=None,
742
                                  x_object_public=None, account=None):
743
        """createas a zero length object"""
744
        account = account or self.account
745
        args = locals().copy()
746
        for elem in ['self', 'container', 'object']:
747
            args.pop(elem)
748
        return OOS_Client.create_zero_length_object(self, container, object,
749
                                                    **args)
750
    
751
    def create_object(self, container, object, f=stdin, format='text',
752
                      meta={}, params={}, etag=None, content_type=None,
753
                      content_encoding=None, content_disposition=None,
754
                      x_object_manifest=None, x_object_sharing=None,
755
                      x_object_public=None, account=None):
756
        """creates an object"""
757
        account = account or self.account
758
        args = locals().copy()
759
        for elem in ['self', 'container', 'object']:
760
            args.pop(elem)
761
        if format != 'text':
762
            params.update({'hashmap':None})
763
        return OOS_Client.create_object(self, container, object, **args)
764
        
765
    def create_object_using_chunks(self, container, object,
766
                                   f=stdin, blocksize=1024, meta={}, etag=None,
767
                                   content_type=None, content_encoding=None,
768
                                   content_disposition=None,
769
                                   x_object_sharing=None, x_object_manifest=None,
770
                                   x_object_public=None, account=None):
771
        """creates an object (incremental upload)"""
772
        account = account or self.account
773
        path = '/%s/%s/%s' % (account, container, object)
774
        headers = {}
775
        l = ['etag', 'content_type', 'content_encoding', 'content_disposition', 
776
             'x_object_sharing', 'x_object_manifest', 'x_object_public']
777
        l = [elem for elem in l if eval(elem)]
778
        for elem in l:
779
            headers.update({elem:eval(elem)})
780
        headers.setdefault('content-type', 'application/octet-stream')
781
        
782
        for k,v in meta.items():
783
            v = v.strip()
784
            headers['x-object-meta-%s' %k.strip()] = v
785
        
786
        return self._chunked_transfer(path, 'PUT', f, headers=headers,
787
                                      blocksize=blocksize)
788
    
789
    def create_object_by_hashmap(self, container, object, hashmap={},
790
                                 meta={}, etag=None, content_encoding=None,
791
                                 content_disposition=None, content_type=None,
792
                                 x_object_sharing=None, x_object_manifest=None,
793
                                 x_object_public = None, account=None):
794
        """creates an object by uploading hashes representing data instead of data"""
795
        account = account or self.account
796
        args = locals().copy()
797
        for elem in ['self', 'container', 'object', 'hashmap']:
798
            args.pop(elem)
799
            
800
        try:
801
            data = json.dumps(hashmap)
802
        except SyntaxError:
803
            raise Fault('Invalid formatting')
804
        args['params'] = {'hashmap':None}
805
        args['format'] = 'json'
806
        
807
        return self.create_object(container, object, f=StringIO(data), **args)
808
    
809
    def create_manifestation(self, container, object, manifest, account=None):
810
        """creates a manifestation"""
811
        account = account or self.account
812
        headers={'x_object_manifest':manifest}
813
        return self.create_object(container, object, f=None, account=account,
814
                                  **headers)
815
    
816
    def update_object(self, container, object, f=stdin,
817
                      offset=None, meta={}, replace=False, content_length=None,
818
                      content_type=None, content_range=None,
819
                      content_encoding=None, content_disposition=None,
820
                      x_object_bytes=None, x_object_manifest=None,
821
                      x_object_sharing=None, x_object_public=None,
822
                      x_source_object=None, account=None):
823
        """updates an object"""
824
        account = account or self.account
825
        args = locals().copy()
826
        for elem in ['self', 'container', 'object', 'replace']:
827
            args.pop(elem)
828
        if not replace:
829
            args['params'] = {'update':None}
830
        return OOS_Client.update_object(self, container, object, **args)
831
    
832
    def update_object_using_chunks(self, container, object, f=stdin,
833
                                   blocksize=1024, offset=None, meta={},
834
                                   replace=False, content_type=None, content_encoding=None,
835
                                   content_disposition=None, x_object_bytes=None,
836
                                   x_object_manifest=None, x_object_sharing=None,
837
                                   x_object_public=None, account=None):
838
        """updates an object (incremental upload)"""
839
        account = account or self.account
840
        args = locals().copy()
841
        for elem in ['self', 'container', 'object', 'replace']:
842
            args.pop(elem)
843
        if not replace:
844
            args['params'] = {'update':None}
845
        return OOS_Client.update_object_using_chunks(self, container, object, **args)
846
    
847
    def update_from_other_source(self, container, object, source,
848
                      offset=None, meta={}, content_range=None,
849
                      content_encoding=None, content_disposition=None,
850
                      x_object_bytes=None, x_object_manifest=None,
851
                      x_object_sharing=None, x_object_public=None, account=None):
852
        """updates an object"""
853
        account = account or self.account
854
        args = locals().copy()
855
        for elem in ['self', 'container', 'object', 'source']:
856
            args.pop(elem)
857
        
858
        args['x_source_object'] = source
859
        return self.update_object(container, object, f=None, **args)
860
    
861
    def delete_object(self, container, object, until=None, account=None):
862
        """deletes an object or the object history until the date provided"""
863
        account = account or self.account
864
        params = {'until':until} if until else {}
865
        return OOS_Client.delete_object(self, container, object, params, account)
866
    
867
    def trash_object(self, container, object):
868
        """trashes an object"""
869
        account = account or self.account
870
        path = '/%s/%s' % (container, object)
871
        meta = {'trash':'true'}
872
        return self._update_metadata(path, 'object', **meta)
873
    
874
    def restore_object(self, container, object, account=None):
875
        """restores a trashed object"""
876
        account = account or self.account
877
        return self.delete_object_metadata(container, object, account, ['trash'])
878
    
879
    def publish_object(self, container, object, account=None):
880
        """sets a previously created object publicly accessible"""
881
        account = account or self.account
882
        path = '/%s/%s/%s' % (account, container, object)
883
        headers = {}
884
        headers['x_object_public'] = True
885
        params = {'update':None}
886
        return self.post(path, headers=headers, params=params)
887
    
888
    def unpublish_object(self, container, object, account=None):
889
        """unpublish an object"""
890
        account = account or self.account
891
        path = '/%s/%s/%s' % (account, container, object)
892
        headers = {}
893
        headers['x_object_public'] = False
894
        params = {'update':None}
895
        return self.post(path, headers=headers, params=params)
896
    
897
    def copy_object(self, src_container, src_object, dst_container, dst_object,
898
                    meta={}, public=False, version=None, account=None,
899
                    content_type=None):
900
        """copies an object"""
901
        account = account or self.account
902
        headers = {}
903
        headers['x_object_public'] = public
904
        if version:
905
            headers['x_source_version'] = version
906
        return OOS_Client.copy_object(self, src_container, src_object,
907
                                      dst_container, dst_object, meta=meta,
908
                                      account=account, content_type=content_type,
909
                                      **headers)
910
    
911
    def move_object(self, src_container, src_object, dst_container,
912
                             dst_object, meta={}, public=False,
913
                             account=None, content_type=None):
914
        """moves an object"""
915
        headers = {}
916
        headers['x_object_public'] = public
917
        return OOS_Client.move_object(self, src_container, src_object,
918
                                      dst_container, dst_object, meta=meta,
919
                                      account=account, content_type=content_type,
920
                                      **headers)
921
    
922
    def list_shared_by_others(self, limit=None, marker=None, format='text'):
923
        """lists other accounts that share objects to the user"""
924
        l = ['limit', 'marker']
925
        params = {}
926
        for elem in [elem for elem in l if eval(elem)]:
927
            params[elem] = eval(elem)
928
        return self._list('', format, params)
929
    
930
    def share_object(self, container, object, l, read=True):
931
        """gives access(read by default) to an object to a user/group list"""
932
        action = 'read' if read else 'write'
933
        sharing = '%s=%s' % (action, ','.join(l))
934
        self.update_object(container, object, f=None, x_object_sharing=sharing)
935

    
936
def _prepare_path(path, format='text', params={}):
937
    full_path = '%s?format=%s' % (quote(path), format)
938
    
939
    for k,v in params.items():
940
        value = quote(str(v)) if v else ''
941
        full_path = '%s&%s=%s' %(full_path, quote(k), value)
942
    return full_path
943

    
944
def _prepare_headers(headers):
945
    for k,v in headers.items():
946
        headers.pop(k)
947
        k = k.replace('_', '-')
948
        headers[quote(k)] = quote(v, safe='/=,:@ *"') if type(v) == types.StringType else v
949
    return headers
950

    
951
def _handle_response(response, verbose=False, debug=False):
952
    headers = response.getheaders()
953
    headers = dict((unquote(h), unquote(v)) for h,v in headers)
954
    
955
    if verbose:
956
        print '%d %s' % (response.status, response.reason)
957
        for key, val in headers.items():
958
            print '%s: %s' % (key.capitalize(), val)
959
        print
960
    
961
    length = response.getheader('content-length', None)
962
    data = response.read(length)
963
    if debug:
964
        print data
965
        print
966
    
967
    if int(response.status) in ERROR_CODES.keys():
968
        raise Fault(data, int(response.status))
969
    
970
    #print '**',  response.status, headers, data, '\n'
971
    return response.status, headers, data