Statistics
| Branch: | Tag: | Revision:

root / pithos / lib / client.py @ f89e3cf4

History | View | Annotate | Download (17.6 kB)

1
# Copyright 2011 GRNET S.A. All rights reserved.
2
# 
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
# 
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
# 
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
# 
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
# 
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
from httplib import HTTPConnection, HTTP
35
from sys import stdin
36

    
37
import json
38
import types
39
import socket
40
import pithos.api.faults
41

    
42
ERROR_CODES = {304:'Not Modified',
43
               400:'Bad Request',
44
               401:'Unauthorized',
45
               404:'Not Found',
46
               409:'Conflict',
47
               411:'Length Required',
48
               412:'Precondition Failed',
49
               416:'Range Not Satisfiable',
50
               422:'Unprocessable Entity',
51
               503:'Service Unavailable'}
52

    
53
class Fault(Exception):
54
    def __init__(self, data='', status=None):
55
        if data == '' and status in ERROR_CODES.keys():
56
            data = ERROR_CODES[status]
57
        Exception.__init__(self, data)
58
        self.data = data
59
        self.status = status
60

    
61
class Client(object):
62
    def __init__(self, host, token, account, api='v1', verbose=False, debug=False):
63
        """`host` can also include a port, e.g '127.0.0.1:8000'."""
64
        
65
        self.host = host
66
        self.account = account
67
        self.api = api
68
        self.verbose = verbose or debug
69
        self.debug = debug
70
        self.token = token
71
    
72
    def _chunked_transfer(self, path, method='PUT', f=stdin, headers=None,
73
                          blocksize=1024):
74
        http = HTTPConnection(self.host)
75
        
76
        # write header
77
        path = '/%s/%s%s' % (self.api, self.account, path)
78
        http.putrequest(method, path)
79
        http.putheader('X-Auth-Token', self.token)
80
        http.putheader('Content-Type', 'application/octet-stream')
81
        http.putheader('Transfer-Encoding', 'chunked')
82
        if headers:
83
            for header,value in headers.items():
84
                http.putheader(header, value)
85
        http.endheaders()
86
        
87
        # write body
88
        data = ''
89
        while True:
90
            if f.closed:
91
                break
92
            block = f.read(blocksize)
93
            if block == '':
94
                break
95
            data = '%s\r\n%s\r\n' % (hex(len(block)), block)
96
            try:
97
                http.send(data)
98
            except:
99
                #retry
100
                http.send(data)
101
        data = '0x0\r\n'
102
        try:
103
            http.send(data)
104
        except:
105
            #retry
106
            http.send(data)
107
        
108
        # get response
109
        resp = http.getresponse()
110
        
111
        headers = dict(resp.getheaders())
112
        
113
        if self.verbose:
114
            print '%d %s' % (resp.status, resp.reason)
115
            for key, val in headers.items():
116
                print '%s: %s' % (key.capitalize(), val)
117
            print
118
        
119
        length = resp.getheader('Content-length', None)
120
        data = resp.read(length)
121
        if self.debug:
122
            print data
123
            print
124
        
125
        if int(resp.status) in ERROR_CODES.keys():
126
            raise Fault(data, int(resp.status))
127
        
128
        #print '*',  resp.status, headers, data
129
        return resp.status, headers, data
130
    
131
    def req(self, method, path, body=None, headers=None, format='text',
132
            params=None):
133
        full_path = '/%s/%s%s?format=%s' % (self.api, self.account, path,
134
                                            format)
135
        if params:
136
            for k,v in params.items():
137
                if v:
138
                    full_path = '%s&%s=%s' %(full_path, k, v)
139
        conn = HTTPConnection(self.host)
140
        
141
        #encode whitespace
142
        full_path = full_path.replace(' ', '%20')
143
        
144
        kwargs = {}
145
        kwargs['headers'] = headers or {}
146
        kwargs['headers']['X-Auth-Token'] = self.token
147
        if not headers or \
148
        'Transfer-Encoding' not in headers \
149
        or headers['Transfer-Encoding'] != 'chunked':
150
            kwargs['headers']['Content-Length'] = len(body) if body else 0
151
        if body:
152
            kwargs['body'] = body
153
        kwargs['headers'].setdefault('Content-Type', 'application/octet-stream')
154
        try:
155
            conn.request(method, full_path, **kwargs)
156
        except socket.error, e:
157
            raise Fault(status=503)
158
            
159
        resp = conn.getresponse()
160
        headers = dict(resp.getheaders())
161
        
162
        if self.verbose:
163
            print '%d %s' % (resp.status, resp.reason)
164
            for key, val in headers.items():
165
                print '%s: %s' % (key.capitalize(), val)
166
            print
167
        
168
        length = resp.getheader('Content-length', None)
169
        data = resp.read(length)
170
        if self.debug:
171
            print data
172
            print
173
        
174
        if int(resp.status) in ERROR_CODES.keys():
175
            raise Fault(data, int(resp.status))
176
        
177
        #print '*',  resp.status, headers, data
178
        return resp.status, headers, data
179
    
180
    def delete(self, path, format='text'):
181
        return self.req('DELETE', path, format=format)
182
    
183
    def get(self, path, format='text', headers=None, params=None):
184
        return self.req('GET', path, headers=headers, format=format,
185
                        params=params)
186
    
187
    def head(self, path, format='text', params=None):
188
        return self.req('HEAD', path, format=format, params=params)
189
    
190
    def post(self, path, body=None, format='text', headers=None):
191
        return self.req('POST', path, body, headers=headers, format=format)
192
    
193
    def put(self, path, body=None, format='text', headers=None):
194
        return self.req('PUT', path, body, headers=headers, format=format)
195
    
196
    def _list(self, path, detail=False, params=None, headers=None):
197
        format = 'json' if detail else 'text'
198
        status, headers, data = self.get(path, format=format, headers=headers,
199
                                         params=params)
200
        if detail:
201
            data = json.loads(data) if data else ''
202
        else:
203
            data = data.strip().split('\n')
204
        return data
205
    
206
    def _get_metadata(self, path, prefix=None, params=None):
207
        status, headers, data = self.head(path, params=params)
208
        prefixlen = len(prefix) if prefix else 0
209
        meta = {}
210
        for key, val in headers.items():
211
            if prefix and not key.startswith(prefix):
212
                continue
213
            elif prefix and key.startswith(prefix):
214
                key = key[prefixlen:]
215
            meta[key] = val
216
        return meta
217
    
218
    def _update_metadata(self, path, entity, **meta):
219
        """
220
         adds new and updates the values of previously set metadata
221
        """
222
        prefix = 'x-%s-meta-' % entity
223
        prev_meta = self._get_metadata(path, prefix)
224
        prev_meta.update(meta)
225
        headers = {}
226
        for key, val in prev_meta.items():
227
            key = '%s%s' % (prefix, key)
228
            key = '-'.join(elem.capitalize() for elem in key.split('-'))
229
            headers[key] = val
230
        self.post(path, headers=headers)
231
    
232
    def _delete_metadata(self, path, entity, meta=[]):
233
        """
234
        delete previously set metadata
235
        """
236
        prefix = 'x-%s-meta-' % entity
237
        prev_meta = self._get_metadata(path, prefix)
238
        headers = {}
239
        for key, val in prev_meta.items():
240
            if key in meta:
241
                continue
242
            key = '%s%s' % (prefix, key)
243
            key = '-'.join(elem.capitalize() for elem in key.split('-'))
244
            headers[key] = val
245
        self.post(path, headers=headers)
246
    
247
    # Storage Account Services
248
    
249
    def list_containers(self, detail=False, params=None, headers=None):
250
        return self._list('', detail, params, headers)
251
    
252
    def account_metadata(self, restricted=False, until=None):
253
        prefix = 'x-account-meta-' if restricted else None
254
        params = {'until':until} if until else None
255
        return self._get_metadata('', prefix, params=params)
256
    
257
    def update_account_metadata(self, **meta):
258
        self._update_metadata('', 'account', **meta)
259
        
260
    def delete_account_metadata(self, meta=[]):
261
        self._delete_metadata('', 'account', meta)
262
    
263
    def set_account_groups(self, **groups):
264
        headers = {}
265
        for key, val in groups.items():
266
            headers['X-Account-Group-%s' % key.capitalize()] = val
267
        self.post('', headers=headers)
268
    
269
    # Storage Container Services
270
    
271
    def _filter(self, l, d):
272
        """
273
        filter out from l elements having the metadata values provided
274
        """
275
        ll = l
276
        for elem in l:
277
            if type(elem) == types.DictionaryType:
278
                for key in d.keys():
279
                    k = 'x_object_meta_%s' % key
280
                    if k in elem.keys() and elem[k] == d[key]:
281
                        ll.remove(elem)
282
                        break
283
        return ll
284
    
285
    def _filter_trashed(self, l):
286
        return self._filter(l, {'trash':'true'})
287
    
288
    def list_objects(self, container, detail=False, headers=None,
289
                     include_trashed=False, **params):
290
        l = self._list('/' + container, detail, params, headers)
291
        if not include_trashed:
292
            l = self._filter_trashed(l)
293
        return l
294
    
295
    def create_container(self, container, headers=None, **meta):
296
        for k,v in meta.items():
297
            headers['X-Container-Meta-%s' %k.strip().upper()] = v.strip()
298
        status, header, data = self.put('/' + container, headers=headers)
299
        if status == 202:
300
            return False
301
        elif status != 201:
302
            raise Fault(data, int(status))
303
        return True
304
    
305
    def delete_container(self, container):
306
        self.delete('/' + container)
307
    
308
    def retrieve_container_metadata(self, container, restricted=False,
309
                                    until=None):
310
        prefix = 'x-container-meta-' if restricted else None
311
        params = {'until':until} if until else None
312
        return self._get_metadata('/%s' % container, prefix, params=params)
313
    
314
    def update_container_metadata(self, container, **meta):
315
        self._update_metadata('/' + container, 'container', **meta)
316
        
317
    def delete_container_metadata(self, container, meta=[]):
318
        path = '/%s' % (container)
319
        self._delete_metadata(path, 'container', meta)
320
    
321
    def set_container_policies(self, container, **policies):
322
        path = '/%s' % (container)
323
        headers = {}
324
        print ''
325
        for key, val in policies.items():
326
            headers['X-Container-Policy-%s' % key.capitalize()] = val
327
        self.post(path, headers=headers)
328
    
329
    # Storage Object Services
330
    
331
    def retrieve_object(self, container, object, detail=False, headers=None,
332
                        version=None):
333
        path = '/%s/%s' % (container, object)
334
        format = 'json' if detail else 'text'
335
        params = {'version':version} if version else None 
336
        status, headers, data = self.get(path, format, headers, params)
337
        return data
338
    
339
    def create_directory_marker(self, container, object):
340
        if not object:
341
            raise Fault('Directory markers have to be nested in a container')
342
        h = {'Content-Type':'application/directory'}
343
        self.create_object(container, object, f=None, headers=h)
344
    
345
    def _set_public(self, headers, public=False):
346
        """
347
        sets the public header
348
        """
349
        if public == None:
350
            return
351
        elif public:
352
            headers['X-Object-Public'] = public
353
        else:
354
            headers['X-Object-Public'] = ''
355
    
356
    def create_object(self, container, object, f=stdin, chunked=False,
357
                      blocksize=1024, headers={}, use_hashes=False,
358
                      public=None, **meta):
359
        """
360
        creates an object
361
        if f is None then creates a zero length object
362
        if f is stdin or chunked is set then performs chunked transfer 
363
        """
364
        path = '/%s/%s' % (container, object)
365
        for k,v in meta.items():
366
            headers['X-Object-Meta-%s' %k.strip().upper()] = v.strip()
367
        self._set_public(headers, public)
368
        headers = headers if headers else None
369
        if not chunked:
370
            format = 'json' if use_hashes else 'text'
371
            data = f.read() if f else None
372
            if data:
373
                if format == 'json':
374
                    data = eval(data)
375
                    data = json.dumps(data)
376
            return self.put(path, data, headers=headers, format=format)
377
        else:
378
            return self._chunked_transfer(path, 'PUT', f, headers=headers,
379
                                   blocksize=1024)
380
    
381
    def update_object(self, container, object, f=stdin, chunked=False,
382
                      blocksize=1024, headers={}, offset=None, public=None,
383
                      **meta):
384
        print locals()
385
        path = '/%s/%s' % (container, object)
386
        for k,v in meta.items():
387
            headers['X-Object-Meta-%s' %k.strip().upper()] = v.strip()
388
        if offset:
389
            headers['Content-Range'] = 'bytes %s-/*' % offset
390
        else:
391
            headers['Content-Range'] = 'bytes */*'
392
        self._set_public(headers, public)
393
        headers = headers if headers else None
394
        if not chunked and f != stdin:
395
            data = f.read() if f else None
396
            self.post(path, data, headers=headers)
397
        else:
398
            self._chunked_transfer(path, 'POST', f, headers=headers,
399
                                   blocksize=1024)
400
    
401
    def _change_obj_location(self, src_container, src_object, dst_container,
402
                             dst_object, remove=False, public=None, headers={}):
403
        path = '/%s/%s' % (dst_container, dst_object)
404
        if not headers:
405
            headers = {}
406
        if remove:
407
            headers['X-Move-From'] = '/%s/%s' % (src_container, src_object)
408
        else:
409
            headers['X-Copy-From'] = '/%s/%s' % (src_container, src_object)
410
        self._set_public(headers, public)
411
        self.headers = headers if headers else None
412
        headers['Content-Length'] = 0
413
        self.put(path, headers=headers)
414
    
415
    def copy_object(self, src_container, src_object, dst_container,
416
                             dst_object, public=False, headers=None):
417
        self._change_obj_location(src_container, src_object,
418
                                   dst_container, dst_object,
419
                                   public, headers)
420
    
421
    def move_object(self, src_container, src_object, dst_container,
422
                             dst_object, headers=None):
423
        self._change_obj_location(src_container, src_object,
424
                                   dst_container, dst_object, True,
425
                                   public, headers)
426
    
427
    def delete_object(self, container, object):
428
        self.delete('/%s/%s' % (container, object))
429
    
430
    def retrieve_object_metadata(self, container, object, restricted=False,
431
                                 version=None):
432
        path = '/%s/%s' % (container, object)
433
        prefix = 'x-object-meta-' if restricted else None
434
        params = {'version':version} if version else None
435
        return self._get_metadata(path, prefix, params=params)
436
    
437
    def update_object_metadata(self, container, object, **meta):
438
        path = '/%s/%s' % (container, object)
439
        self._update_metadata(path, 'object', **meta)
440
    
441
    def delete_object_metadata(self, container, object, meta=[]):
442
        path = '/%s/%s' % (container, object)
443
        self._delete_metadata(path, 'object', meta)
444
    
445
    def trash_object(self, container, object):
446
        """
447
        trashes an object
448
        actually resets all object metadata with trash = true 
449
        """
450
        path = '/%s/%s' % (container, object)
451
        meta = {'trash':'true'}
452
        self._update_metadata(path, 'object', **meta)
453
    
454
    def restore_object(self, container, object):
455
        """
456
        restores a trashed object
457
        actualy removes trash object metadata info
458
        """
459
        self.delete_object_metadata(container, object, ['trash'])
460
    
461
    def publish_object(self, container, object):
462
        """
463
        sets a previously created object publicly accessible
464
        """
465
        path = '/%s/%s' % (container, object)
466
        headers = {}
467
        headers['Content-Range'] = 'bytes */*'
468
        self._set_public(headers, public=True)
469
        self.post(path, headers=headers)
470
    
471
    def unpublish_object(self, container, object):
472
        """
473
        unpublish an object
474
        """
475
        path = '/%s/%s' % (container, object)
476
        headers = {}
477
        headers['Content-Range'] = 'bytes */*'
478
        self._set_public(headers, public=False)
479
        self.post(path, headers=headers)