Statistics
| Branch: | Tag: | Revision:

root / pithos / lib / client.py @ a4c10cbc

History | View | Annotate | Download (14.9 kB)

1
# Copyright 2011 GRNET S.A. All rights reserved.
2
# 
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
# 
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
# 
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
# 
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
# 
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
from httplib import HTTPConnection, HTTP
35
from sys import stdin
36

    
37
import json
38
import types
39
import socket
40
import pithos.api.faults
41

    
42
ERROR_CODES = {304:'Not Modified',
43
               400:'Bad Request',
44
               401:'Unauthorized',
45
               404:'Not Found',
46
               409:'Conflict',
47
               411:'Length Required',
48
               412:'Precondition Failed',
49
               416:'Range Not Satisfiable',
50
               422:'Unprocessable Entity',
51
               503:'Service Unavailable'}
52

    
53
class Fault(Exception):
54
    def __init__(self, data='', status=None):
55
        if data == '' and status in ERROR_CODES.keys():
56
            data = ERROR_CODES[status]
57
        Exception.__init__(self, data)
58
        self.data = data
59
        self.status = status
60

    
61
class Client(object):
62
    def __init__(self, host, account, api='v1', verbose=False, debug=False):
63
        """`host` can also include a port, e.g '127.0.0.1:8000'."""
64
        
65
        self.host = host
66
        self.account = account
67
        self.api = api
68
        self.verbose = verbose or debug
69
        self.debug = debug
70
    
71
    def _chunked_transfer(self, path, method='PUT', f=stdin, headers=None,
72
                          blocksize=1024):
73
        http = HTTPConnection(self.host)
74
        
75
        # write header
76
        path = '/%s/%s%s' % (self.api, self.account, path)
77
        http.putrequest(method, path)
78
        http.putheader('Content-Type', 'application/octet-stream')
79
        http.putheader('Transfer-Encoding', 'chunked')
80
        if headers:
81
            for header,value in headers.items():
82
                http.putheader(header, value)
83
        http.endheaders()
84
        
85
        # write body
86
        data = ''
87
        while True:
88
            if f.closed:
89
                break
90
            block = f.read(blocksize)
91
            if block == '':
92
                break
93
            data = '%s\r\n%s\r\n' % (hex(len(block)), block)
94
            try:
95
                http.send(data)
96
            except:
97
                #retry
98
                http.send(data)
99
        data = '0x0\r\n'
100
        try:
101
            http.send(data)
102
        except:
103
            #retry
104
            http.send(data)
105
        
106
        # get response
107
        resp = http.getresponse()
108
        
109
        headers = dict(resp.getheaders())
110
        
111
        if self.verbose:
112
            print '%d %s' % (resp.status, resp.reason)
113
            for key, val in headers.items():
114
                print '%s: %s' % (key.capitalize(), val)
115
            print
116
        
117
        length = resp.getheader('Content-length', None)
118
        data = resp.read(length)
119
        if self.debug:
120
            print data
121
            print
122
        
123
        if int(resp.status) in ERROR_CODES.keys():
124
            raise Fault(data, int(resp.status))
125
        
126
        #print '*',  resp.status, headers, data
127
        return resp.status, headers, data
128
    
129
    def req(self, method, path, body=None, headers=None, format='text',
130
            params=None):
131
        full_path = '/%s/%s%s?format=%s' % (self.api, self.account, path,
132
                                            format)
133
        if params:
134
            for k,v in params.items():
135
                if v:
136
                    full_path = '%s&%s=%s' %(full_path, k, v)
137
        conn = HTTPConnection(self.host)
138
        
139
        #encode whitespace
140
        full_path = full_path.replace(' ', '%20')
141
        
142
        kwargs = {}
143
        kwargs['headers'] = headers or {}
144
        if not headers or \
145
        'Transfer-Encoding' not in headers \
146
        or headers['Transfer-Encoding'] != 'chunked':
147
            kwargs['headers']['Content-Length'] = len(body) if body else 0
148
        if body:
149
            kwargs['body'] = body
150
            kwargs['headers']['Content-Type'] = 'application/octet-stream'
151
        #print '****', method, full_path, kwargs
152
        try:
153
            conn.request(method, full_path, **kwargs)
154
        except socket.error, e:
155
            raise Fault(status=503)
156
            
157
        resp = conn.getresponse()
158
        headers = dict(resp.getheaders())
159
        
160
        if self.verbose:
161
            print '%d %s' % (resp.status, resp.reason)
162
            for key, val in headers.items():
163
                print '%s: %s' % (key.capitalize(), val)
164
            print
165
        
166
        length = resp.getheader('Content-length', None)
167
        data = resp.read(length)
168
        if self.debug:
169
            print data
170
            print
171
        
172
        if int(resp.status) in ERROR_CODES.keys():
173
            raise Fault(data, int(resp.status))
174
        
175
        #print '*',  resp.status, headers, data
176
        return resp.status, headers, data
177
    
178
    def delete(self, path, format='text'):
179
        return self.req('DELETE', path, format=format)
180
    
181
    def get(self, path, format='text', headers=None, params=None):
182
        return self.req('GET', path, headers=headers, format=format,
183
                        params=params)
184
    
185
    def head(self, path, format='text', params=None):
186
        return self.req('HEAD', path, format=format, params=params)
187
    
188
    def post(self, path, body=None, format='text', headers=None):
189
        return self.req('POST', path, body, headers=headers, format=format)
190
    
191
    def put(self, path, body=None, format='text', headers=None):
192
        return self.req('PUT', path, body, headers=headers, format=format)
193
    
194
    def _list(self, path, detail=False, params=None, headers=None):
195
        format = 'json' if detail else 'text'
196
        status, headers, data = self.get(path, format=format, headers=headers,
197
                                         params=params)
198
        if detail:
199
            data = json.loads(data)
200
        else:
201
            data = data.strip().split('\n')
202
        return data
203
    
204
    def _get_metadata(self, path, prefix=None, params=None):
205
        status, headers, data = self.head(path, params=params)
206
        prefixlen = len(prefix) if prefix else 0
207
        meta = {}
208
        for key, val in headers.items():
209
            if prefix and not key.startswith(prefix):
210
                continue
211
            elif prefix and key.startswith(prefix):
212
                key = key[prefixlen:]
213
            meta[key] = val
214
        return meta
215
    
216
    def _update_metadata(self, path, entity, **meta):
217
        """
218
         adds new and updates the values of previously set metadata
219
        """
220
        for key, val in meta.items():
221
            meta.pop(key)
222
            meta['X-%s-Meta-%s' %(entity.capitalize(), key.capitalize())] = val
223
        prev_meta = self._get_metadata(path)
224
        prev_meta.update(meta)
225
        headers = {}
226
        for key, val in prev_meta.items():
227
            headers[key.capitalize()] = val
228
        self.post(path, headers=headers)
229
    
230
    def _delete_metadata(self, path, entity, meta=[]):
231
        """
232
        delete previously set metadata
233
        """
234
        prev_meta = self._get_metadata(path)
235
        headers = {}
236
        for key, val in prev_meta.items():
237
            if key.split('-')[-1] in meta:
238
                continue
239
            http_key = key.capitalize()
240
            headers[http_key] = val
241
        self.post(path, headers=headers)
242
    
243
    # Storage Account Services
244
    
245
    def list_containers(self, detail=False, params=None, headers=None):
246
        return self._list('', detail, params, headers)
247
    
248
    def account_metadata(self, restricted=False, until=None):
249
        prefix = 'x-account-meta-' if restricted else None
250
        params = {'until':until} if until else None
251
        return self._get_metadata('', prefix, params=params)
252
    
253
    def update_account_metadata(self, **meta):
254
        self._update_metadata('', 'account', **meta)
255
        
256
    def delete_account_metadata(self, meta=[]):
257
        self._delete_metadata('', 'account', meta)
258
    
259
    # Storage Container Services
260
    
261
    def _filter(self, l, d):
262
        """
263
        filter out from l elements having the metadata values provided
264
        """
265
        ll = l
266
        for elem in l:
267
            if type(elem) == types.DictionaryType:
268
                for key in d.keys():
269
                    k = 'x_object_meta_%s' % key
270
                    if k in elem.keys() and elem[k] == d[key]:
271
                        ll.remove(elem)
272
                        break
273
        return ll
274
    
275
    def _filter_trashed(self, l):
276
        return self._filter(l, {'trash':'true'})
277
    
278
    def list_objects(self, container, detail=False, params=None, headers=None,
279
                     include_trashed=False):
280
        l = self._list('/' + container, detail, params, headers)
281
        if not include_trashed:
282
            l = self._filter_trashed(l)
283
        return l
284
    
285
    def create_container(self, container, headers=None):
286
        status, header, data = self.put('/' + container, headers=headers)
287
        if status == 202:
288
            return False
289
        elif status != 201:
290
            raise Fault(data, int(status))
291
        return True
292
    
293
    def delete_container(self, container):
294
        self.delete('/' + container)
295
    
296
    def retrieve_container_metadata(self, container, restricted=False,
297
                                    until=None):
298
        prefix = 'x-container-meta-' if restricted else None
299
        params = {'until':until} if until else None
300
        return self._get_metadata('/%s' % container, prefix, params=params)
301
    
302
    def update_container_metadata(self, container, **meta):
303
        self._update_metadata('/' + container, 'container', **meta)
304
        
305
    def delete_container_metadata(self, container, meta=[]):
306
        path = '/%s' % (container)
307
        self._delete_metadata(path, 'container', meta)
308
    
309
    # Storage Object Services
310
    
311
    def retrieve_object(self, container, object, detail=False, headers=None,
312
                        version=None):
313
        path = '/%s/%s' % (container, object)
314
        format = 'json' if detail else 'text'
315
        params = {'version':version} if version else None 
316
        status, headers, data = self.get(path, format, headers, params)
317
        return data
318
    
319
    def create_directory_marker(self, container, object):
320
        if not object:
321
            raise Fault('Directory markers have to be nested in a container')
322
        h = {'Content-Type':'application/directory'}
323
        self.create_object(container, object, f=None, headers=h)
324
    
325
    def create_object(self, container, object, f=stdin, chunked=False,
326
                      blocksize=1024, headers=None):
327
        """
328
        creates an object
329
        if f is None then creates a zero length object
330
        if f is stdin or chunked is set then performs chunked transfer 
331
        """
332
        path = '/%s/%s' % (container, object)
333
        if not chunked and f != stdin:
334
            data = f.read() if f else None
335
            return self.put(path, data, headers=headers)
336
        else:
337
            return self._chunked_transfer(path, 'PUT', f, headers=headers,
338
                                   blocksize=1024)
339
    
340
    def update_object(self, container, object, f=stdin, chunked=False,
341
                      blocksize=1024, headers=None):
342
        if not f:
343
            return
344
        path = '/%s/%s' % (container, object)
345
        if not chunked and f != stdin:
346
            data = f.read()
347
            self.post(path, data, headers=headers)
348
        else:
349
            self._chunked_transfer(path, 'POST', f, headers=headers,
350
                                   blocksize=1024)
351
    
352
    def _change_obj_location(self, src_container, src_object, dst_container,
353
                             dst_object, remove=False, headers=None):
354
        path = '/%s/%s' % (dst_container, dst_object)
355
        if not headers:
356
            headers = {}
357
        if remove:
358
            headers['X-Move-From'] = '/%s/%s' % (src_container, src_object)
359
        else:
360
            headers['X-Copy-From'] = '/%s/%s' % (src_container, src_object)
361
        headers['Content-Length'] = 0
362
        self.put(path, headers=headers)
363
    
364
    def copy_object(self, src_container, src_object, dst_container,
365
                             dst_object, headers=None):
366
        self._change_obj_location(src_container, src_object,
367
                                   dst_container, dst_object,
368
                                   headers=headers)
369
    
370
    def move_object(self, src_container, src_object, dst_container,
371
                             dst_object, headers=None):
372
        self._change_obj_location(src_container, src_object,
373
                                   dst_container, dst_object, True, headers)
374
    
375
    def delete_object(self, container, object):
376
        self.delete('/%s/%s' % (container, object))
377
    
378
    def retrieve_object_metadata(self, container, object, restricted=False,
379
                                 version=None):
380
        path = '/%s/%s' % (container, object)
381
        prefix = 'x-object-meta-' if restricted else None
382
        params = {'version':version} if version else None
383
        return self._get_metadata(path, prefix, params=params)
384
    
385
    def update_object_metadata(self, container, object, **meta):
386
        path = '/%s/%s' % (container, object)
387
        self._update_metadata(path, 'object', **meta)
388
    
389
    def delete_object_metadata(self, container, object, meta=[]):
390
        path = '/%s/%s' % (container, object)
391
        self._delete_metadata(path, 'object', meta)
392
    
393
    def trash_object(self, container, object):
394
        """
395
        trashes an object
396
        actually resets all object metadata with trash = true 
397
        """
398
        path = '/%s/%s' % (container, object)
399
        meta = {'trash':'true'}
400
        self._update_metadata(path, 'object', **meta)
401
    
402
    def restore_object(self, container, object):
403
        """
404
        restores a trashed object
405
        actualy just resets all object metadata except trash
406
        """
407
        self.delete_object_metadata(container, object, ['trash'])
408