Statistics
| Branch: | Tag: | Revision:

root / pithos / lib / client.py @ 961f2fbe

History | View | Annotate | Download (15 kB)

1
# Copyright 2011 GRNET S.A. All rights reserved.
2
# 
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
# 
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
# 
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
# 
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
# 
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
from httplib import HTTPConnection, HTTP
35
from sys import stdin
36

    
37
import json
38
import types
39
import socket
40
import pithos.api.faults
41

    
42
ERROR_CODES = {304:'Not Modified',
43
               400:'Bad Request',
44
               401:'Unauthorized',
45
               404:'Not Found',
46
               409:'Conflict',
47
               411:'Length Required',
48
               412:'Precondition Failed',
49
               416:'Range Not Satisfiable',
50
               422:'Unprocessable Entity',
51
               503:'Service Unavailable'}
52

    
53
class Fault(Exception):
54
    def __init__(self, data='', status=None):
55
        if data == '' and status in ERROR_CODES.keys():
56
            data = ERROR_CODES[status]
57
        Exception.__init__(self, data)
58
        self.data = data
59
        self.status = status
60

    
61
class Client(object):
62
    def __init__(self, host, account, api='v1', verbose=False, debug=False):
63
        """`host` can also include a port, e.g '127.0.0.1:8000'."""
64
        
65
        self.host = host
66
        self.account = account
67
        self.api = api
68
        self.verbose = verbose or debug
69
        self.debug = debug
70
    
71
    def _chunked_transfer(self, path, method='PUT', f=stdin, headers=None,
72
                          blocksize=1024):
73
        http = HTTPConnection(self.host)
74
        
75
        # write header
76
        path = '/%s/%s%s' % (self.api, self.account, path)
77
        http.putrequest(method, path)
78
        http.putheader('Content-Type', 'application/octet-stream')
79
        http.putheader('Transfer-Encoding', 'chunked')
80
        if headers:
81
            for header,value in headers.items():
82
                http.putheader(header, value)
83
        http.endheaders()
84
        
85
        # write body
86
        data = ''
87
        while True:
88
            if f.closed:
89
                break
90
            block = f.read(blocksize)
91
            if block == '':
92
                break
93
            data = '%s\r\n%s\r\n' % (hex(len(block)), block)
94
            try:
95
                http.send(data)
96
            except:
97
                #retry
98
                http.send(data)
99
        data = '0x0\r\n'
100
        try:
101
            http.send(data)
102
        except:
103
            #retry
104
            http.send(data)
105
        
106
        # get response
107
        resp = http.getresponse()
108
        
109
        headers = dict(resp.getheaders())
110
        
111
        if self.verbose:
112
            print '%d %s' % (resp.status, resp.reason)
113
            for key, val in headers.items():
114
                print '%s: %s' % (key.capitalize(), val)
115
            print
116
        
117
        length = resp.getheader('Content-length', None)
118
        data = resp.read(length)
119
        if self.debug:
120
            print data
121
            print
122
        
123
        if int(resp.status) in ERROR_CODES.keys():
124
            raise Fault(data, int(resp.status))
125
        
126
        #print '*',  resp.status, headers, data
127
        return resp.status, headers, data
128
    
129
    def req(self, method, path, body=None, headers=None, format='text',
130
            params=None):
131
        full_path = '/%s/%s%s?format=%s' % (self.api, self.account, path,
132
                                            format)
133
        if params:
134
            for k,v in params.items():
135
                if v:
136
                    full_path = '%s&%s=%s' %(full_path, k, v)
137
        conn = HTTPConnection(self.host)
138
        
139
        #encode whitespace
140
        full_path = full_path.replace(' ', '%20')
141
        
142
        kwargs = {}
143
        kwargs['headers'] = headers or {}
144
        if not headers or \
145
        'Transfer-Encoding' not in headers \
146
        or headers['Transfer-Encoding'] != 'chunked':
147
            kwargs['headers']['Content-Length'] = len(body) if body else 0
148
        if body:
149
            kwargs['body'] = body
150
            kwargs['headers']['Content-Type'] = 'application/octet-stream'
151
        #print '****', method, full_path, kwargs
152
        try:
153
            conn.request(method, full_path, **kwargs)
154
        except socket.error, e:
155
            raise Fault(status=503)
156
            
157
        resp = conn.getresponse()
158
        headers = dict(resp.getheaders())
159
        
160
        if self.verbose:
161
            print '%d %s' % (resp.status, resp.reason)
162
            for key, val in headers.items():
163
                print '%s: %s' % (key.capitalize(), val)
164
            print
165
        
166
        length = resp.getheader('Content-length', None)
167
        data = resp.read(length)
168
        if self.debug:
169
            print data
170
            print
171
        
172
        if int(resp.status) in ERROR_CODES.keys():
173
            raise Fault(data, int(resp.status))
174
        
175
        #print '*',  resp.status, headers, data
176
        return resp.status, headers, data
177
    
178
    def delete(self, path, format='text'):
179
        return self.req('DELETE', path, format=format)
180
    
181
    def get(self, path, format='text', headers=None, params=None):
182
        return self.req('GET', path, headers=headers, format=format,
183
                        params=params)
184
    
185
    def head(self, path, format='text', params=None):
186
        return self.req('HEAD', path, format=format, params=params)
187
    
188
    def post(self, path, body=None, format='text', headers=None):
189
        return self.req('POST', path, body, headers=headers, format=format)
190
    
191
    def put(self, path, body=None, format='text', headers=None):
192
        return self.req('PUT', path, body, headers=headers, format=format)
193
    
194
    def _list(self, path, detail=False, params=None, headers=None):
195
        format = 'json' if detail else 'text'
196
        status, headers, data = self.get(path, format=format, headers=headers,
197
                                         params=params)
198
        if detail:
199
            data = json.loads(data)
200
        else:
201
            data = data.strip().split('\n')
202
        return data
203
    
204
    def _get_metadata(self, path, prefix=None, params=None):
205
        status, headers, data = self.head(path, params=params)
206
        prefixlen = len(prefix) if prefix else 0
207
        meta = {}
208
        for key, val in headers.items():
209
            if prefix and not key.startswith(prefix):
210
                continue
211
            elif prefix and key.startswith(prefix):
212
                key = key[prefixlen:]
213
            meta[key] = val
214
        return meta
215
    
216
    def _update_metadata(self, path, entity, **meta):
217
        """
218
         adds new and updates the values of previously set metadata
219
        """
220
        prefix = 'x-%s-meta-' % entity
221
        prev_meta = self._get_metadata(path, prefix)
222
        prev_meta.update(meta)
223
        headers = {}
224
        for key, val in prev_meta.items():
225
            key = '%s%s' % (prefix, key)
226
            key = '-'.join(elem.capitalize() for elem in key.split('-'))
227
            headers[key] = val
228
        self.post(path, headers=headers)
229
    
230
    def _delete_metadata(self, path, entity, meta=[]):
231
        """
232
        delete previously set metadata
233
        """
234
        prefix = 'x-%s-meta-' % entity
235
        prev_meta = self._get_metadata(path, prefix)
236
        headers = {}
237
        for key, val in prev_meta.items():
238
            if key in meta:
239
                continue
240
            key = '%s%s' % (prefix, key)
241
            key = '-'.join(elem.capitalize() for elem in key.split('-'))
242
            headers[key] = val
243
        self.post(path, headers=headers)
244
    
245
    # Storage Account Services
246
    
247
    def list_containers(self, detail=False, params=None, headers=None):
248
        return self._list('', detail, params, headers)
249
    
250
    def account_metadata(self, restricted=False, until=None):
251
        prefix = 'x-account-meta-' if restricted else None
252
        params = {'until':until} if until else None
253
        return self._get_metadata('', prefix, params=params)
254
    
255
    def update_account_metadata(self, **meta):
256
        self._update_metadata('', 'account', **meta)
257
        
258
    def delete_account_metadata(self, meta=[]):
259
        self._delete_metadata('', 'account', meta)
260
    
261
    # Storage Container Services
262
    
263
    def _filter(self, l, d):
264
        """
265
        filter out from l elements having the metadata values provided
266
        """
267
        ll = l
268
        for elem in l:
269
            if type(elem) == types.DictionaryType:
270
                for key in d.keys():
271
                    k = 'x_object_meta_%s' % key
272
                    if k in elem.keys() and elem[k] == d[key]:
273
                        ll.remove(elem)
274
                        break
275
        return ll
276
    
277
    def _filter_trashed(self, l):
278
        return self._filter(l, {'trash':'true'})
279
    
280
    def list_objects(self, container, detail=False, params=None, headers=None,
281
                     include_trashed=False):
282
        l = self._list('/' + container, detail, params, headers)
283
        if not include_trashed:
284
            l = self._filter_trashed(l)
285
        return l
286
    
287
    def create_container(self, container, headers=None):
288
        status, header, data = self.put('/' + container, headers=headers)
289
        if status == 202:
290
            return False
291
        elif status != 201:
292
            raise Fault(data, int(status))
293
        return True
294
    
295
    def delete_container(self, container):
296
        self.delete('/' + container)
297
    
298
    def retrieve_container_metadata(self, container, restricted=False,
299
                                    until=None):
300
        prefix = 'x-container-meta-' if restricted else None
301
        params = {'until':until} if until else None
302
        return self._get_metadata('/%s' % container, prefix, params=params)
303
    
304
    def update_container_metadata(self, container, **meta):
305
        self._update_metadata('/' + container, 'container', **meta)
306
        
307
    def delete_container_metadata(self, container, meta=[]):
308
        path = '/%s' % (container)
309
        self._delete_metadata(path, 'container', meta)
310
    
311
    # Storage Object Services
312
    
313
    def retrieve_object(self, container, object, detail=False, headers=None,
314
                        version=None):
315
        path = '/%s/%s' % (container, object)
316
        format = 'json' if detail else 'text'
317
        params = {'version':version} if version else None 
318
        status, headers, data = self.get(path, format, headers, params)
319
        return data
320
    
321
    def create_directory_marker(self, container, object):
322
        if not object:
323
            raise Fault('Directory markers have to be nested in a container')
324
        h = {'Content-Type':'application/directory'}
325
        self.create_object(container, object, f=None, headers=h)
326
    
327
    def create_object(self, container, object, f=stdin, chunked=False,
328
                      blocksize=1024, headers=None):
329
        """
330
        creates an object
331
        if f is None then creates a zero length object
332
        if f is stdin or chunked is set then performs chunked transfer 
333
        """
334
        path = '/%s/%s' % (container, object)
335
        if not chunked and f != stdin:
336
            data = f.read() if f else None
337
            return self.put(path, data, headers=headers)
338
        else:
339
            return self._chunked_transfer(path, 'PUT', f, headers=headers,
340
                                   blocksize=1024)
341
    
342
    def update_object(self, container, object, f=stdin, chunked=False,
343
                      blocksize=1024, headers=None):
344
        if not f:
345
            return
346
        path = '/%s/%s' % (container, object)
347
        if not chunked and f != stdin:
348
            data = f.read()
349
            self.post(path, data, headers=headers)
350
        else:
351
            self._chunked_transfer(path, 'POST', f, headers=headers,
352
                                   blocksize=1024)
353
    
354
    def _change_obj_location(self, src_container, src_object, dst_container,
355
                             dst_object, remove=False, headers=None):
356
        path = '/%s/%s' % (dst_container, dst_object)
357
        if not headers:
358
            headers = {}
359
        if remove:
360
            headers['X-Move-From'] = '/%s/%s' % (src_container, src_object)
361
        else:
362
            headers['X-Copy-From'] = '/%s/%s' % (src_container, src_object)
363
        headers['Content-Length'] = 0
364
        self.put(path, headers=headers)
365
    
366
    def copy_object(self, src_container, src_object, dst_container,
367
                             dst_object, headers=None):
368
        self._change_obj_location(src_container, src_object,
369
                                   dst_container, dst_object,
370
                                   headers=headers)
371
    
372
    def move_object(self, src_container, src_object, dst_container,
373
                             dst_object, headers=None):
374
        self._change_obj_location(src_container, src_object,
375
                                   dst_container, dst_object, True, headers)
376
    
377
    def delete_object(self, container, object):
378
        self.delete('/%s/%s' % (container, object))
379
    
380
    def retrieve_object_metadata(self, container, object, restricted=False,
381
                                 version=None):
382
        path = '/%s/%s' % (container, object)
383
        prefix = 'x-object-meta-' if restricted else None
384
        params = {'version':version} if version else None
385
        return self._get_metadata(path, prefix, params=params)
386
    
387
    def update_object_metadata(self, container, object, **meta):
388
        path = '/%s/%s' % (container, object)
389
        self._update_metadata(path, 'object', **meta)
390
    
391
    def delete_object_metadata(self, container, object, meta=[]):
392
        path = '/%s/%s' % (container, object)
393
        self._delete_metadata(path, 'object', meta)
394
    
395
    def trash_object(self, container, object):
396
        """
397
        trashes an object
398
        actually resets all object metadata with trash = true 
399
        """
400
        path = '/%s/%s' % (container, object)
401
        meta = {'trash':'true'}
402
        self._update_metadata(path, 'object', **meta)
403
    
404
    def restore_object(self, container, object):
405
        """
406
        restores a trashed object
407
        actualy removes trash object metadata info
408
        """
409
        self.delete_object_metadata(container, object, ['trash'])
410