Statistics
| Branch: | Tag: | Revision:

root / snf-cyclades-app / synnefo / plankton / backend.py @ 7784ab88

History | View | Annotate | Download (16.4 kB)

1
# Copyright 2011 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
"""
35
The Plankton attributes are the following:
36
  - checksum: the 'hash' meta
37
  - container_format: stored as a user meta
38
  - created_at: the 'modified' meta of the first version
39
  - deleted_at: the timestamp of the last version
40
  - disk_format: stored as a user meta
41
  - id: the 'uuid' meta
42
  - is_public: True if there is a * entry for the read permission
43
  - location: generated based on the file's path
44
  - name: stored as a user meta
45
  - owner: the file's account
46
  - properties: stored as user meta prefixed with PROPERTY_PREFIX
47
  - size: the 'bytes' meta
48
  - status: stored as a system meta
49
  - store: is always 'pithos'
50
  - updated_at: the 'modified' meta
51
"""
52

    
53
import json
54
import warnings
55

    
56
from operator import itemgetter
57
from time import gmtime, strftime, time
58

    
59
from django.conf import settings
60

    
61
from pithos.backends.base import NotAllowedError
62

    
63

    
64
PLANKTON_DOMAIN = 'plankton'
65
PLANKTON_PREFIX = 'plankton:'
66
PROPERTY_PREFIX = 'property:'
67

    
68
PLANKTON_META = ('container_format', 'disk_format', 'name', 'properties',
69
                 'status')
70

    
71

    
72
def get_location(account, container, object):
73
    assert '/' not in account, "Invalid account"
74
    assert '/' not in container, "Invalid container"
75
    return 'pithos://%s/%s/%s' % (account, container, object)
76

    
77

    
78
def split_location(location):
79
    """Returns (accout, container, object) from a location string"""
80
    t = location.split('/', 4)
81
    assert len(t) == 5, "Invalid location"
82
    return t[2:5]
83

    
84

    
85
class BackendException(Exception):
86
    pass
87

    
88

    
89
from pithos.backends.util import PithosBackendPool
90
POOL_SIZE = 8
91
_pithos_backend_pool = \
92
        PithosBackendPool(POOL_SIZE,
93
                         db_connection=settings.BACKEND_DB_CONNECTION,
94
                         block_path=settings.BACKEND_BLOCK_PATH)
95

    
96

    
97
def get_pithos_backend():
98
    return _pithos_backend_pool.pool_get()
99

    
100

    
101
class ImageBackend(object):
102
    """A wrapper arround the pithos backend to simplify image handling."""
103
    
104
    def __init__(self, user):
105
        self.user = user
106

    
107
        original_filters = warnings.filters
108
        warnings.simplefilter('ignore')         # Suppress SQLAlchemy warnings
109
        self.backend = get_pithos_backend()
110
        warnings.filters = original_filters     # Restore warnings
111

    
112
    def _get_image(self, location):
113
        def format_timestamp(t):
114
            return strftime('%Y-%m-%d %H:%M:%S', gmtime(t))
115
        
116
        account, container, object = split_location(location)
117
        
118
        try:
119
            versions = self.backend.list_versions(self.user, account,
120
                    container, object)
121
        except NameError:
122
            return None
123
        
124
        image = {}
125
        
126
        meta = self._get_meta(location)
127
        if meta:
128
            image['deleted_at'] = ''
129
        else:
130
            # Object was deleted, use the latest version
131
            version, timestamp = versions[-1]
132
            meta = self._get_meta(location, version)
133
            image['deleted_at'] = format_timestamp(timestamp)
134
        
135
        if PLANKTON_PREFIX + 'name' not in meta:
136
            return None     # Not a Plankton image
137
        
138
        permissions = self._get_permissions(location)
139
        
140
        image['checksum'] = meta['hash']
141
        image['created_at'] = format_timestamp(versions[0][1])
142
        image['id'] = meta['uuid']
143
        image['is_public'] = '*' in permissions.get('read', [])
144
        image['location'] = location
145
        image['owner'] = account
146
        image['size'] = meta['bytes']
147
        image['store'] = 'pithos'
148
        image['updated_at'] = format_timestamp(meta['modified'])
149
        image['properties'] = {}
150
        
151
        for key, val in meta.items():
152
            if not key.startswith(PLANKTON_PREFIX):
153
                continue
154
            key = key[len(PLANKTON_PREFIX):]
155
            if key == 'properties':
156
                val = json.loads(val)
157
            if key in PLANKTON_META:
158
                image[key] = val
159
        
160
        return image
161
    
162
    def _get_meta(self, location, version=None):
163
        account, container, object = split_location(location)
164
        try:
165
            return self.backend.get_object_meta(self.user, account, container,
166
                    object, PLANKTON_DOMAIN, version)
167
        except NameError:
168
            return None
169
    
170
    def _get_permissions(self, location):
171
        account, container, object = split_location(location)
172
        action, path, permissions = self.backend.get_object_permissions(
173
                self.user, account, container, object)
174
        return permissions
175
    
176
    def _store(self, f, size=None):
177
        """Breaks data into blocks and stores them in the backend"""
178
        
179
        bytes = 0
180
        hashmap = []
181
        backend = self.backend
182
        blocksize = backend.block_size
183
        
184
        data = f.read(blocksize)
185
        while data:
186
            hash = backend.put_block(data)
187
            hashmap.append(hash)
188
            bytes += len(data)
189
            data = f.read(blocksize)
190
        
191
        if size and size != bytes:
192
            raise BackendException("Invalid size")
193
        
194
        return hashmap, bytes
195
    
196
    def _update(self, location, size, hashmap, meta, permissions):
197
        account, container, object = split_location(location)
198
        self.backend.update_object_hashmap(self.user, account, container,
199
                object, size, hashmap, '', PLANKTON_DOMAIN,
200
                permissions=permissions)
201
        self._update_meta(location, meta, replace=True)
202
    
203
    def _update_meta(self, location, meta, replace=False):
204
        account, container, object = split_location(location)
205
        
206
        prefixed = {}
207
        for key, val in meta.items():
208
            if key == 'properties':
209
                val = json.dumps(val)
210
            if key in PLANKTON_META:
211
                prefixed[PLANKTON_PREFIX + key] = val
212
        
213
        self.backend.update_object_meta(self.user, account, container, object,
214
                PLANKTON_DOMAIN, prefixed, replace)
215
    
216
    def _update_permissions(self, location, permissions):
217
        account, container, object = split_location(location)
218
        self.backend.update_object_permissions(self.user, account, container,
219
                object, permissions)
220
    
221
    def add_user(self, image_id, user):
222
        image = self.get_image(image_id)
223
        assert image, "Image not found"
224
        
225
        location = image['location']
226
        permissions = self._get_permissions(location)
227
        read = set(permissions.get('read', []))
228
        read.add(user)
229
        permissions['read'] = list(read)
230
        self._update_permissions(location, permissions)
231
    
232
    def close(self):
233
        self.backend.close()
234
    
235
    def delete(self, image_id):
236
        image = self.get_image(image_id)
237
        account, container, object = split_location(image['location'])
238
        self.backend.delete_object(self.user, account, container, object)
239
    
240
    def get_data(self, location):
241
        account, container, object = split_location(location)
242
        size, hashmap = self.backend.get_object_hashmap(self.user, account,
243
                container, object)
244
        data = ''.join(self.backend.get_block(hash) for hash in hashmap)
245
        assert len(data) == size
246
        return data
247
    
248
    def get_image(self, image_id):
249
        try:
250
            account, container, object = self.backend.get_uuid(self.user,
251
                    image_id)
252
        except NameError:
253
            return None
254
        
255
        location = get_location(account, container, object)
256
        return self._get_image(location)
257
    
258
    def iter(self):
259
        """Iter over all images available to the user"""
260
        
261
        backend = self.backend
262
        for account in backend.list_accounts(self.user):
263
            for container in backend.list_containers(self.user, account,
264
                                                     shared=True):
265
                for path, version_id in backend.list_objects(self.user,
266
                        account, container, domain=PLANKTON_DOMAIN):
267
                    location = get_location(account, container, path)
268
                    image = self._get_image(location)
269
                    if image:
270
                        yield image
271
    
272
    def iter_public(self, filters=None):
273
        filters = filters or {}
274
        backend = self.backend
275
        
276
        keys = [PLANKTON_PREFIX + 'name']
277
        size_range = (None, None)
278
        
279
        for key, val in filters.items():
280
            if key == 'size_min':
281
                size_range = (int(val), size_range[1])
282
            elif key == 'size_max':
283
                size_range = (size_range[0], int(val))
284
            else:
285
                keys.append('%s = %s' % (PLANKTON_PREFIX + key, val))
286
        
287
        for account in backend.list_accounts(None):
288
            for container in backend.list_containers(None, account,
289
                                                     shared=True):
290
                for path, version_id in backend.list_objects(None, account,
291
                        container, domain=PLANKTON_DOMAIN, keys=keys,
292
                        shared=True, size_range=size_range):
293
                    location = get_location(account, container, path)
294
                    image = self._get_image(location)
295
                    if image:
296
                        yield image
297
    
298
    def iter_shared(self, member):
299
        """Iterate over image ids shared to this member"""
300
        
301
        backend = self.backend
302
        
303
        # To get the list we connect as member and get the list shared by us
304
        for container in  backend.list_containers(member, self.user):
305
            for object, version_id in backend.list_objects(member, self.user,
306
                    container, domain=PLANKTON_DOMAIN):
307
                try:
308
                    location = get_location(self.user, container, object)
309
                    meta = backend.get_object_meta(member, self.user,
310
                            container, object, PLANKTON_DOMAIN)
311
                    if PLANKTON_PREFIX + 'name' in meta:
312
                        yield meta['uuid']
313
                except (NameError, NotAllowedError):
314
                    continue
315
    
316
    def list(self):
317
        """Iter over all images available to the user"""
318
        
319
        return list(self.iter())
320
    
321
    def list_public(self, filters, params):
322
        images = list(self.iter_public(filters))
323
        key = itemgetter(params.get('sort_key', 'created_at'))
324
        reverse = params.get('sort_dir', 'desc') == 'desc'
325
        images.sort(key=key, reverse=reverse)
326
        return images
327
    
328
    def list_users(self, image_id):
329
        image = self.get_image(image_id)
330
        assert image, "Image not found"
331
        
332
        permissions = self._get_permissions(image['location'])
333
        return [user for user in permissions.get('read', []) if user != '*']
334
    
335
    def put(self, name, f, params):
336
        assert 'checksum' not in params, "Passing a checksum is not supported"
337
        assert 'id' not in params, "Passing an ID is not supported"
338
        assert params.pop('store', 'pithos') == 'pithos', "Invalid store"
339
        assert params.setdefault('disk_format',
340
                settings.DEFAULT_DISK_FORMAT) in \
341
                settings.ALLOWED_DISK_FORMATS, "Invalid disk_format"
342
        assert params.setdefault('container_format',
343
                settings.DEFAULT_CONTAINER_FORMAT) in \
344
                settings.ALLOWED_CONTAINER_FORMATS, "Invalid container_format"
345
        
346
        container = settings.DEFAULT_PLANKTON_CONTAINER
347
        filename = params.pop('filename', name)
348
        location = 'pithos://%s/%s/%s' % (self.user, container, filename)
349
        is_public = params.pop('is_public', False)
350
        permissions = {'read': ['*']} if is_public else {}
351
        size = params.pop('size', None)
352
        
353
        hashmap, size = self._store(f, size)
354
        
355
        meta = {}
356
        meta['properties'] = params.pop('properties', {})
357
        meta.update(name=name, status='available', **params)
358
        
359
        self._update(location, size, hashmap, meta, permissions)
360
        return self._get_image(location)
361
    
362
    def register(self, name, location, params):
363
        assert 'id' not in params, "Passing an ID is not supported"
364
        assert location.startswith('pithos://'), "Invalid location"
365
        assert params.pop('store', 'pithos') == 'pithos', "Invalid store"
366
        assert params.setdefault('disk_format',
367
                settings.DEFAULT_DISK_FORMAT) in \
368
                settings.ALLOWED_DISK_FORMATS, "Invalid disk_format"
369
        assert params.setdefault('container_format',
370
                settings.DEFAULT_CONTAINER_FORMAT) in \
371
                settings.ALLOWED_CONTAINER_FORMATS, "Invalid container_format"
372
        
373
        user = self.user
374
        account, container, object = split_location(location)
375
        
376
        meta = self._get_meta(location)
377
        assert meta, "File not found"
378
        
379
        size = int(params.pop('size', meta['bytes']))
380
        if size != meta['bytes']:
381
            raise BackendException("Invalid size")
382
        
383
        checksum = params.pop('checksum', meta['hash'])
384
        if checksum != meta['hash']:
385
            raise BackendException("Invalid checksum")
386
        
387
        is_public = params.pop('is_public', False)
388
        permissions = {'read': ['*']} if is_public else {}
389
        
390
        meta = {}
391
        meta['properties'] = params.pop('properties', {})
392
        meta.update(name=name, status='available', **params)
393
        
394
        self._update_meta(location, meta)
395
        self._update_permissions(location, permissions)
396
        return self._get_image(location)
397
    
398
    def remove_user(self, image_id, user):
399
        image = self.get_image(image_id)
400
        assert image, "Image not found"
401
        
402
        location = image['location']
403
        permissions = self._get_permissions(location)
404
        try:
405
            permissions.get('read', []).remove(user)
406
        except ValueError:
407
            return      # User did not have access anyway
408
        self._update_permissions(location, permissions)
409
    
410
    def replace_users(self, image_id, users):
411
        image = self.get_image(image_id)
412
        assert image, "Image not found"
413
        
414
        location = image['location']
415
        permissions = self._get_permissions(location)
416
        permissions['read'] = users
417
        if image.get('is_public', False):
418
            permissions['read'].append('*')
419
        self._update_permissions(location, permissions)
420
    
421
    def update(self, image_id, params):
422
        image = self.get_image(image_id)
423
        assert image, "Image not found"
424
        
425
        location = image['location']
426
        is_public = params.pop('is_public', None)
427
        if is_public is not None:
428
            permissions = self._get_permissions(location)
429
            read = set(permissions.get('read', []))
430
            if is_public:
431
                read.add('*')
432
            else:
433
                read.discard('*')
434
            permissions['read'] = list(read)
435
            self.backend._update_permissions(location, permissions)
436
        
437
        meta = {}
438
        meta['properties'] = params.pop('properties', {})
439
        meta.update(**params)
440
        
441
        self._update_meta(location, meta)
442
        return self.get_image(image_id)