Statistics
| Branch: | Tag: | Revision:

root / snf-cyclades-app / synnefo / plankton / backend.py @ c3bcaeff

History | View | Annotate | Download (16.6 kB)

1
# Copyright 2011 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
"""
35
The Plankton attributes are the following:
36
  - checksum: the 'hash' meta
37
  - container_format: stored as a user meta
38
  - created_at: the 'modified' meta of the first version
39
  - deleted_at: the timestamp of the last version
40
  - disk_format: stored as a user meta
41
  - id: the 'uuid' meta
42
  - is_public: True if there is a * entry for the read permission
43
  - location: generated based on the file's path
44
  - name: stored as a user meta
45
  - owner: the file's account
46
  - properties: stored as user meta prefixed with PROPERTY_PREFIX
47
  - size: the 'bytes' meta
48
  - status: stored as a system meta
49
  - store: is always 'pithos'
50
  - updated_at: the 'modified' meta
51
"""
52

    
53
import json
54
import warnings
55

    
56
from operator import itemgetter
57
from time import gmtime, strftime
58
from functools import wraps
59

    
60
from django.conf import settings
61

    
62
from pithos.backends.base import NotAllowedError as PithosNotAllowedError
63

    
64

    
65
PLANKTON_DOMAIN = 'plankton'
66
PLANKTON_PREFIX = 'plankton:'
67
PROPERTY_PREFIX = 'property:'
68

    
69
PLANKTON_META = ('container_format', 'disk_format', 'name', 'properties',
70
                 'status')
71

    
72

    
73
def get_location(account, container, object):
74
    assert '/' not in account, "Invalid account"
75
    assert '/' not in container, "Invalid container"
76
    return 'pithos://%s/%s/%s' % (account, container, object)
77

    
78

    
79
def split_location(location):
80
    """Returns (accout, container, object) from a location string"""
81
    t = location.split('/', 4)
82
    assert len(t) == 5, "Invalid location"
83
    return t[2:5]
84

    
85

    
86
class BackendException(Exception):
87
    pass
88

    
89

    
90
class NotAllowedError(BackendException):
91
    pass
92

    
93

    
94
from pithos.backends.util import PithosBackendPool
95
POOL_SIZE = 8
96
_pithos_backend_pool = \
97
        PithosBackendPool(POOL_SIZE,
98
                         db_connection=settings.BACKEND_DB_CONNECTION,
99
                         block_path=settings.BACKEND_BLOCK_PATH)
100

    
101

    
102
def get_pithos_backend():
103
    return _pithos_backend_pool.pool_get()
104

    
105

    
106
def handle_backend_exceptions(func):
107
    @wraps(func)
108
    def wrapper(*args, **kwargs):
109
        try:
110
            return func(*args, **kwargs)
111
        except PithosNotAllowedError:
112
            raise NotAllowedError()
113
    return wrapper
114

    
115

    
116
class ImageBackend(object):
117
    """A wrapper arround the pithos backend to simplify image handling."""
118

    
119
    def __init__(self, user):
120
        self.user = user
121

    
122
        original_filters = warnings.filters
123
        warnings.simplefilter('ignore')         # Suppress SQLAlchemy warnings
124
        self.backend = get_pithos_backend()
125
        warnings.filters = original_filters     # Restore warnings
126

    
127
    @handle_backend_exceptions
128
    def _get_image(self, location):
129
        def format_timestamp(t):
130
            return strftime('%Y-%m-%d %H:%M:%S', gmtime(t))
131

    
132
        account, container, object = split_location(location)
133

    
134
        try:
135
            versions = self.backend.list_versions(self.user, account,
136
                    container, object)
137
        except NameError:
138
            return None
139

    
140
        image = {}
141

    
142
        meta = self._get_meta(location)
143
        if meta:
144
            image['deleted_at'] = ''
145
        else:
146
            # Object was deleted, use the latest version
147
            version, timestamp = versions[-1]
148
            meta = self._get_meta(location, version)
149
            image['deleted_at'] = format_timestamp(timestamp)
150

    
151
        if PLANKTON_PREFIX + 'name' not in meta:
152
            return None     # Not a Plankton image
153

    
154
        permissions = self._get_permissions(location)
155

    
156
        image['checksum'] = meta['hash']
157
        image['created_at'] = format_timestamp(versions[0][1])
158
        image['id'] = meta['uuid']
159
        image['is_public'] = '*' in permissions.get('read', [])
160
        image['location'] = location
161
        image['owner'] = account
162
        image['size'] = meta['bytes']
163
        image['store'] = 'pithos'
164
        image['updated_at'] = format_timestamp(meta['modified'])
165
        image['properties'] = {}
166

    
167
        for key, val in meta.items():
168
            if not key.startswith(PLANKTON_PREFIX):
169
                continue
170
            key = key[len(PLANKTON_PREFIX):]
171
            if key == 'properties':
172
                val = json.loads(val)
173
            if key in PLANKTON_META:
174
                image[key] = val
175

    
176
        return image
177

    
178
    @handle_backend_exceptions
179
    def _get_meta(self, location, version=None):
180
        account, container, object = split_location(location)
181
        try:
182
            return self.backend.get_object_meta(self.user, account, container,
183
                    object, PLANKTON_DOMAIN, version)
184
        except NameError:
185
            return None
186

    
187
    @handle_backend_exceptions
188
    def _get_permissions(self, location):
189
        account, container, object = split_location(location)
190
        action, path, permissions = self.backend.get_object_permissions(
191
                self.user, account, container, object)
192
        return permissions
193

    
194
    @handle_backend_exceptions
195
    def _store(self, f, size=None):
196
        """Breaks data into blocks and stores them in the backend"""
197

    
198
        bytes = 0
199
        hashmap = []
200
        backend = self.backend
201
        blocksize = backend.block_size
202

    
203
        data = f.read(blocksize)
204
        while data:
205
            hash = backend.put_block(data)
206
            hashmap.append(hash)
207
            bytes += len(data)
208
            data = f.read(blocksize)
209

    
210
        if size and size != bytes:
211
            raise BackendException("Invalid size")
212

    
213
        return hashmap, bytes
214

    
215
    @handle_backend_exceptions
216
    def _update(self, location, size, hashmap, meta, permissions):
217
        account, container, object = split_location(location)
218
        self.backend.update_object_hashmap(self.user, account, container,
219
                object, size, hashmap, '', PLANKTON_DOMAIN,
220
                permissions=permissions)
221
        self._update_meta(location, meta, replace=True)
222

    
223
    @handle_backend_exceptions
224
    def _update_meta(self, location, meta, replace=False):
225
        account, container, object = split_location(location)
226

    
227
        prefixed = {}
228
        for key, val in meta.items():
229
            if key == 'properties':
230
                val = json.dumps(val)
231
            if key in PLANKTON_META:
232
                prefixed[PLANKTON_PREFIX + key] = val
233

    
234
        self.backend.update_object_meta(self.user, account, container, object,
235
                PLANKTON_DOMAIN, prefixed, replace)
236

    
237
    @handle_backend_exceptions
238
    def _update_permissions(self, location, permissions):
239
        account, container, object = split_location(location)
240
        self.backend.update_object_permissions(self.user, account, container,
241
                object, permissions)
242

    
243
    @handle_backend_exceptions
244
    def add_user(self, image_id, user):
245
        image = self.get_image(image_id)
246
        assert image, "Image not found"
247

    
248
        location = image['location']
249
        permissions = self._get_permissions(location)
250
        read = set(permissions.get('read', []))
251
        read.add(user)
252
        permissions['read'] = list(read)
253
        self._update_permissions(location, permissions)
254

    
255
    def close(self):
256
        self.backend.close()
257

    
258
    @handle_backend_exceptions
259
    def delete(self, image_id):
260
        image = self.get_image(image_id)
261
        account, container, object = split_location(image['location'])
262
        self.backend.delete_object(self.user, account, container, object)
263

    
264
    @handle_backend_exceptions
265
    def get_data(self, location):
266
        account, container, object = split_location(location)
267
        size, hashmap = self.backend.get_object_hashmap(self.user, account,
268
                container, object)
269
        data = ''.join(self.backend.get_block(hash) for hash in hashmap)
270
        assert len(data) == size
271
        return data
272

    
273
    @handle_backend_exceptions
274
    def get_image(self, image_id):
275
        try:
276
            account, container, object = self.backend.get_uuid(self.user,
277
                    image_id)
278
        except NameError:
279
            return None
280

    
281
        location = get_location(account, container, object)
282
        return self._get_image(location)
283

    
284
    @handle_backend_exceptions
285
    def _iter(self, public=False, filters=None, shared_from=None):
286
        filters = filters or {}
287

    
288
        # Fix keys
289
        keys = [PLANKTON_PREFIX + 'name']
290
        size_range = (None, None)
291
        for key, val in filters.items():
292
            if key == 'size_min':
293
                size_range = (val, size_range[1])
294
            elif key == 'size_max':
295
                size_range = (size_range[0], val)
296
            else:
297
                keys.append('%s = %s' % (PLANKTON_PREFIX + key, val))
298

    
299
        backend = self.backend
300
        if shared_from:
301
            # To get shared images, we connect as shared_from member and
302
            # get the list shared by us
303
            user = shared_from
304
            accounts = [self.user]
305
        else:
306
            user = None if public else self.user
307
            accounts = backend.list_accounts(user)
308

    
309
        for account in accounts:
310
            for container in backend.list_containers(user, account,
311
                                                     shared=True):
312
                for path, _ in backend.list_objects(user, account, container,
313
                                                    domain=PLANKTON_DOMAIN,
314
                                                    keys=keys, shared=True,
315
                                                    size_range=size_range):
316
                    location = get_location(account, container, path)
317
                    image = self._get_image(location)
318
                    if image:
319
                        yield image
320

    
321
    def iter(self, filters=None):
322
        """Iter over all images available to the user"""
323
        return self._iter(filters=filters)
324

    
325
    def iter_public(self, filters=None):
326
        """Iter over public images"""
327
        return self._iter(public=True, filters=filters)
328

    
329
    def iter_shared(self, filters=None, member=None):
330
        """Iter over images shared to member"""
331
        return self._iter(filters=filters, shared_from=member)
332

    
333
    def list(self, filters=None, params={}):
334
        """Return all images available to the user"""
335
        images = list(self.iter(filters))
336
        key = itemgetter(params.get('sort_key', 'created_at'))
337
        reverse = params.get('sort_dir', 'desc') == 'desc'
338
        images.sort(key=key, reverse=reverse)
339
        return images
340

    
341
    def list_public(self, filters, params={}):
342
        """Return public images"""
343
        images = list(self.iter_public(filters))
344
        key = itemgetter(params.get('sort_key', 'created_at'))
345
        reverse = params.get('sort_dir', 'desc') == 'desc'
346
        images.sort(key=key, reverse=reverse)
347
        return images
348

    
349
    def list_users(self, image_id):
350
        image = self.get_image(image_id)
351
        assert image, "Image not found"
352

    
353
        permissions = self._get_permissions(image['location'])
354
        return [user for user in permissions.get('read', []) if user != '*']
355

    
356
    @handle_backend_exceptions
357
    def put(self, name, f, params):
358
        assert 'checksum' not in params, "Passing a checksum is not supported"
359
        assert 'id' not in params, "Passing an ID is not supported"
360
        assert params.pop('store', 'pithos') == 'pithos', "Invalid store"
361
        assert params.setdefault('disk_format',
362
                settings.DEFAULT_DISK_FORMAT) in \
363
                settings.ALLOWED_DISK_FORMATS, "Invalid disk_format"
364
        assert params.setdefault('container_format',
365
                settings.DEFAULT_CONTAINER_FORMAT) in \
366
                settings.ALLOWED_CONTAINER_FORMATS, "Invalid container_format"
367

    
368
        container = settings.DEFAULT_PLANKTON_CONTAINER
369
        filename = params.pop('filename', name)
370
        location = 'pithos://%s/%s/%s' % (self.user, container, filename)
371
        is_public = params.pop('is_public', False)
372
        permissions = {'read': ['*']} if is_public else {}
373
        size = params.pop('size', None)
374

    
375
        hashmap, size = self._store(f, size)
376

    
377
        meta = {}
378
        meta['properties'] = params.pop('properties', {})
379
        meta.update(name=name, status='available', **params)
380

    
381
        self._update(location, size, hashmap, meta, permissions)
382
        return self._get_image(location)
383

    
384
    @handle_backend_exceptions
385
    def register(self, name, location, params):
386
        assert 'id' not in params, "Passing an ID is not supported"
387
        assert location.startswith('pithos://'), "Invalid location"
388
        assert params.pop('store', 'pithos') == 'pithos', "Invalid store"
389
        assert params.setdefault('disk_format',
390
                settings.DEFAULT_DISK_FORMAT) in \
391
                settings.ALLOWED_DISK_FORMATS, "Invalid disk_format"
392
        assert params.setdefault('container_format',
393
                settings.DEFAULT_CONTAINER_FORMAT) in \
394
                settings.ALLOWED_CONTAINER_FORMATS, "Invalid container_format"
395

    
396
        # user = self.user
397
        account, container, object = split_location(location)
398

    
399
        meta = self._get_meta(location)
400
        assert meta, "File not found"
401

    
402
        size = int(params.pop('size', meta['bytes']))
403
        if size != meta['bytes']:
404
            raise BackendException("Invalid size")
405

    
406
        checksum = params.pop('checksum', meta['hash'])
407
        if checksum != meta['hash']:
408
            raise BackendException("Invalid checksum")
409

    
410
        is_public = params.pop('is_public', False)
411
        if is_public:
412
            permissions = {'read': ['*']}
413
        else:
414
            permissions = {'read': [self.user]}
415

    
416
        meta = {}
417
        meta['properties'] = params.pop('properties', {})
418
        meta.update(name=name, status='available', **params)
419

    
420
        self._update_meta(location, meta)
421
        self._update_permissions(location, permissions)
422
        return self._get_image(location)
423

    
424
    @handle_backend_exceptions
425
    def remove_user(self, image_id, user):
426
        image = self.get_image(image_id)
427
        assert image, "Image not found"
428

    
429
        location = image['location']
430
        permissions = self._get_permissions(location)
431
        try:
432
            permissions.get('read', []).remove(user)
433
        except ValueError:
434
            return      # User did not have access anyway
435
        self._update_permissions(location, permissions)
436

    
437
    @handle_backend_exceptions
438
    def replace_users(self, image_id, users):
439
        image = self.get_image(image_id)
440
        assert image, "Image not found"
441

    
442
        location = image['location']
443
        permissions = self._get_permissions(location)
444
        permissions['read'] = users
445
        if image.get('is_public', False):
446
            permissions['read'].append('*')
447
        self._update_permissions(location, permissions)
448

    
449
    @handle_backend_exceptions
450
    def update(self, image_id, params):
451
        image = self.get_image(image_id)
452
        assert image, "Image not found"
453

    
454
        location = image['location']
455
        is_public = params.pop('is_public', None)
456
        if is_public is not None:
457
            permissions = self._get_permissions(location)
458
            read = set(permissions.get('read', []))
459
            if is_public:
460
                read.add('*')
461
            else:
462
                read.discard('*')
463
            permissions['read'] = list(read)
464
            self.backend._update_permissions(location, permissions)
465

    
466
        meta = {}
467
        meta['properties'] = params.pop('properties', {})
468
        meta.update(**params)
469

    
470
        self._update_meta(location, meta)
471
        return self.get_image(image_id)