Statistics
| Branch: | Tag: | Revision:

root / snf-cyclades-app / synnefo / plankton / backend.py @ 1e28ba40

History | View | Annotate | Download (16 kB)

1
# Copyright 2011 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
"""
35
The Plankton attributes are the following:
36
  - checksum: the 'hash' meta
37
  - container_format: stored as a user meta
38
  - created_at: the 'modified' meta of the first version
39
  - deleted_at: the timestamp of the last version
40
  - disk_format: stored as a user meta
41
  - id: the 'uuid' meta
42
  - is_public: True if there is a * entry for the read permission
43
  - location: generated based on the file's path
44
  - name: stored as a user meta
45
  - owner: the file's account
46
  - properties: stored as user meta prefixed with PROPERTY_PREFIX
47
  - size: the 'bytes' meta
48
  - status: stored as a system meta
49
  - store: is always 'pithos'
50
  - updated_at: the 'modified' meta
51
"""
52

    
53
import json
54
import warnings
55

    
56
from operator import itemgetter
57
from time import gmtime, strftime, time
58

    
59
from django.conf import settings
60

    
61
from pithos.backends.base import NotAllowedError
62

    
63

    
64
PLANKTON_DOMAIN = 'plankton'
65
PLANKTON_PREFIX = 'plankton:'
66
PROPERTY_PREFIX = 'property:'
67

    
68
PLANKTON_META = ('container_format', 'disk_format', 'name', 'properties',
69
                 'status')
70

    
71

    
72
def get_location(account, container, object):
73
    assert '/' not in account, "Invalid account"
74
    assert '/' not in container, "Invalid container"
75
    return 'pithos://%s/%s/%s' % (account, container, object)
76

    
77

    
78
def split_location(location):
79
    """Returns (accout, container, object) from a location string"""
80
    t = location.split('/', 4)
81
    assert len(t) == 5, "Invalid location"
82
    return t[2:5]
83

    
84

    
85
class BackendException(Exception):
86
    pass
87

    
88

    
89
from pithos.backends.util import PithosBackendPool
90
POOL_SIZE = 8
91
_pithos_backend_pool = \
92
        PithosBackendPool(POOL_SIZE,
93
                         db_connection=settings.BACKEND_DB_CONNECTION,
94
                         block_path=settings.BACKEND_BLOCK_PATH)
95

    
96

    
97
def get_pithos_backend():
98
    return _pithos_backend_pool.pool_get()
99

    
100

    
101
class ImageBackend(object):
102
    """A wrapper arround the pithos backend to simplify image handling."""
103

    
104
    def __init__(self, user):
105
        self.user = user
106

    
107
        original_filters = warnings.filters
108
        warnings.simplefilter('ignore')         # Suppress SQLAlchemy warnings
109
        self.backend = get_pithos_backend()
110
        warnings.filters = original_filters     # Restore warnings
111

    
112
    def _get_image(self, location):
113
        def format_timestamp(t):
114
            return strftime('%Y-%m-%d %H:%M:%S', gmtime(t))
115

    
116
        account, container, object = split_location(location)
117

    
118
        try:
119
            versions = self.backend.list_versions(self.user, account,
120
                    container, object)
121
        except NameError:
122
            return None
123

    
124
        image = {}
125

    
126
        meta = self._get_meta(location)
127
        if meta:
128
            image['deleted_at'] = ''
129
        else:
130
            # Object was deleted, use the latest version
131
            version, timestamp = versions[-1]
132
            meta = self._get_meta(location, version)
133
            image['deleted_at'] = format_timestamp(timestamp)
134

    
135
        if PLANKTON_PREFIX + 'name' not in meta:
136
            return None     # Not a Plankton image
137

    
138
        permissions = self._get_permissions(location)
139

    
140
        image['checksum'] = meta['hash']
141
        image['created_at'] = format_timestamp(versions[0][1])
142
        image['id'] = meta['uuid']
143
        image['is_public'] = '*' in permissions.get('read', [])
144
        image['location'] = location
145
        image['owner'] = account
146
        image['size'] = meta['bytes']
147
        image['store'] = 'pithos'
148
        image['updated_at'] = format_timestamp(meta['modified'])
149
        image['properties'] = {}
150

    
151
        for key, val in meta.items():
152
            if not key.startswith(PLANKTON_PREFIX):
153
                continue
154
            key = key[len(PLANKTON_PREFIX):]
155
            if key == 'properties':
156
                val = json.loads(val)
157
            if key in PLANKTON_META:
158
                image[key] = val
159

    
160
        return image
161

    
162
    def _get_meta(self, location, version=None):
163
        account, container, object = split_location(location)
164
        try:
165
            return self.backend.get_object_meta(self.user, account, container,
166
                    object, PLANKTON_DOMAIN, version)
167
        except NameError:
168
            return None
169

    
170
    def _get_permissions(self, location):
171
        account, container, object = split_location(location)
172
        action, path, permissions = self.backend.get_object_permissions(
173
                self.user, account, container, object)
174
        return permissions
175

    
176
    def _store(self, f, size=None):
177
        """Breaks data into blocks and stores them in the backend"""
178

    
179
        bytes = 0
180
        hashmap = []
181
        backend = self.backend
182
        blocksize = backend.block_size
183

    
184
        data = f.read(blocksize)
185
        while data:
186
            hash = backend.put_block(data)
187
            hashmap.append(hash)
188
            bytes += len(data)
189
            data = f.read(blocksize)
190

    
191
        if size and size != bytes:
192
            raise BackendException("Invalid size")
193

    
194
        return hashmap, bytes
195

    
196
    def _update(self, location, size, hashmap, meta, permissions):
197
        account, container, object = split_location(location)
198
        self.backend.update_object_hashmap(self.user, account, container,
199
                object, size, hashmap, '', PLANKTON_DOMAIN,
200
                permissions=permissions)
201
        self._update_meta(location, meta, replace=True)
202

    
203
    def _update_meta(self, location, meta, replace=False):
204
        account, container, object = split_location(location)
205

    
206
        prefixed = {}
207
        for key, val in meta.items():
208
            if key == 'properties':
209
                val = json.dumps(val)
210
            if key in PLANKTON_META:
211
                prefixed[PLANKTON_PREFIX + key] = val
212

    
213
        self.backend.update_object_meta(self.user, account, container, object,
214
                PLANKTON_DOMAIN, prefixed, replace)
215

    
216
    def _update_permissions(self, location, permissions):
217
        account, container, object = split_location(location)
218
        self.backend.update_object_permissions(self.user, account, container,
219
                object, permissions)
220

    
221
    def add_user(self, image_id, user):
222
        image = self.get_image(image_id)
223
        assert image, "Image not found"
224

    
225
        location = image['location']
226
        permissions = self._get_permissions(location)
227
        read = set(permissions.get('read', []))
228
        read.add(user)
229
        permissions['read'] = list(read)
230
        self._update_permissions(location, permissions)
231

    
232
    def close(self):
233
        self.backend.close()
234

    
235
    def delete(self, image_id):
236
        image = self.get_image(image_id)
237
        account, container, object = split_location(image['location'])
238
        self.backend.delete_object(self.user, account, container, object)
239

    
240
    def get_data(self, location):
241
        account, container, object = split_location(location)
242
        size, hashmap = self.backend.get_object_hashmap(self.user, account,
243
                container, object)
244
        data = ''.join(self.backend.get_block(hash) for hash in hashmap)
245
        assert len(data) == size
246
        return data
247

    
248
    def get_image(self, image_id):
249
        try:
250
            account, container, object = self.backend.get_uuid(self.user,
251
                    image_id)
252
        except NameError:
253
            return None
254

    
255
        location = get_location(account, container, object)
256
        return self._get_image(location)
257

    
258
    def iter(self):
259
        """Iter over all images available to the user"""
260

    
261
        backend = self.backend
262
        for account in backend.list_accounts(self.user):
263
            for container in backend.list_containers(self.user, account,
264
                                                     shared=True):
265
                for path, version_id in backend.list_objects(self.user,
266
                        account, container, domain=PLANKTON_DOMAIN):
267
                    location = get_location(account, container, path)
268
                    image = self._get_image(location)
269
                    if image:
270
                        yield image
271

    
272
    def iter_public(self, filters=None):
273
        filters = filters or {}
274
        backend = self.backend
275

    
276
        keys = [PLANKTON_PREFIX + 'name']
277
        size_range = (None, None)
278

    
279
        for key, val in filters.items():
280
            if key == 'size_min':
281
                size_range = (int(val), size_range[1])
282
            elif key == 'size_max':
283
                size_range = (size_range[0], int(val))
284
            else:
285
                keys.append('%s = %s' % (PLANKTON_PREFIX + key, val))
286

    
287
        for account in backend.list_accounts(None):
288
            for container in backend.list_containers(None, account,
289
                                                     shared=True):
290
                for path, version_id in backend.list_objects(None, account,
291
                        container, domain=PLANKTON_DOMAIN, keys=keys,
292
                        shared=True, size_range=size_range):
293
                    location = get_location(account, container, path)
294
                    image = self._get_image(location)
295
                    if image:
296
                        yield image
297

    
298
    def iter_shared(self, member):
299
        """Iterate over image ids shared to this member"""
300

    
301
        backend = self.backend
302

    
303
        # To get the list we connect as member and get the list shared by us
304
        for container in  backend.list_containers(member, self.user):
305
            for object, version_id in backend.list_objects(member, self.user,
306
                    container, domain=PLANKTON_DOMAIN):
307
                try:
308
                    location = get_location(self.user, container, object)
309
                    meta = backend.get_object_meta(member, self.user,
310
                            container, object, PLANKTON_DOMAIN)
311
                    if PLANKTON_PREFIX + 'name' in meta:
312
                        yield meta['uuid']
313
                except (NameError, NotAllowedError):
314
                    continue
315

    
316
    def list(self):
317
        """Iter over all images available to the user"""
318

    
319
        return list(self.iter())
320

    
321
    def list_public(self, filters, params):
322
        images = list(self.iter_public(filters))
323
        key = itemgetter(params.get('sort_key', 'created_at'))
324
        reverse = params.get('sort_dir', 'desc') == 'desc'
325
        images.sort(key=key, reverse=reverse)
326
        return images
327

    
328
    def list_users(self, image_id):
329
        image = self.get_image(image_id)
330
        assert image, "Image not found"
331

    
332
        permissions = self._get_permissions(image['location'])
333
        return [user for user in permissions.get('read', []) if user != '*']
334

    
335
    def put(self, name, f, params):
336
        assert 'checksum' not in params, "Passing a checksum is not supported"
337
        assert 'id' not in params, "Passing an ID is not supported"
338
        assert params.pop('store', 'pithos') == 'pithos', "Invalid store"
339
        assert params.setdefault('disk_format',
340
                settings.DEFAULT_DISK_FORMAT) in \
341
                settings.ALLOWED_DISK_FORMATS, "Invalid disk_format"
342
        assert params.setdefault('container_format',
343
                settings.DEFAULT_CONTAINER_FORMAT) in \
344
                settings.ALLOWED_CONTAINER_FORMATS, "Invalid container_format"
345

    
346
        container = settings.DEFAULT_PLANKTON_CONTAINER
347
        filename = params.pop('filename', name)
348
        location = 'pithos://%s/%s/%s' % (self.user, container, filename)
349
        is_public = params.pop('is_public', False)
350
        permissions = {'read': ['*']} if is_public else {}
351
        size = params.pop('size', None)
352

    
353
        hashmap, size = self._store(f, size)
354

    
355
        meta = {}
356
        meta['properties'] = params.pop('properties', {})
357
        meta.update(name=name, status='available', **params)
358

    
359
        self._update(location, size, hashmap, meta, permissions)
360
        return self._get_image(location)
361

    
362
    def register(self, name, location, params):
363
        assert 'id' not in params, "Passing an ID is not supported"
364
        assert location.startswith('pithos://'), "Invalid location"
365
        assert params.pop('store', 'pithos') == 'pithos', "Invalid store"
366
        assert params.setdefault('disk_format',
367
                settings.DEFAULT_DISK_FORMAT) in \
368
                settings.ALLOWED_DISK_FORMATS, "Invalid disk_format"
369
        assert params.setdefault('container_format',
370
                settings.DEFAULT_CONTAINER_FORMAT) in \
371
                settings.ALLOWED_CONTAINER_FORMATS, "Invalid container_format"
372

    
373
        # user = self.user
374
        account, container, object = split_location(location)
375

    
376
        meta = self._get_meta(location)
377
        assert meta, "File not found"
378

    
379
        size = int(params.pop('size', meta['bytes']))
380
        if size != meta['bytes']:
381
            raise BackendException("Invalid size")
382

    
383
        checksum = params.pop('checksum', meta['hash'])
384
        if checksum != meta['hash']:
385
            raise BackendException("Invalid checksum")
386

    
387
        is_public = params.pop('is_public', False)
388
        permissions = {'read': ['*']} if is_public else {}
389

    
390
        meta = {}
391
        meta['properties'] = params.pop('properties', {})
392
        meta.update(name=name, status='available', **params)
393

    
394
        self._update_meta(location, meta)
395
        self._update_permissions(location, permissions)
396
        return self._get_image(location)
397

    
398
    def remove_user(self, image_id, user):
399
        image = self.get_image(image_id)
400
        assert image, "Image not found"
401

    
402
        location = image['location']
403
        permissions = self._get_permissions(location)
404
        try:
405
            permissions.get('read', []).remove(user)
406
        except ValueError:
407
            return      # User did not have access anyway
408
        self._update_permissions(location, permissions)
409

    
410
    def replace_users(self, image_id, users):
411
        image = self.get_image(image_id)
412
        assert image, "Image not found"
413

    
414
        location = image['location']
415
        permissions = self._get_permissions(location)
416
        permissions['read'] = users
417
        if image.get('is_public', False):
418
            permissions['read'].append('*')
419
        self._update_permissions(location, permissions)
420

    
421
    def update(self, image_id, params):
422
        image = self.get_image(image_id)
423
        assert image, "Image not found"
424

    
425
        location = image['location']
426
        is_public = params.pop('is_public', None)
427
        if is_public is not None:
428
            permissions = self._get_permissions(location)
429
            read = set(permissions.get('read', []))
430
            if is_public:
431
                read.add('*')
432
            else:
433
                read.discard('*')
434
            permissions['read'] = list(read)
435
            self.backend._update_permissions(location, permissions)
436

    
437
        meta = {}
438
        meta['properties'] = params.pop('properties', {})
439
        meta.update(**params)
440

    
441
        self._update_meta(location, meta)
442
        return self.get_image(image_id)