Statistics
| Branch: | Tag: | Revision:

root / snf-cyclades-app / synnefo / plankton / backend.py @ 59573532

History | View | Annotate | Download (16.6 kB)

1
# Copyright 2011-2013 GRNET S.A. All rights reserved.
2

    
3
#
4
# Redistribution and use in source and binary forms, with or
5
# without modification, are permitted provided that the following
6
# conditions are met:
7
#
8
#   1. Redistributions of source code must retain the above
9
#      copyright notice, this list of conditions and the following
10
#      disclaimer.
11
#
12
#   2. Redistributions in binary form must reproduce the above
13
#      copyright notice, this list of conditions and the following
14
#      disclaimer in the documentation and/or other materials
15
#      provided with the distribution.
16
#
17
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
18
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
19
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
21
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
24
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
25
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
27
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28
# POSSIBILITY OF SUCH DAMAGE.
29
#
30
# The views and conclusions contained in the software and
31
# documentation are those of the authors and should not be
32
# interpreted as representing official policies, either expressed
33
# or implied, of GRNET S.A.
34

    
35
"""
36
The Plankton attributes are the following:
37
  - checksum: the 'hash' meta
38
  - container_format: stored as a user meta
39
  - created_at: the 'modified' meta of the first version
40
  - deleted_at: the timestamp of the last version
41
  - disk_format: stored as a user meta
42
  - id: the 'uuid' meta
43
  - is_public: True if there is a * entry for the read permission
44
  - location: generated based on the file's path
45
  - name: stored as a user meta
46
  - owner: the file's account
47
  - properties: stored as user meta prefixed with PROPERTY_PREFIX
48
  - size: the 'bytes' meta
49
  - status: stored as a system meta
50
  - store: is always 'pithos'
51
  - updated_at: the 'modified' meta
52
"""
53

    
54
import json
55
import warnings
56
import logging
57
from time import gmtime, strftime
58
from functools import wraps
59
from operator import itemgetter
60

    
61
from django.conf import settings
62
from pithos.backends.base import NotAllowedError, VersionNotExists
63

    
64
logger = logging.getLogger(__name__)
65

    
66

    
67
PLANKTON_DOMAIN = 'plankton'
68
PLANKTON_PREFIX = 'plankton:'
69
PROPERTY_PREFIX = 'property:'
70

    
71
PLANKTON_META = ('container_format', 'disk_format', 'name', 'properties',
72
                 'status')
73

    
74
from pithos.backends.util import PithosBackendPool
75
POOL_SIZE = 8
76
_pithos_backend_pool = \
77
    PithosBackendPool(
78
        POOL_SIZE,
79
        quotaholder_enabled=settings.CYCLADES_USE_QUOTAHOLDER,
80
        quotaholder_url=settings.CYCLADES_QUOTAHOLDER_URL,
81
        quotaholder_token=settings.CYCLADES_QUOTAHOLDER_TOKEN,
82
        quotaholder_client_poolsize=settings.CYCLADES_QUOTAHOLDER_POOLSIZE,
83
        db_connection=settings.BACKEND_DB_CONNECTION,
84
        block_path=settings.BACKEND_BLOCK_PATH)
85

    
86

    
87
def get_pithos_backend():
88
    return _pithos_backend_pool.pool_get()
89

    
90

    
91
def create_url(account, container, name):
92
    assert "/" not in account, "Invalid account"
93
    assert "/" not in container, "Invalid container"
94
    return "pithos://%s/%s/%s" % (account, container, name)
95

    
96

    
97
def split_url(url):
98
    """Returns (accout, container, object) from a url string"""
99
    t = url.split('/', 4)
100
    assert t[0] == "pithos:", "Invalid url"
101
    assert len(t) == 5, "Invalid url"
102
    return t[2:5]
103

    
104

    
105
def format_timestamp(t):
106
    return strftime('%Y-%m-%d %H:%M:%S', gmtime(t))
107

    
108

    
109
def handle_backend_exceptions(func):
110
    @wraps(func)
111
    def wrapper(*args, **kwargs):
112
        try:
113
            return func(*args, **kwargs)
114
        except NotAllowedError:
115
            raise Forbidden
116
        except NameError:
117
            raise ImageNotFound
118
        except VersionNotExists:
119
            raise ImageNotFound
120
    return wrapper
121

    
122

    
123
class ImageBackend(object):
124
    """A wrapper arround the pithos backend to simplify image handling."""
125

    
126
    def __init__(self, user):
127
        self.user = user
128

    
129
        original_filters = warnings.filters
130
        warnings.simplefilter('ignore')         # Suppress SQLAlchemy warnings
131
        self.backend = get_pithos_backend()
132
        warnings.filters = original_filters     # Restore warnings
133

    
134
    def close(self):
135
        """Close PithosBackend(return to pool)"""
136
        self.backend.close()
137

    
138
    @handle_backend_exceptions
139
    def get_image(self, image_uuid):
140
        """Retrieve information about an image."""
141
        image_url = self._get_image_url(image_uuid)
142
        return self._get_image(image_url)
143

    
144
    def _get_image_url(self, image_uuid):
145
        """Get the Pithos url that corresponds to an image UUID."""
146
        account, container, name = self.backend.get_uuid(self.user, image_uuid)
147
        return create_url(account, container, name)
148

    
149
    def _get_image(self, image_url):
150
        """Get information about an Image.
151

152
        Get all available information about an Image.
153
        """
154
        account, container, name = split_url(image_url)
155
        versions = self.backend.list_versions(self.user, account, container,
156
                                              name)
157
        if not versions:
158
            raise Exception("Image without versions %s" % image_url)
159
        try:
160
            meta = self._get_meta(image_url)
161
            meta["deleted"] = ""
162
        except NameError:
163
            # Object was deleted, use the latest version
164
            version, timestamp = versions[-1]
165
            meta = self._get_meta(image_url, version)
166
            meta["deleted"] = timestamp
167

    
168
        meta["created"] = versions[0][1]
169

    
170
        if PLANKTON_PREFIX + 'name' not in meta:
171
            raise ImageNotFound("'%s' is not a Plankton image" % image_url)
172

    
173
        permissions = self._get_permissions(image_url)
174
        return image_to_dict(image_url, meta, permissions)
175

    
176
    def _get_meta(self, image_url, version=None):
177
        """Get object's metadata."""
178
        account, container, name = split_url(image_url)
179
        return self.backend.get_object_meta(self.user, account, container,
180
                                            name, PLANKTON_DOMAIN, version)
181

    
182
    def _update_meta(self, image_url, meta, replace=False):
183
        """Update object's metadata."""
184
        account, container, name = split_url(image_url)
185

    
186
        prefixed = {}
187
        for key, val in meta.items():
188
            if key in PLANKTON_META:
189
                if key == "properties":
190
                    val = json.dumps(val)
191
                prefixed[PLANKTON_PREFIX + key] = val
192

    
193
        self.backend.update_object_meta(self.user, account, container, name,
194
                                        PLANKTON_DOMAIN, prefixed, replace)
195

    
196
    def _get_permissions(self, image_url):
197
        """Get object's permissions."""
198
        account, container, name = split_url(image_url)
199
        _a, path, permissions = \
200
            self.backend.get_object_permissions(self.user, account, container,
201
                                                name)
202

    
203
        if path is None:
204
            logger.warning("Image '%s' got permissions from None path",
205
                           image_url)
206

    
207
        return permissions
208

    
209
    def _update_permissions(self, image_url, permissions):
210
        """Update object's permissions."""
211
        account, container, name = split_url(image_url)
212
        self.backend.update_object_permissions(self.user, account, container,
213
                                               name, permissions)
214

    
215
    @handle_backend_exceptions
216
    def unregister(self, image_uuid):
217
        """Unregister an image.
218

219
        Unregister an image, by removing all metadata from the Pithos
220
        file that exist in the PLANKTON_DOMAIN.
221

222
        """
223
        image_url = self._get_image_url(image_uuid)
224
        self._get_image(image_url)  # Assert that it is an image
225
        # Unregister the image by removing all metadata from domain
226
        # 'PLANKTON_DOMAIN'
227
        meta = self._get_meta(image_url)
228
        for k in meta.keys():
229
            meta[k] = ""
230
        self._update_meta(image_url, meta, False)
231

    
232
    @handle_backend_exceptions
233
    def add_user(self, image_uuid, add_user):
234
        """Add a user as an image member.
235

236
        Update read permissions of Pithos file, to include the specified user.
237

238
        """
239
        image_url = self._get_image_url(image_uuid)
240
        self._get_image(image_url)  # Assert that it is an image
241
        permissions = self._get_permissions(image_url)
242
        read = set(permissions.get("read", []))
243
        assert(isinstance(add_user, (str, unicode)))
244
        read.add(add_user)
245
        permissions["read"] = list(read)
246
        self._update_permissions(image_url, permissions)
247

    
248
    @handle_backend_exceptions
249
    def remove_user(self, image_uuid, remove_user):
250
        """Remove the user from image members.
251

252
        Remove the specified user from the read permissions of the Pithos file.
253

254
        """
255
        image_url = self._get_image_url(image_uuid)
256
        self._get_image(image_url)  # Assert that it is an image
257
        permissions = self._get_permissions(image_url)
258
        read = set(permissions.get("read", []))
259
        assert(isinstance(remove_user, (str, unicode)))
260
        try:
261
            read.remove(remove_user)
262
        except ValueError:
263
            return  # TODO: User did not have access
264
        permissions["read"] = list(read)
265
        self._update_permissions(image_url, permissions)
266

    
267
    @handle_backend_exceptions
268
    def replace_users(self, image_uuid, replace_users):
269
        """Replace image members.
270

271
        Replace the read permissions of the Pithos files with the specified
272
        users. If image is specified as public, we must preserve * permission.
273

274
        """
275
        image_url = self._get_image_url(image_uuid)
276
        image = self._get_image(image_url)
277
        permissions = self._get_permissions(image_url)
278
        assert(isinstance(replace_users, list))
279
        permissions["read"] = replace_users
280
        if image.get("is_public", False):
281
            permissions["read"].append("*")
282
        self._update_permissions(image_url, permissions)
283

    
284
    @handle_backend_exceptions
285
    def list_users(self, image_uuid):
286
        """List the image members.
287

288
        List the image members, by listing all users that have read permission
289
        to the corresponding Pithos file.
290

291
        """
292
        image_url = self._get_image_url(image_uuid)
293
        self._get_image(image_url)  # Assert that it is an image
294
        permissions = self._get_permissions(image_url)
295
        return [user for user in permissions.get('read', []) if user != '*']
296

    
297
    @handle_backend_exceptions
298
    def update_metadata(self, image_uuid, metadata):
299
        """Update Image metadata."""
300
        image_url = self._get_image_url(image_uuid)
301
        self._get_image(image_url)  # Assert that it is an image
302

    
303
        is_public = metadata.pop("is_public", None)
304
        if is_public is not None:
305
            permissions = self._get_permissions(image_url)
306
            read = set(permissions.get("read", []))
307
            if is_public:
308
                read.add("*")
309
            else:
310
                read.discard("*")
311
            permissions["read"] = list(read)
312
            self._update_permissions(image_url, permissions)
313
        meta = {}
314
        meta["properties"] = metadata.pop("properties", {})
315
        meta.update(**metadata)
316

    
317
        self._update_meta(image_url, meta)
318
        return self.get_image(image_uuid)
319

    
320
    @handle_backend_exceptions
321
    def register(self, name, image_url, metadata):
322
        # Validate that metadata are allowed
323
        if "id" in metadata:
324
            raise ValueError("Passing an ID is not supported")
325
        store = metadata.pop("store", "pithos")
326
        if store != "pithos":
327
            raise ValueError("Invalid store '%s'. Only 'pithos' store is"
328
                             "supported" % store)
329
        disk_format = metadata.setdefault("disk_format",
330
                                          settings.DEFAULT_DISK_FORMAT)
331
        if disk_format not in settings.ALLOWED_DISK_FORMATS:
332
            raise ValueError("Invalid disk format '%s'" % disk_format)
333
        container_format =\
334
            metadata.setdefault("container_format",
335
                                settings.DEFAULT_CONTAINER_FORMAT)
336
        if container_format not in settings.ALLOWED_CONTAINER_FORMATS:
337
            raise ValueError("Invalid container format '%s'" %
338
                             container_format)
339

    
340
        # Validate that 'size' and 'checksum' are valid
341
        account, container, object = split_url(image_url)
342

    
343
        meta = self._get_meta(image_url)
344

    
345
        size = int(metadata.pop('size', meta['bytes']))
346
        if size != meta['bytes']:
347
            raise ValueError("Invalid size")
348

    
349
        checksum = metadata.pop('checksum', meta['hash'])
350
        if checksum != meta['hash']:
351
            raise ValueError("Invalid checksum")
352

    
353
        # Fix permissions
354
        is_public = metadata.pop('is_public', False)
355
        if is_public:
356
            permissions = {'read': ['*']}
357
        else:
358
            permissions = {'read': [self.user]}
359

    
360
        # Update rest metadata
361
        meta = {}
362
        meta['properties'] = metadata.pop('properties', {})
363
        meta.update(name=name, status='available', **metadata)
364

    
365
        # Do the actualy update in the Pithos backend
366
        self._update_meta(image_url, meta)
367
        self._update_permissions(image_url, permissions)
368
        return self._get_image(image_url)
369

    
370
    def _list_images(self, user=None, filters=None, params=None):
371
        filters = filters or {}
372

    
373
        # TODO: Use filters
374
        # # Fix keys
375
        # keys = [PLANKTON_PREFIX + 'name']
376
        # size_range = (None, None)
377
        # for key, val in filters.items():
378
        #     if key == 'size_min':
379
        #         size_range = (val, size_range[1])
380
        #     elif key == 'size_max':
381
        #         size_range = (size_range[0], val)
382
        #     else:
383
        #         keys.append('%s = %s' % (PLANKTON_PREFIX + key, val))
384
        _images = self.backend.get_domain_objects(domain=PLANKTON_DOMAIN,
385
                                                  user=user)
386

    
387
        images = []
388
        for (location, meta, permissions) in _images:
389
            image_url = "pithos://" + location
390
            meta["modified"] = meta["version_timestamp"]
391
            # TODO: Create metadata when registering an Image
392
            meta["created"] = meta["version_timestamp"]
393
            images.append(image_to_dict(image_url, meta, permissions))
394

    
395
        if params is None:
396
            params = {}
397
        key = itemgetter(params.get('sort_key', 'created_at'))
398
        reverse = params.get('sort_dir', 'desc') == 'desc'
399
        images.sort(key=key, reverse=reverse)
400
        return images
401

    
402
    def list_images(self, filters=None, params=None):
403
        return self._list_images(user=self.user, filters=filters,
404
                                 params=params)
405

    
406
    def list_shared_images(self, member, filters=None, params=None):
407
        images = self._list_images(user=self.user, filters=filters,
408
                                   params=params)
409
        is_shared = lambda img: not img["is_public"] and img["owner"] == member
410
        return filter(is_shared, images)
411

    
412
    def list_public_images(self, filters=None, params=None):
413
        images = self._list_images(user=None, filters=filters, params=params)
414
        return filter(lambda img: img["is_public"], images)
415

    
416

    
417
class ImageBackendError(Exception):
418
    pass
419

    
420

    
421
class ImageNotFound(ImageBackendError):
422
    pass
423

    
424

    
425
class Forbidden(ImageBackendError):
426
    pass
427

    
428

    
429
def image_to_dict(image_url, meta, permissions):
430
    """Render an image to a dictionary"""
431
    account, container, name = split_url(image_url)
432

    
433
    image = {}
434
    if PLANKTON_PREFIX + 'name' not in meta:
435
        raise ImageNotFound("'%s' is not a Plankton image" % image_url)
436

    
437
    image["id"] = meta["uuid"]
438
    image["location"] = image_url
439
    image["checksum"] = meta["hash"]
440
    image["created_at"] = format_timestamp(meta["created"])
441
    deleted = meta.get("deleted", None)
442
    image["deleted_at"] = format_timestamp(deleted) if deleted else ""
443
    image["updated_at"] = format_timestamp(meta["modified"])
444
    image["size"] = meta["bytes"]
445
    image["store"] = "pithos"
446
    image['owner'] = account
447

    
448
    # Permissions
449
    image["is_public"] = "*" in permissions.get('read', [])
450

    
451
    for key, val in meta.items():
452
        # Get plankton properties
453
        if key.startswith(PLANKTON_PREFIX):
454
            # Remove plankton prefix
455
            key = key.replace(PLANKTON_PREFIX, "")
456
            # Keep only those in plankton meta
457
            if key in PLANKTON_META:
458
                if key == "properties":
459
                    val = json.loads(val)
460
                image[key] = val
461

    
462
    return image