Statistics
| Branch: | Tag: | Revision:

root / snf-cyclades-app / synnefo / plankton / backend.py @ 0efb43cd

History | View | Annotate | Download (21.1 kB)

1
# Copyright 2011-2013 GRNET S.A. All rights reserved.
2

    
3
#
4
# Redistribution and use in source and binary forms, with or
5
# without modification, are permitted provided that the following
6
# conditions are met:
7
#
8
#   1. Redistributions of source code must retain the above
9
#      copyright notice, this list of conditions and the following
10
#      disclaimer.
11
#
12
#   2. Redistributions in binary form must reproduce the above
13
#      copyright notice, this list of conditions and the following
14
#      disclaimer in the documentation and/or other materials
15
#      provided with the distribution.
16
#
17
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
18
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
19
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
21
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
24
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
25
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
27
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28
# POSSIBILITY OF SUCH DAMAGE.
29
#
30
# The views and conclusions contained in the software and
31
# documentation are those of the authors and should not be
32
# interpreted as representing official policies, either expressed
33
# or implied, of GRNET S.A.
34

    
35
"""
36
The Plankton attributes are the following:
37
  - checksum: the 'hash' meta
38
  - container_format: stored as a user meta
39
  - created_at: the 'modified' meta of the first version
40
  - deleted_at: the timestamp of the last version
41
  - disk_format: stored as a user meta
42
  - id: the 'uuid' meta
43
  - is_public: True if there is a * entry for the read permission
44
  - location: generated based on the file's path
45
  - name: stored as a user meta
46
  - owner: the file's account
47
  - properties: stored as user meta prefixed with PROPERTY_PREFIX
48
  - size: the 'bytes' meta
49
  - status: stored as a system meta
50
  - store: is always 'pithos'
51
  - updated_at: the 'modified' meta
52
"""
53

    
54
import json
55
import warnings
56
import logging
57
import os
58

    
59
from time import time, gmtime, strftime
60
from functools import wraps
61
from operator import itemgetter
62

    
63
from django.conf import settings
64
from django.utils import importlib
65
from pithos.backends.base import NotAllowedError, VersionNotExists
66

    
67
logger = logging.getLogger(__name__)
68

    
69

    
70
PLANKTON_DOMAIN = 'plankton'
71
PLANKTON_PREFIX = 'plankton:'
72
PROPERTY_PREFIX = 'property:'
73

    
74
PLANKTON_META = ('container_format', 'disk_format', 'name',
75
                 'status', 'created_at')
76

    
77
MAX_META_KEY_LENGTH = 128 - len(PLANKTON_DOMAIN) - len(PROPERTY_PREFIX)
78
MAX_META_VALUE_LENGTH = 256
79

    
80
from pithos.backends.util import PithosBackendPool
81
_pithos_backend_pool = \
82
    PithosBackendPool(
83
        settings.PITHOS_BACKEND_POOL_SIZE,
84
        astakos_auth_url=settings.ASTAKOS_AUTH_URL,
85
        service_token=settings.CYCLADES_SERVICE_TOKEN,
86
        astakosclient_poolsize=settings.CYCLADES_ASTAKOSCLIENT_POOLSIZE,
87
        db_connection=settings.BACKEND_DB_CONNECTION,
88
        block_path=settings.BACKEND_BLOCK_PATH)
89

    
90

    
91
def get_pithos_backend():
92
    return _pithos_backend_pool.pool_get()
93

    
94

    
95
def create_url(account, container, name):
96
    assert "/" not in account, "Invalid account"
97
    assert "/" not in container, "Invalid container"
98
    return "pithos://%s/%s/%s" % (account, container, name)
99

    
100

    
101
def split_url(url):
102
    """Returns (accout, container, object) from a url string"""
103
    t = url.split('/', 4)
104
    assert t[0] == "pithos:", "Invalid url"
105
    assert len(t) == 5, "Invalid url"
106
    return t[2:5]
107

    
108

    
109
def format_timestamp(t):
110
    return strftime('%Y-%m-%d %H:%M:%S', gmtime(t))
111

    
112

    
113
def handle_backend_exceptions(func):
114
    @wraps(func)
115
    def wrapper(*args, **kwargs):
116
        try:
117
            return func(*args, **kwargs)
118
        except NotAllowedError:
119
            raise Forbidden
120
        except NameError:
121
            raise ImageNotFound
122
        except VersionNotExists:
123
            raise ImageNotFound
124
    return wrapper
125

    
126

    
127
def commit_on_success(func):
128
    def wrapper(self, *args, **kwargs):
129
        backend = self.backend
130
        backend.pre_exec()
131
        try:
132
            ret = func(self, *args, **kwargs)
133
        except:
134
            backend.post_exec(False)
135
            raise
136
        else:
137
            backend.post_exec(True)
138
        return ret
139
    return wrapper
140

    
141

    
142
class ImageBackend(object):
143
    """A wrapper arround the pithos backend to simplify image handling."""
144

    
145
    def __init__(self, user):
146
        self.user = user
147

    
148
        original_filters = warnings.filters
149
        warnings.simplefilter('ignore')         # Suppress SQLAlchemy warnings
150
        self.backend = get_pithos_backend()
151
        warnings.filters = original_filters     # Restore warnings
152

    
153
    def close(self):
154
        """Close PithosBackend(return to pool)"""
155
        self.backend.close()
156

    
157
    @handle_backend_exceptions
158
    @commit_on_success
159
    def get_image(self, image_uuid):
160
        """Retrieve information about an image."""
161
        image_url = self._get_image_url(image_uuid)
162
        return self._get_image(image_url)
163

    
164
    def _get_image_url(self, image_uuid):
165
        """Get the Pithos url that corresponds to an image UUID."""
166
        account, container, name = self.backend.get_uuid(self.user, image_uuid)
167
        return create_url(account, container, name)
168

    
169
    def _get_image(self, image_url):
170
        """Get information about an Image.
171

172
        Get all available information about an Image.
173
        """
174
        account, container, name = split_url(image_url)
175
        try:
176
            meta = self._get_meta(image_url)
177
            meta["deleted"] = ""
178
        except NameError:
179
            versions = self.backend.list_versions(self.user, account,
180
                                                  container, name)
181
            if not versions:
182
                raise Exception("Image without versions %s" % image_url)
183
            # Object was deleted, use the latest version
184
            version, timestamp = versions[-1]
185
            meta = self._get_meta(image_url, version)
186
            meta["deleted"] = timestamp
187

    
188
        # XXX: Check that an object is a plankton image! PithosBackend will
189
        # return common metadata for an object, even if it has no metadata in
190
        # plankton domain. All images must have a name, so we check if a file
191
        # is an image by checking if they are having an image name.
192
        if PLANKTON_PREFIX + 'name' not in meta:
193
            raise ImageNotFound
194

    
195
        permissions = self._get_permissions(image_url)
196
        return image_to_dict(image_url, meta, permissions)
197

    
198
    def _get_meta(self, image_url, version=None):
199
        """Get object's metadata."""
200
        account, container, name = split_url(image_url)
201
        return self.backend.get_object_meta(self.user, account, container,
202
                                            name, PLANKTON_DOMAIN, version)
203

    
204
    def _update_meta(self, image_url, meta, replace=False):
205
        """Update object's metadata."""
206
        account, container, name = split_url(image_url)
207

    
208
        prefixed = [(PLANKTON_PREFIX + k, v) for k, v in meta.items()
209
                    if k in PLANKTON_META or k.startswith(PROPERTY_PREFIX)]
210
        prefixed = dict(prefixed)
211

    
212
        for k, v in prefixed.items():
213
            if len(k) > 128:
214
                raise InvalidMetadata('Metadata keys should be less than %s '
215
                                      'characters' % MAX_META_KEY_LENGTH)
216
            if len(v) > 256:
217
                raise InvalidMetadata('Metadata values should be less than %s '
218
                                      'characters.' % MAX_META_VALUE_LENGTH)
219

    
220
        self.backend.update_object_meta(self.user, account, container, name,
221
                                        PLANKTON_DOMAIN, prefixed, replace)
222
        logger.debug("User '%s' updated image '%s', meta: '%s'", self.user,
223
                     image_url, prefixed)
224

    
225
    def _get_permissions(self, image_url):
226
        """Get object's permissions."""
227
        account, container, name = split_url(image_url)
228
        _a, path, permissions = \
229
            self.backend.get_object_permissions(self.user, account, container,
230
                                                name)
231

    
232
        if path is None and permissions != {}:
233
            logger.warning("Image '%s' got permissions '%s' from 'None' path.",
234
                           image_url, permissions)
235
            raise Exception("Database Inconsistency Error:"
236
                            " Image '%s' got permissions from 'None' path." %
237
                            image_url)
238

    
239
        return permissions
240

    
241
    def _update_permissions(self, image_url, permissions):
242
        """Update object's permissions."""
243
        account, container, name = split_url(image_url)
244
        self.backend.update_object_permissions(self.user, account, container,
245
                                               name, permissions)
246
        logger.debug("User '%s' updated image '%s', permissions: '%s'",
247
                     self.user, image_url, permissions)
248

    
249
    @handle_backend_exceptions
250
    @commit_on_success
251
    def unregister(self, image_uuid):
252
        """Unregister an image.
253

254
        Unregister an image, by removing all metadata from the Pithos
255
        file that exist in the PLANKTON_DOMAIN.
256

257
        """
258
        image_url = self._get_image_url(image_uuid)
259
        self._get_image(image_url)  # Assert that it is an image
260
        # Unregister the image by removing all metadata from domain
261
        # 'PLANKTON_DOMAIN'
262
        meta = {}
263
        self._update_meta(image_url, meta, True)
264
        logger.debug("User '%s' deleted image '%s'", self.user, image_url)
265

    
266
    @handle_backend_exceptions
267
    @commit_on_success
268
    def add_user(self, image_uuid, add_user):
269
        """Add a user as an image member.
270

271
        Update read permissions of Pithos file, to include the specified user.
272

273
        """
274
        image_url = self._get_image_url(image_uuid)
275
        self._get_image(image_url)  # Assert that it is an image
276
        permissions = self._get_permissions(image_url)
277
        read = set(permissions.get("read", []))
278
        assert(isinstance(add_user, (str, unicode)))
279
        read.add(add_user)
280
        permissions["read"] = list(read)
281
        self._update_permissions(image_url, permissions)
282

    
283
    @handle_backend_exceptions
284
    @commit_on_success
285
    def remove_user(self, image_uuid, remove_user):
286
        """Remove the user from image members.
287

288
        Remove the specified user from the read permissions of the Pithos file.
289

290
        """
291
        image_url = self._get_image_url(image_uuid)
292
        self._get_image(image_url)  # Assert that it is an image
293
        permissions = self._get_permissions(image_url)
294
        read = set(permissions.get("read", []))
295
        assert(isinstance(remove_user, (str, unicode)))
296
        try:
297
            read.remove(remove_user)
298
        except ValueError:
299
            return  # TODO: User did not have access
300
        permissions["read"] = list(read)
301
        self._update_permissions(image_url, permissions)
302

    
303
    @handle_backend_exceptions
304
    @commit_on_success
305
    def replace_users(self, image_uuid, replace_users):
306
        """Replace image members.
307

308
        Replace the read permissions of the Pithos files with the specified
309
        users. If image is specified as public, we must preserve * permission.
310

311
        """
312
        image_url = self._get_image_url(image_uuid)
313
        image = self._get_image(image_url)
314
        permissions = self._get_permissions(image_url)
315
        assert(isinstance(replace_users, list))
316
        permissions["read"] = replace_users
317
        if image.get("is_public", False):
318
            permissions["read"].append("*")
319
        self._update_permissions(image_url, permissions)
320

    
321
    @handle_backend_exceptions
322
    @commit_on_success
323
    def list_users(self, image_uuid):
324
        """List the image members.
325

326
        List the image members, by listing all users that have read permission
327
        to the corresponding Pithos file.
328

329
        """
330
        image_url = self._get_image_url(image_uuid)
331
        self._get_image(image_url)  # Assert that it is an image
332
        permissions = self._get_permissions(image_url)
333
        return [user for user in permissions.get('read', []) if user != '*']
334

    
335
    @handle_backend_exceptions
336
    @commit_on_success
337
    def update_metadata(self, image_uuid, metadata):
338
        """Update Image metadata."""
339
        image_url = self._get_image_url(image_uuid)
340
        self._get_image(image_url)  # Assert that it is an image
341

    
342
        # 'is_public' metadata is translated in proper file permissions
343
        is_public = metadata.pop("is_public", None)
344
        if is_public is not None:
345
            permissions = self._get_permissions(image_url)
346
            read = set(permissions.get("read", []))
347
            if is_public:
348
                read.add("*")
349
            else:
350
                read.discard("*")
351
            permissions["read"] = list(read)
352
            self._update_permissions(image_url, permissions)
353

    
354
        # Extract the properties dictionary from metadata, and store each
355
        # property as a separeted, prefixed metadata
356
        properties = metadata.pop("properties", {})
357
        meta = dict([(PROPERTY_PREFIX + k, v) for k, v in properties.items()])
358
        # Also add the following metadata
359
        meta.update(**metadata)
360

    
361
        self._update_meta(image_url, meta)
362
        image_url = self._get_image_url(image_uuid)
363
        return self._get_image(image_url)
364

    
365
    @handle_backend_exceptions
366
    @commit_on_success
367
    def register(self, name, image_url, metadata):
368
        # Validate that metadata are allowed
369
        if "id" in metadata:
370
            raise ValueError("Passing an ID is not supported")
371
        store = metadata.pop("store", "pithos")
372
        if store != "pithos":
373
            raise ValueError("Invalid store '%s'. Only 'pithos' store is"
374
                             "supported" % store)
375
        disk_format = metadata.setdefault("disk_format",
376
                                          settings.DEFAULT_DISK_FORMAT)
377
        if disk_format not in settings.ALLOWED_DISK_FORMATS:
378
            raise ValueError("Invalid disk format '%s'" % disk_format)
379
        container_format =\
380
            metadata.setdefault("container_format",
381
                                settings.DEFAULT_CONTAINER_FORMAT)
382
        if container_format not in settings.ALLOWED_CONTAINER_FORMATS:
383
            raise ValueError("Invalid container format '%s'" %
384
                             container_format)
385

    
386
        # Validate that 'size' and 'checksum' are valid
387
        account, container, object = split_url(image_url)
388

    
389
        meta = self._get_meta(image_url)
390

    
391
        size = int(metadata.pop('size', meta['bytes']))
392
        if size != meta['bytes']:
393
            raise ValueError("Invalid size")
394

    
395
        checksum = metadata.pop('checksum', meta['hash'])
396
        if checksum != meta['hash']:
397
            raise ValueError("Invalid checksum")
398

    
399
        # Fix permissions
400
        is_public = metadata.pop('is_public', False)
401
        if is_public:
402
            permissions = {'read': ['*']}
403
        else:
404
            permissions = {'read': [self.user]}
405

    
406
        # Extract the properties dictionary from metadata, and store each
407
        # property as a separeted, prefixed metadata
408
        properties = metadata.pop("properties", {})
409
        meta = dict([(PROPERTY_PREFIX + k, v) for k, v in properties.items()])
410
        # Add creation(register) timestamp as a metadata, to avoid extra
411
        # queries when retrieving the list of images.
412
        meta['created_at'] = time()
413
        # Update rest metadata
414
        meta.update(name=name, status='available', **metadata)
415

    
416
        # Do the actualy update in the Pithos backend
417
        self._update_meta(image_url, meta)
418
        self._update_permissions(image_url, permissions)
419
        logger.debug("User '%s' created image '%s'('%s')", self.user,
420
                     image_url, name)
421
        return self._get_image(image_url)
422

    
423
    def _list_images(self, user=None, filters=None, params=None):
424
        filters = filters or {}
425

    
426
        # TODO: Use filters
427
        # # Fix keys
428
        # keys = [PLANKTON_PREFIX + 'name']
429
        # size_range = (None, None)
430
        # for key, val in filters.items():
431
        #     if key == 'size_min':
432
        #         size_range = (val, size_range[1])
433
        #     elif key == 'size_max':
434
        #         size_range = (size_range[0], val)
435
        #     else:
436
        #         keys.append('%s = %s' % (PLANKTON_PREFIX + key, val))
437
        _images = self.backend.get_domain_objects(domain=PLANKTON_DOMAIN,
438
                                                  user=user)
439

    
440
        images = []
441
        for (location, meta, permissions) in _images:
442
            image_url = "pithos://" + location
443
            meta["modified"] = meta["version_timestamp"]
444
            images.append(image_to_dict(image_url, meta, permissions))
445

    
446
        if params is None:
447
            params = {}
448
        key = itemgetter(params.get('sort_key', 'created_at'))
449
        reverse = params.get('sort_dir', 'desc') == 'desc'
450
        images.sort(key=key, reverse=reverse)
451
        return images
452

    
453
    @commit_on_success
454
    def list_images(self, filters=None, params=None):
455
        return self._list_images(user=self.user, filters=filters,
456
                                 params=params)
457

    
458
    @commit_on_success
459
    def list_shared_images(self, member, filters=None, params=None):
460
        images = self._list_images(user=self.user, filters=filters,
461
                                   params=params)
462
        is_shared = lambda img: not img["is_public"] and img["owner"] == member
463
        return filter(is_shared, images)
464

    
465
    @commit_on_success
466
    def list_public_images(self, filters=None, params=None):
467
        images = self._list_images(user=None, filters=filters, params=params)
468
        return filter(lambda img: img["is_public"], images)
469

    
470

    
471
class ImageBackendError(Exception):
472
    pass
473

    
474

    
475
class ImageNotFound(ImageBackendError):
476
    pass
477

    
478

    
479
class Forbidden(ImageBackendError):
480
    pass
481

    
482

    
483
class InvalidMetadata(ImageBackendError):
484
    pass
485

    
486

    
487
def image_to_dict(image_url, meta, permissions):
488
    """Render an image to a dictionary"""
489
    account, container, name = split_url(image_url)
490

    
491
    image = {}
492
    if PLANKTON_PREFIX + 'name' not in meta:
493
        logger.warning("Image without Plankton name!! url %s meta %s",
494
                       image_url, meta)
495
        image[PLANKTON_PREFIX + "name"] = ""
496

    
497
    image["id"] = meta["uuid"]
498
    image["location"] = image_url
499
    image["checksum"] = meta["hash"]
500
    created = meta.get("created_at", meta["modified"])
501
    image["created_at"] = format_timestamp(created)
502
    deleted = meta.get("deleted", None)
503
    image["deleted_at"] = format_timestamp(deleted) if deleted else ""
504
    image["updated_at"] = format_timestamp(meta["modified"])
505
    image["size"] = meta["bytes"]
506
    image["store"] = "pithos"
507
    image['owner'] = account
508

    
509
    # Permissions
510
    image["is_public"] = "*" in permissions.get('read', [])
511

    
512
    properties = {}
513
    for key, val in meta.items():
514
        # Get plankton properties
515
        if key.startswith(PLANKTON_PREFIX):
516
            # Remove plankton prefix
517
            key = key.replace(PLANKTON_PREFIX, "")
518
            # Keep only those in plankton meta
519
            if key in PLANKTON_META:
520
                if key != "created_at":
521
                    # created timestamp is return in 'created_at' field
522
                    image[key] = val
523
            elif key.startswith(PROPERTY_PREFIX):
524
                key = key.replace(PROPERTY_PREFIX, "")
525
                properties[key] = val
526
    image["properties"] = properties
527

    
528
    return image
529

    
530

    
531
class JSONFileBackend(object):
532
    """
533
    A dummy image backend that loads available images from a file with json
534
    formatted content.
535

536
    usage:
537
        PLANKTON_BACKEND_MODULE = 'synnefo.plankton.backend.JSONFileBackend'
538
        PLANKTON_IMAGES_JSON_BACKEND_FILE = '/tmp/images.json'
539

540
        # loading images from an existing plankton service
541
        $ curl -H "X-Auth-Token: <MYTOKEN>" \
542
                https://cyclades.synnefo.org/plankton/images/detail | \
543
                python -m json.tool > /tmp/images.json
544
    """
545
    def __init__(self, userid):
546
        self.images_file = getattr(settings,
547
                                   'PLANKTON_IMAGES_JSON_BACKEND_FILE', '')
548
        if not os.path.exists(self.images_file):
549
            raise Exception("Invalid plankgon images json backend file: %s",
550
                            self.images_file)
551
        fp = file(self.images_file)
552
        self.images = json.load(fp)
553
        fp.close()
554

    
555
    def iter(self, *args, **kwargs):
556
        return self.images.__iter__()
557

    
558
    def list_images(self, *args, **kwargs):
559
        return self.images
560

    
561
    def get_image(self, image_uuid):
562
        try:
563
            return filter(lambda i: i['id'] == image_uuid, self.images)[0]
564
        except IndexError:
565
            raise Exception("Unknown image uuid: %s" % image_uuid)
566

    
567
    def close(self):
568
        pass
569

    
570

    
571
def get_backend():
572
    backend_module = getattr(settings, 'PLANKTON_BACKEND_MODULE', None)
573
    if not backend_module:
574
        # no setting set
575
        return ImageBackend
576

    
577
    parts = backend_module.split(".")
578
    module = ".".join(parts[:-1])
579
    cls = parts[-1]
580
    try:
581
        return getattr(importlib.import_module(module), cls)
582
    except (ImportError, AttributeError), e:
583
        raise ImportError("Cannot import plankton module: %s (%s)" %
584
                          (backend_module, e.message))