Statistics
| Branch: | Tag: | Revision:

root / snf-cyclades-app / synnefo / plankton / backend.py @ 23808592

History | View | Annotate | Download (22.5 kB)

1
# Copyright 2011-2014 GRNET S.A. All rights reserved.
2

    
3
#
4
# Redistribution and use in source and binary forms, with or
5
# without modification, are permitted provided that the following
6
# conditions are met:
7
#
8
#   1. Redistributions of source code must retain the above
9
#      copyright notice, this list of conditions and the following
10
#      disclaimer.
11
#
12
#   2. Redistributions in binary form must reproduce the above
13
#      copyright notice, this list of conditions and the following
14
#      disclaimer in the documentation and/or other materials
15
#      provided with the distribution.
16
#
17
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
18
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
19
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
21
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
24
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
25
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
27
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28
# POSSIBILITY OF SUCH DAMAGE.
29
#
30
# The views and conclusions contained in the software and
31
# documentation are those of the authors and should not be
32
# interpreted as representing official policies, either expressed
33
# or implied, of GRNET S.A.
34

    
35
"""
36
The Plankton attributes are the following:
37
  - checksum: the 'hash' meta
38
  - container_format: stored as a user meta
39
  - created_at: the 'modified' meta of the first version
40
  - deleted_at: the timestamp of the last version
41
  - disk_format: stored as a user meta
42
  - id: the 'uuid' meta
43
  - is_public: True if there is a * entry for the read permission
44
  - location: generated based on the file's path
45
  - name: stored as a user meta
46
  - owner: the file's account
47
  - properties: stored as user meta prefixed with PROPERTY_PREFIX
48
  - size: the 'bytes' meta
49
  - status: stored as a system meta
50
  - store: is always 'pithos'
51
  - updated_at: the 'modified' meta
52
"""
53

    
54
import json
55
import logging
56
import os
57

    
58
from time import time, gmtime, strftime
59
from functools import wraps
60
from operator import itemgetter
61
from collections import namedtuple
62

    
63
from django.conf import settings
64
from django.utils import importlib
65
from pithos.backends.base import NotAllowedError, VersionNotExists, QuotaError
66
from synnefo.util.text import uenc
67
from copy import deepcopy
68
from snf_django.lib.api import faults
69

    
70
Location = namedtuple("ObjectLocation", ["account", "container", "path"])
71

    
72
logger = logging.getLogger(__name__)
73

    
74

    
75
PLANKTON_DOMAIN = 'plankton'
76
PLANKTON_PREFIX = 'plankton:'
77
PROPERTY_PREFIX = 'property:'
78

    
79
PLANKTON_META = ('container_format', 'disk_format', 'name',
80
                 'status', 'created_at')
81

    
82
MAX_META_KEY_LENGTH = 128 - len(PLANKTON_DOMAIN) - len(PROPERTY_PREFIX)
83
MAX_META_VALUE_LENGTH = 256
84

    
85
from pithos.backends.util import PithosBackendPool
86
_pithos_backend_pool = \
87
    PithosBackendPool(
88
        settings.PITHOS_BACKEND_POOL_SIZE,
89
        astakos_auth_url=settings.ASTAKOS_AUTH_URL,
90
        service_token=settings.CYCLADES_SERVICE_TOKEN,
91
        astakosclient_poolsize=settings.CYCLADES_ASTAKOSCLIENT_POOLSIZE,
92
        db_connection=settings.BACKEND_DB_CONNECTION,
93
        block_path=settings.BACKEND_BLOCK_PATH)
94

    
95

    
96
def get_pithos_backend():
97
    return _pithos_backend_pool.pool_get()
98

    
99

    
100
def format_timestamp(t):
101
    return strftime('%Y-%m-%d %H:%M:%S', gmtime(t))
102

    
103

    
104
def handle_pithos_backend(func):
105
    @wraps(func)
106
    def wrapper(self, *args, **kwargs):
107
        backend = self.backend
108
        backend.pre_exec()
109
        commit = False
110
        try:
111
            ret = func(self, *args, **kwargs)
112
        except NotAllowedError:
113
            raise faults.Forbidden
114
        except (NameError, VersionNotExists):
115
            raise faults.ItemNotFound
116
        except (AssertionError, ValueError):
117
            raise faults.BadRequest
118
        except QuotaError:
119
            raise faults.OverLimit
120
        else:
121
            commit = True
122
        finally:
123
            backend.post_exec(commit)
124
        return ret
125
    return wrapper
126

    
127

    
128
class PlanktonBackend(object):
129
    """A wrapper arround the pithos backend to simplify image handling."""
130

    
131
    def __init__(self, user):
132
        self.user = user
133
        self.backend = get_pithos_backend()
134

    
135
    def close(self):
136
        """Close PithosBackend(return to pool)"""
137
        self.backend.close()
138

    
139
    def __enter__(self):
140
        return self
141

    
142
    def __exit__(self, exc_type, exc_val, exc_tb):
143
        self.close()
144
        self.backend = None
145
        return False
146

    
147
    @handle_pithos_backend
148
    def get_image(self, uuid):
149
        return self._get_image(uuid)
150

    
151
    def _get_image(self, uuid):
152
        location, metadata = self._get_raw_metadata(uuid)
153
        permissions = self._get_raw_permissions(uuid, location)
154
        return image_to_dict(location, metadata, permissions)
155

    
156
    @handle_pithos_backend
157
    def add_property(self, uuid, key, value):
158
        location, _ = self._get_raw_metadata(uuid)
159
        properties = self._prefix_properties({key: value})
160
        self._update_metadata(uuid, location, properties, replace=False)
161

    
162
    @handle_pithos_backend
163
    def remove_property(self, uuid, key):
164
        location, _ = self._get_raw_metadata(uuid)
165
        # Use empty string to delete a property
166
        properties = self._prefix_properties({key: ""})
167
        self._update_metadata(uuid, location, properties, replace=False)
168

    
169
    @handle_pithos_backend
170
    def update_properties(self, uuid, properties):
171
        location, _ = self._get_raw_metadata(uuid)
172
        properties = self._prefix_properties(properties)
173
        self._update_metadata(uuid, location, properties, replace=False)
174

    
175
    @staticmethod
176
    def _prefix_properties(properties):
177
        """Add property prefix to properties."""
178
        return dict([(PROPERTY_PREFIX + k, v) for k, v in properties.items()])
179

    
180
    @staticmethod
181
    def _unprefix_properties(properties):
182
        """Remove property prefix from properties."""
183
        return dict([(k.replace(PROPERTY_PREFIX, "", 1), v)
184
                     for k, v in properties.items()])
185

    
186
    @staticmethod
187
    def _prefix_metadata(metadata):
188
        """Add plankton prefix to metadata."""
189
        return dict([(PLANKTON_PREFIX + k, v) for k, v in metadata.items()])
190

    
191
    @staticmethod
192
    def _unprefix_metadata(metadata):
193
        """Remove plankton prefix from metadata."""
194
        return dict([(k.replace(PLANKTON_PREFIX, "", 1), v)
195
                     for k, v in metadata.items()])
196

    
197
    @handle_pithos_backend
198
    def update_metadata(self, uuid, metadata):
199
        location, _ = self._get_raw_metadata(uuid)
200

    
201
        is_public = metadata.pop("is_public", None)
202
        if is_public is not None:
203
            self._set_public(uuid, location, public=is_public)
204

    
205
        # Each property is stored as a separate prefixed metadata
206
        meta = deepcopy(metadata)
207
        properties = meta.pop("properties", {})
208
        meta.update(self._prefix_properties(properties))
209

    
210
        self._update_metadata(uuid, location, metadata=meta, replace=False)
211

    
212
        return self._get_image(uuid)
213

    
214
    def _update_metadata(self, uuid, location, metadata, replace=False):
215
        _prefixed_metadata = self._prefix_metadata(metadata)
216
        prefixed = {}
217
        for k, v in _prefixed_metadata.items():
218
            # Encode to UTF-8
219
            k, v = uenc(k), uenc(v)
220
            # Check the length of key/value
221
            if len(k) > 128:
222
                raise faults.BadRequest('Metadata keys should be less than %s'
223
                                        ' characters' % MAX_META_KEY_LENGTH)
224
            if len(v) > 256:
225
                raise faults.BadRequest('Metadata values should be less than'
226
                                        ' %scharacters.'
227
                                        % MAX_META_VALUE_LENGTH)
228
            prefixed[k] = v
229

    
230
        account, container, path = location
231
        self.backend.update_object_meta(self.user, account, container, path,
232
                                        PLANKTON_DOMAIN, prefixed, replace)
233
        logger.debug("User '%s' updated image '%s', metadata: '%s'", self.user,
234
                     uuid, prefixed)
235

    
236
    def _get_raw_metadata(self, uuid, version=None, check_image=True):
237
        """Get info and metadata in Plankton doamin for the Pithos object.
238

239
        Return the location and the metadata of the Pithos object.
240
        If 'check_image' is set, check that the Pithos object is a registered
241
        Plankton Image.
242

243
        """
244
        # Convert uuid to location
245
        account, container, path = self.backend.get_uuid(self.user, uuid)
246
        try:
247
            meta = self.backend.get_object_meta(self.user, account, container,
248
                                                path, PLANKTON_DOMAIN, version)
249
            meta["deleted"] = False
250
        except NameError:
251
            if version is not None:
252
                raise
253
            versions = self.backend.list_versions(self.user, account,
254
                                                  container, path)
255
            assert(versions), ("Object without versions: %s/%s/%s" %
256
                               (account, container, path))
257
            # Object was deleted, use the latest version
258
            version, timestamp = versions[-1]
259
            meta = self.backend.get_object_meta(self.user, account, container,
260
                                                path, PLANKTON_DOMAIN, version)
261
            meta["deleted"] = True
262

    
263
        if check_image and PLANKTON_PREFIX + "name" not in meta:
264
            # Check that object is an image by checking if it has an Image name
265
            # in Plankton metadata
266
            raise faults.ItemNotFound("Image '%s' does not exist." % uuid)
267

    
268
        return Location(account, container, path), meta
269

    
270
    # Users and Permissions
271
    @handle_pithos_backend
272
    def add_user(self, uuid, user):
273
        assert(isinstance(user, basestring))
274
        location, _ = self._get_raw_metadata(uuid)
275
        permissions = self._get_raw_permissions(uuid, location)
276
        read = set(permissions.get("read", []))
277
        if not user in read:
278
            read.add(user)
279
            permissions["read"] = list(read)
280
            self._update_permissions(uuid, location, permissions)
281

    
282
    @handle_pithos_backend
283
    def remove_user(self, uuid, user):
284
        assert(isinstance(user, basestring))
285
        location, _ = self._get_raw_metadata(uuid)
286
        permissions = self._get_raw_permissions(uuid, location)
287
        read = set(permissions.get("read", []))
288
        if user in read:
289
            read.remove(user)
290
            permissions["read"] = list(read)
291
            self._update_permissions(uuid, location, permissions)
292

    
293
    @handle_pithos_backend
294
    def replace_users(self, uuid, users):
295
        assert(isinstance(users, list))
296
        location, _ = self._get_raw_metadata(uuid)
297
        permissions = self._get_raw_permissions(uuid, location)
298
        read = set(permissions.get("read", []))
299
        if "*" in read:  # Retain public permissions
300
            users.append("*")
301
        permissions["read"] = list(users)
302
        self._update_permissions(uuid, location, permissions)
303

    
304
    @handle_pithos_backend
305
    def list_users(self, uuid):
306
        location, _ = self._get_raw_metadata(uuid)
307
        permissions = self._get_raw_permissions(uuid, location)
308
        return [user for user in permissions.get('read', []) if user != '*']
309

    
310
    def _set_public(self, uuid, location, public):
311
        permissions = self._get_raw_permissions(uuid, location)
312
        assert(isinstance(public, bool))
313
        read = set(permissions.get("read", []))
314
        if public and "*" not in read:
315
            read.add("*")
316
        elif not public and "*" in read:
317
            read.discard("*")
318
        permissions["read"] = list(read)
319
        self._update_permissions(uuid, location, permissions)
320
        return permissions
321

    
322
    def _get_raw_permissions(self, uuid, location):
323
        account, container, path = location
324
        _a, path, permissions = \
325
            self.backend.get_object_permissions(self.user, account, container,
326
                                                path)
327

    
328
        if path is None and permissions != {}:
329
            raise Exception("Database Inconsistency Error:"
330
                            " Image '%s' got permissions from 'None' path." %
331
                            uuid)
332

    
333
        return permissions
334

    
335
    def _update_permissions(self, uuid, location, permissions):
336
        account, container, path = location
337
        self.backend.update_object_permissions(self.user, account, container,
338
                                               path, permissions)
339
        logger.debug("User '%s' updated image '%s' permissions: '%s'",
340
                     self.user, uuid, permissions)
341

    
342
    @handle_pithos_backend
343
    def register(self, name, image_url, metadata):
344
        # Validate that metadata are allowed
345
        if "id" in metadata:
346
            raise faults.BadRequest("Passing an ID is not supported")
347
        store = metadata.pop("store", "pithos")
348
        if store != "pithos":
349
            raise faults.BadRequest("Invalid store '%s'. Only 'pithos' store"
350
                                    " is supported" % store)
351
        disk_format = metadata.setdefault("disk_format",
352
                                          settings.DEFAULT_DISK_FORMAT)
353
        if disk_format not in settings.ALLOWED_DISK_FORMATS:
354
            raise faults.BadRequest("Invalid disk format '%s'" % disk_format)
355
        container_format =\
356
            metadata.setdefault("container_format",
357
                                settings.DEFAULT_CONTAINER_FORMAT)
358
        if container_format not in settings.ALLOWED_CONTAINER_FORMATS:
359
            raise faults.BadRequest("Invalid container format '%s'" %
360
                                    container_format)
361

    
362
        account, container, path = split_url(image_url)
363
        location = Location(account, container, path)
364
        meta = self.backend.get_object_meta(self.user, account, container,
365
                                            path, PLANKTON_DOMAIN, None)
366
        uuid = meta["uuid"]
367

    
368
        # Validate that 'size' and 'checksum'
369
        size = metadata.pop('size', int(meta['bytes']))
370
        if not isinstance(size, int) or int(size) != int(meta["bytes"]):
371
            raise faults.BadRequest("Invalid 'size' field")
372

    
373
        checksum = metadata.pop('checksum', meta['hash'])
374
        if not isinstance(checksum, basestring) or checksum != meta['hash']:
375
            raise faults.BadRequest("Invalid checksum field")
376

    
377
        users = [self.user]
378
        public = metadata.pop("is_public", False)
379
        if not isinstance(public, bool):
380
            raise faults.BadRequest("Invalid value for 'is_public' metadata")
381
        if public:
382
            users.append("*")
383
        permissions = {'read': users}
384
        self._update_permissions(uuid, location, permissions)
385

    
386
        # Each property is stored as a separate prefixed metadata
387
        meta = deepcopy(metadata)
388
        properties = meta.pop("properties", {})
389
        meta.update(self._prefix_properties(properties))
390
        # Add extra metadata
391
        meta["name"] = name
392
        meta["status"] = "AVAILABLE"
393
        meta['created_at'] = str(time())
394
        #meta["is_snapshot"] = False
395
        self._update_metadata(uuid, location, metadata=meta, replace=False)
396

    
397
        logger.debug("User '%s' registered image '%s'('%s')", self.user,
398
                     uuid, location)
399
        return self._get_image(uuid)
400

    
401
    @handle_pithos_backend
402
    def unregister(self, uuid):
403
        """Unregister an Image.
404

405
        Unregister an Image by removing all the metadata in the Plankton
406
        domain. The Pithos file is not deleted.
407

408
        """
409
        location, _ = self._get_raw_metadata(uuid)
410
        self._update_metadata(uuid, location, metadata={}, replace=True)
411
        logger.debug("User '%s' unregistered image '%s'", self.user, uuid)
412

    
413
    # List functions
414
    def _list_images(self, user=None, filters=None, params=None):
415
        filters = filters or {}
416

    
417
        # TODO: Use filters
418
        # # Fix keys
419
        # keys = [PLANKTON_PREFIX + 'name']
420
        # size_range = (None, None)
421
        # for key, val in filters.items():
422
        #     if key == 'size_min':
423
        #         size_range = (val, size_range[1])
424
        #     elif key == 'size_max':
425
        #         size_range = (size_range[0], val)
426
        #     else:
427
        #         keys.append('%s = %s' % (PLANKTON_PREFIX + key, val))
428
        _images = self.backend.get_domain_objects(domain=PLANKTON_DOMAIN,
429
                                                  user=user)
430

    
431
        images = []
432
        for (location, metadata, permissions) in _images:
433
            location = Location(*location.split("/", 2))
434
            images.append(image_to_dict(location, metadata, permissions))
435

    
436
        if params is None:
437
            params = {}
438

    
439
        key = itemgetter(params.get('sort_key', 'created_at'))
440
        reverse = params.get('sort_dir', 'desc') == 'desc'
441
        images.sort(key=key, reverse=reverse)
442
        return images
443

    
444
    @handle_pithos_backend
445
    def list_images(self, filters=None, params=None):
446
        return self._list_images(user=self.user, filters=filters,
447
                                 params=params)
448

    
449
    @handle_pithos_backend
450
    def list_shared_images(self, member, filters=None, params=None):
451
        images = self._list_images(user=self.user, filters=filters,
452
                                   params=params)
453
        is_shared = lambda img: not img["is_public"] and img["owner"] == member
454
        return filter(is_shared, images)
455

    
456
    @handle_pithos_backend
457
    def list_public_images(self, filters=None, params=None):
458
        images = self._list_images(user=None, filters=filters, params=params)
459
        return filter(lambda img: img["is_public"], images)
460

    
461
    # # Snapshots
462
    # def list_snapshots(self, user=None):
463
    #     _snapshots = self.list_images()
464
    #     return [s for s in _snapshots if s["is_snapshot"]]
465

    
466
    # @handle_pithos_backend
467
    # def get_snapshot(self, user, snapshot_uuid):
468
    #     snap = self._get_image(snapshot_uuid)
469
    #     if snap.get("is_snapshot", False) is False:
470
    #         raise faults.ItemNotFound("Snapshots '%s' does not exist" %
471
    #                                   snapshot_uuid)
472
    #     return snap
473

    
474
    # @handle_pithos_backend
475
    # def delete_snapshot(self, snapshot_uuid):
476
    #     self.backend.delete_object_for_uuid(self.user, snapshot_uuid)
477

    
478
    # @handle_pithos_backend
479
    # def update_status(self, image_uuid, status):
480
    #     """Update status of snapshot"""
481
    #     location, _ = self._get_raw_metadata(image_uuid)
482
    #     properties = {"status": status.upper()}
483
    #     self._update_metadata(image_uuid, location, properties,
484
    #     replace=False)
485
    #     return self._get_image(image_uuid)
486

    
487

    
488
def create_url(account, container, name):
489
    """Create a Pithos URL from the object info"""
490
    assert "/" not in account, "Invalid account"
491
    assert "/" not in container, "Invalid container"
492
    return "pithos://%s/%s/%s" % (account, container, name)
493

    
494

    
495
def split_url(url):
496
    """Get object info from the Pithos URL"""
497
    assert(isinstance(url, basestring))
498
    t = url.split('/', 4)
499
    assert t[0] == "pithos:", "Invalid url"
500
    assert len(t) == 5, "Invalid url"
501
    return t[2:5]
502

    
503

    
504
def image_to_dict(location, metadata, permissions):
505
    """Render an image to a dictionary"""
506
    account, container, name = location
507

    
508
    image = {}
509
    image["id"] = metadata["uuid"]
510
    image["mapfile"] = metadata["hash"]
511
    image["checksum"] = metadata["hash"]
512
    image["location"] = create_url(account, container, name)
513
    image["size"] = metadata["bytes"]
514
    image['owner'] = account
515
    image["store"] = u"pithos"
516
    #image["is_snapshot"] = metadata.pop(PLANKTON_PREFIX + "is_snapshot",
517
    #False)
518
    # Permissions
519
    users = list(permissions.get("read", []))
520
    image["is_public"] = "*" in users
521
    image["users"] = [u for u in users if u != "*"]
522
    # Timestamps
523
    updated_at = metadata["version_timestamp"]
524
    created_at = metadata.get("created_at", updated_at)
525
    image["created_at"] = format_timestamp(created_at)
526
    image["updated_at"] = format_timestamp(updated_at)
527
    if metadata.get("deleted", False):
528
        image["deleted_at"] = image["updated_at"]
529
    else:
530
        image["deleted_at"] = ""
531

    
532
    properties = {}
533
    for key, val in metadata.items():
534
        # Get plankton properties
535
        if key.startswith(PLANKTON_PREFIX):
536
            # Remove plankton prefix
537
            key = key.replace(PLANKTON_PREFIX, "")
538
            # Keep only those in plankton metadata
539
            if key in PLANKTON_META:
540
                if key != "created_at":
541
                    # created timestamp is return in 'created_at' field
542
                    image[key] = val
543
            elif key.startswith(PROPERTY_PREFIX):
544
                key = key.replace(PROPERTY_PREFIX, "")
545
                properties[key] = val
546
    image["properties"] = properties
547

    
548
    return image
549

    
550

    
551
class JSONFileBackend(object):
552
    """
553
    A dummy image backend that loads available images from a file with json
554
    formatted content.
555

556
    usage:
557
        PLANKTON_BACKEND_MODULE = 'synnefo.plankton.backend.JSONFileBackend'
558
        PLANKTON_IMAGES_JSON_BACKEND_FILE = '/tmp/images.json'
559

560
        # loading images from an existing plankton service
561
        $ curl -H "X-Auth-Token: <MYTOKEN>" \
562
                https://cyclades.synnefo.org/plankton/images/detail | \
563
                python -m json.tool > /tmp/images.json
564
    """
565
    def __init__(self, userid):
566
        self.images_file = getattr(settings,
567
                                   'PLANKTON_IMAGES_JSON_BACKEND_FILE', '')
568
        if not os.path.exists(self.images_file):
569
            raise Exception("Invalid plankgon images json backend file: %s",
570
                            self.images_file)
571
        fp = file(self.images_file)
572
        self.images = json.load(fp)
573
        fp.close()
574

    
575
    def iter(self, *args, **kwargs):
576
        return self.images.__iter__()
577

    
578
    def list_images(self, *args, **kwargs):
579
        return self.images
580

    
581
    def get_image(self, image_uuid):
582
        try:
583
            return filter(lambda i: i['id'] == image_uuid, self.images)[0]
584
        except IndexError:
585
            raise Exception("Unknown image uuid: %s" % image_uuid)
586

    
587
    def close(self):
588
        pass
589

    
590

    
591
def get_backend():
592
    backend_module = getattr(settings, 'PLANKTON_BACKEND_MODULE', None)
593
    if not backend_module:
594
        # no setting set
595
        return PlanktonBackend
596

    
597
    parts = backend_module.split(".")
598
    module = ".".join(parts[:-1])
599
    cls = parts[-1]
600
    try:
601
        return getattr(importlib.import_module(module), cls)
602
    except (ImportError, AttributeError), e:
603
        raise ImportError("Cannot import plankton module: %s (%s)" %
604
                          (backend_module, e.message))