Statistics
| Branch: | Tag: | Revision:

root / snf-cyclades-app / synnefo / plankton / backend.py @ 62d3ea53

History | View | Annotate | Download (16.9 kB)

1
# Copyright 2011-2013 GRNET S.A. All rights reserved.
2

    
3
#
4
# Redistribution and use in source and binary forms, with or
5
# without modification, are permitted provided that the following
6
# conditions are met:
7
#
8
#   1. Redistributions of source code must retain the above
9
#      copyright notice, this list of conditions and the following
10
#      disclaimer.
11
#
12
#   2. Redistributions in binary form must reproduce the above
13
#      copyright notice, this list of conditions and the following
14
#      disclaimer in the documentation and/or other materials
15
#      provided with the distribution.
16
#
17
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
18
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
19
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
21
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
24
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
25
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
27
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28
# POSSIBILITY OF SUCH DAMAGE.
29
#
30
# The views and conclusions contained in the software and
31
# documentation are those of the authors and should not be
32
# interpreted as representing official policies, either expressed
33
# or implied, of GRNET S.A.
34

    
35
"""
36
The Plankton attributes are the following:
37
  - checksum: the 'hash' meta
38
  - container_format: stored as a user meta
39
  - created_at: the 'modified' meta of the first version
40
  - deleted_at: the timestamp of the last version
41
  - disk_format: stored as a user meta
42
  - id: the 'uuid' meta
43
  - is_public: True if there is a * entry for the read permission
44
  - location: generated based on the file's path
45
  - name: stored as a user meta
46
  - owner: the file's account
47
  - properties: stored as user meta prefixed with PROPERTY_PREFIX
48
  - size: the 'bytes' meta
49
  - status: stored as a system meta
50
  - store: is always 'pithos'
51
  - updated_at: the 'modified' meta
52
"""
53

    
54
import json
55
import warnings
56
import logging
57
from time import gmtime, strftime
58
from functools import wraps
59
from operator import itemgetter
60

    
61
from django.conf import settings
62
from pithos.backends.base import NotAllowedError, VersionNotExists
63

    
64
logger = logging.getLogger(__name__)
65

    
66

    
67
PLANKTON_DOMAIN = 'plankton'
68
PLANKTON_PREFIX = 'plankton:'
69
PROPERTY_PREFIX = 'property:'
70

    
71
PLANKTON_META = ('container_format', 'disk_format', 'name', 'properties',
72
                 'status')
73

    
74
from pithos.backends.util import PithosBackendPool
75
POOL_SIZE = 8
76
_pithos_backend_pool = \
77
    PithosBackendPool(
78
        POOL_SIZE,
79
        quotaholder_enabled=settings.CYCLADES_USE_QUOTAHOLDER,
80
        quotaholder_url=settings.CYCLADES_QUOTAHOLDER_URL,
81
        quotaholder_token=settings.CYCLADES_QUOTAHOLDER_TOKEN,
82
        quotaholder_client_poolsize=settings.CYCLADES_QUOTAHOLDER_POOLSIZE,
83
        db_connection=settings.BACKEND_DB_CONNECTION,
84
        block_path=settings.BACKEND_BLOCK_PATH)
85

    
86

    
87
def get_pithos_backend():
88
    return _pithos_backend_pool.pool_get()
89

    
90

    
91
def create_url(account, container, name):
92
    assert "/" not in account, "Invalid account"
93
    assert "/" not in container, "Invalid container"
94
    return "pithos://%s/%s/%s" % (account, container, name)
95

    
96

    
97
def split_url(url):
98
    """Returns (accout, container, object) from a url string"""
99
    t = url.split('/', 4)
100
    assert t[0] == "pithos:", "Invalid url"
101
    assert len(t) == 5, "Invalid url"
102
    return t[2:5]
103

    
104

    
105
def format_timestamp(t):
106
    return strftime('%Y-%m-%d %H:%M:%S', gmtime(t))
107

    
108

    
109
def handle_backend_exceptions(func):
110
    @wraps(func)
111
    def wrapper(*args, **kwargs):
112
        try:
113
            return func(*args, **kwargs)
114
        except NotAllowedError:
115
            raise Forbidden
116
        except NameError:
117
            raise ImageNotFound
118
        except VersionNotExists:
119
            raise ImageNotFound
120
    return wrapper
121

    
122

    
123
class ImageBackend(object):
124
    """A wrapper arround the pithos backend to simplify image handling."""
125

    
126
    def __init__(self, user):
127
        self.user = user
128

    
129
        original_filters = warnings.filters
130
        warnings.simplefilter('ignore')         # Suppress SQLAlchemy warnings
131
        self.backend = get_pithos_backend()
132
        warnings.filters = original_filters     # Restore warnings
133

    
134
    def close(self):
135
        """Close PithosBackend(return to pool)"""
136
        self.backend.close()
137

    
138
    @handle_backend_exceptions
139
    def get_image(self, image_uuid):
140
        """Retrieve information about an image."""
141
        image_url = self._get_image_url(image_uuid)
142
        return self._get_image(image_url)
143

    
144
    def _get_image_url(self, image_uuid):
145
        """Get the Pithos url that corresponds to an image UUID."""
146
        account, container, name = self.backend.get_uuid(self.user, image_uuid)
147
        return create_url(account, container, name)
148

    
149
    def _get_image(self, image_url):
150
        """Get information about an Image.
151

152
        Get all available information about an Image.
153
        """
154
        account, container, name = split_url(image_url)
155
        versions = self.backend.list_versions(self.user, account, container,
156
                                              name)
157
        if not versions:
158
            raise Exception("Image without versions %s" % image_url)
159
        try:
160
            meta = self._get_meta(image_url)
161
            meta["deleted"] = ""
162
        except NameError:
163
            # Object was deleted, use the latest version
164
            version, timestamp = versions[-1]
165
            meta = self._get_meta(image_url, version)
166
            meta["deleted"] = timestamp
167

    
168
        meta["created"] = versions[0][1]
169

    
170
        if PLANKTON_PREFIX + 'name' not in meta:
171
            raise ImageNotFound("'%s' is not a Plankton image" % image_url)
172

    
173
        permissions = self._get_permissions(image_url)
174
        return image_to_dict(image_url, meta, permissions)
175

    
176
    def _get_meta(self, image_url, version=None):
177
        """Get object's metadata."""
178
        account, container, name = split_url(image_url)
179
        return self.backend.get_object_meta(self.user, account, container,
180
                                            name, PLANKTON_DOMAIN, version)
181

    
182
    def _update_meta(self, image_url, meta, replace=False):
183
        """Update object's metadata."""
184
        account, container, name = split_url(image_url)
185

    
186
        prefixed = {}
187
        for key, val in meta.items():
188
            if key in PLANKTON_META:
189
                if key == "properties":
190
                    val = json.dumps(val)
191
                prefixed[PLANKTON_PREFIX + key] = val
192

    
193
        self.backend.update_object_meta(self.user, account, container, name,
194
                                        PLANKTON_DOMAIN, prefixed, replace)
195
        logger.debug("User '%s' updated image '%s', meta: '%s'", self.user,
196
                     image_url, prefixed)
197

    
198
    def _get_permissions(self, image_url):
199
        """Get object's permissions."""
200
        account, container, name = split_url(image_url)
201
        _a, path, permissions = \
202
            self.backend.get_object_permissions(self.user, account, container,
203
                                                name)
204

    
205
        if path is None:
206
            logger.warning("Image '%s' got permissions from None path",
207
                           image_url)
208

    
209
        return permissions
210

    
211
    def _update_permissions(self, image_url, permissions):
212
        """Update object's permissions."""
213
        account, container, name = split_url(image_url)
214
        self.backend.update_object_permissions(self.user, account, container,
215
                                               name, permissions)
216
        logger.debug("User '%s' updated image '%s', permissions: '%s'",
217
                     self.user, image_url, permissions)
218

    
219
    @handle_backend_exceptions
220
    def unregister(self, image_uuid):
221
        """Unregister an image.
222

223
        Unregister an image, by removing all metadata from the Pithos
224
        file that exist in the PLANKTON_DOMAIN.
225

226
        """
227
        image_url = self._get_image_url(image_uuid)
228
        self._get_image(image_url)  # Assert that it is an image
229
        # Unregister the image by removing all metadata from domain
230
        # 'PLANKTON_DOMAIN'
231
        meta = {}
232
        self._update_meta(image_url, meta, True)
233
        logger.debug("User '%s' deleted image '%s'", self.user, image_url)
234

    
235
    @handle_backend_exceptions
236
    def add_user(self, image_uuid, add_user):
237
        """Add a user as an image member.
238

239
        Update read permissions of Pithos file, to include the specified user.
240

241
        """
242
        image_url = self._get_image_url(image_uuid)
243
        self._get_image(image_url)  # Assert that it is an image
244
        permissions = self._get_permissions(image_url)
245
        read = set(permissions.get("read", []))
246
        assert(isinstance(add_user, (str, unicode)))
247
        read.add(add_user)
248
        permissions["read"] = list(read)
249
        self._update_permissions(image_url, permissions)
250

    
251
    @handle_backend_exceptions
252
    def remove_user(self, image_uuid, remove_user):
253
        """Remove the user from image members.
254

255
        Remove the specified user from the read permissions of the Pithos file.
256

257
        """
258
        image_url = self._get_image_url(image_uuid)
259
        self._get_image(image_url)  # Assert that it is an image
260
        permissions = self._get_permissions(image_url)
261
        read = set(permissions.get("read", []))
262
        assert(isinstance(remove_user, (str, unicode)))
263
        try:
264
            read.remove(remove_user)
265
        except ValueError:
266
            return  # TODO: User did not have access
267
        permissions["read"] = list(read)
268
        self._update_permissions(image_url, permissions)
269

    
270
    @handle_backend_exceptions
271
    def replace_users(self, image_uuid, replace_users):
272
        """Replace image members.
273

274
        Replace the read permissions of the Pithos files with the specified
275
        users. If image is specified as public, we must preserve * permission.
276

277
        """
278
        image_url = self._get_image_url(image_uuid)
279
        image = self._get_image(image_url)
280
        permissions = self._get_permissions(image_url)
281
        assert(isinstance(replace_users, list))
282
        permissions["read"] = replace_users
283
        if image.get("is_public", False):
284
            permissions["read"].append("*")
285
        self._update_permissions(image_url, permissions)
286

    
287
    @handle_backend_exceptions
288
    def list_users(self, image_uuid):
289
        """List the image members.
290

291
        List the image members, by listing all users that have read permission
292
        to the corresponding Pithos file.
293

294
        """
295
        image_url = self._get_image_url(image_uuid)
296
        self._get_image(image_url)  # Assert that it is an image
297
        permissions = self._get_permissions(image_url)
298
        return [user for user in permissions.get('read', []) if user != '*']
299

    
300
    @handle_backend_exceptions
301
    def update_metadata(self, image_uuid, metadata):
302
        """Update Image metadata."""
303
        image_url = self._get_image_url(image_uuid)
304
        self._get_image(image_url)  # Assert that it is an image
305

    
306
        is_public = metadata.pop("is_public", None)
307
        if is_public is not None:
308
            permissions = self._get_permissions(image_url)
309
            read = set(permissions.get("read", []))
310
            if is_public:
311
                read.add("*")
312
            else:
313
                read.discard("*")
314
            permissions["read"] = list(read)
315
            self._update_permissions(image_url, permissions)
316
        meta = {}
317
        meta["properties"] = metadata.pop("properties", {})
318
        meta.update(**metadata)
319

    
320
        self._update_meta(image_url, meta)
321
        return self.get_image(image_uuid)
322

    
323
    @handle_backend_exceptions
324
    def register(self, name, image_url, metadata):
325
        # Validate that metadata are allowed
326
        if "id" in metadata:
327
            raise ValueError("Passing an ID is not supported")
328
        store = metadata.pop("store", "pithos")
329
        if store != "pithos":
330
            raise ValueError("Invalid store '%s'. Only 'pithos' store is"
331
                             "supported" % store)
332
        disk_format = metadata.setdefault("disk_format",
333
                                          settings.DEFAULT_DISK_FORMAT)
334
        if disk_format not in settings.ALLOWED_DISK_FORMATS:
335
            raise ValueError("Invalid disk format '%s'" % disk_format)
336
        container_format =\
337
            metadata.setdefault("container_format",
338
                                settings.DEFAULT_CONTAINER_FORMAT)
339
        if container_format not in settings.ALLOWED_CONTAINER_FORMATS:
340
            raise ValueError("Invalid container format '%s'" %
341
                             container_format)
342

    
343
        # Validate that 'size' and 'checksum' are valid
344
        account, container, object = split_url(image_url)
345

    
346
        meta = self._get_meta(image_url)
347

    
348
        size = int(metadata.pop('size', meta['bytes']))
349
        if size != meta['bytes']:
350
            raise ValueError("Invalid size")
351

    
352
        checksum = metadata.pop('checksum', meta['hash'])
353
        if checksum != meta['hash']:
354
            raise ValueError("Invalid checksum")
355

    
356
        # Fix permissions
357
        is_public = metadata.pop('is_public', False)
358
        if is_public:
359
            permissions = {'read': ['*']}
360
        else:
361
            permissions = {'read': [self.user]}
362

    
363
        # Update rest metadata
364
        meta = {}
365
        meta['properties'] = metadata.pop('properties', {})
366
        meta.update(name=name, status='available', **metadata)
367

    
368
        # Do the actualy update in the Pithos backend
369
        self._update_meta(image_url, meta)
370
        self._update_permissions(image_url, permissions)
371
        logger.debug("User '%s' created image '%s'('%s')", self.user,
372
                     image_url, name)
373
        return self._get_image(image_url)
374

    
375
    def _list_images(self, user=None, filters=None, params=None):
376
        filters = filters or {}
377

    
378
        # TODO: Use filters
379
        # # Fix keys
380
        # keys = [PLANKTON_PREFIX + 'name']
381
        # size_range = (None, None)
382
        # for key, val in filters.items():
383
        #     if key == 'size_min':
384
        #         size_range = (val, size_range[1])
385
        #     elif key == 'size_max':
386
        #         size_range = (size_range[0], val)
387
        #     else:
388
        #         keys.append('%s = %s' % (PLANKTON_PREFIX + key, val))
389
        _images = self.backend.get_domain_objects(domain=PLANKTON_DOMAIN,
390
                                                  user=user)
391

    
392
        images = []
393
        for (location, meta, permissions) in _images:
394
            image_url = "pithos://" + location
395
            meta["modified"] = meta["version_timestamp"]
396
            # TODO: Create metadata when registering an Image
397
            meta["created"] = meta["version_timestamp"]
398
            images.append(image_to_dict(image_url, meta, permissions))
399

    
400
        if params is None:
401
            params = {}
402
        key = itemgetter(params.get('sort_key', 'created_at'))
403
        reverse = params.get('sort_dir', 'desc') == 'desc'
404
        images.sort(key=key, reverse=reverse)
405
        return images
406

    
407
    def list_images(self, filters=None, params=None):
408
        return self._list_images(user=self.user, filters=filters,
409
                                 params=params)
410

    
411
    def list_shared_images(self, member, filters=None, params=None):
412
        images = self._list_images(user=self.user, filters=filters,
413
                                   params=params)
414
        is_shared = lambda img: not img["is_public"] and img["owner"] == member
415
        return filter(is_shared, images)
416

    
417
    def list_public_images(self, filters=None, params=None):
418
        images = self._list_images(user=None, filters=filters, params=params)
419
        return filter(lambda img: img["is_public"], images)
420

    
421

    
422
class ImageBackendError(Exception):
423
    pass
424

    
425

    
426
class ImageNotFound(ImageBackendError):
427
    pass
428

    
429

    
430
class Forbidden(ImageBackendError):
431
    pass
432

    
433

    
434
def image_to_dict(image_url, meta, permissions):
435
    """Render an image to a dictionary"""
436
    account, container, name = split_url(image_url)
437

    
438
    image = {}
439
    if PLANKTON_PREFIX + 'name' not in meta:
440
        raise ImageNotFound("'%s' is not a Plankton image" % image_url)
441

    
442
    image["id"] = meta["uuid"]
443
    image["location"] = image_url
444
    image["checksum"] = meta["hash"]
445
    image["created_at"] = format_timestamp(meta["created"])
446
    deleted = meta.get("deleted", None)
447
    image["deleted_at"] = format_timestamp(deleted) if deleted else ""
448
    image["updated_at"] = format_timestamp(meta["modified"])
449
    image["size"] = meta["bytes"]
450
    image["store"] = "pithos"
451
    image['owner'] = account
452

    
453
    # Permissions
454
    image["is_public"] = "*" in permissions.get('read', [])
455

    
456
    for key, val in meta.items():
457
        # Get plankton properties
458
        if key.startswith(PLANKTON_PREFIX):
459
            # Remove plankton prefix
460
            key = key.replace(PLANKTON_PREFIX, "")
461
            # Keep only those in plankton meta
462
            if key in PLANKTON_META:
463
                if key == "properties":
464
                    val = json.loads(val)
465
                image[key] = val
466

    
467
    return image