Revision 23808592 snf-cyclades-app/synnefo/plankton/backend.py
b/snf-cyclades-app/synnefo/plankton/backend.py | ||
---|---|---|
1 |
# Copyright 2011-2013 GRNET S.A. All rights reserved.
|
|
1 |
# Copyright 2011-2014 GRNET S.A. All rights reserved.
|
|
2 | 2 |
|
3 | 3 |
# |
4 | 4 |
# Redistribution and use in source and binary forms, with or |
... | ... | |
52 | 52 |
""" |
53 | 53 |
|
54 | 54 |
import json |
55 |
import warnings |
|
56 | 55 |
import logging |
57 | 56 |
import os |
58 | 57 |
|
59 | 58 |
from time import time, gmtime, strftime |
60 | 59 |
from functools import wraps |
61 | 60 |
from operator import itemgetter |
61 |
from collections import namedtuple |
|
62 | 62 |
|
63 | 63 |
from django.conf import settings |
64 | 64 |
from django.utils import importlib |
65 |
from pithos.backends.base import NotAllowedError, VersionNotExists |
|
65 |
from pithos.backends.base import NotAllowedError, VersionNotExists, QuotaError
|
|
66 | 66 |
from synnefo.util.text import uenc |
67 |
from copy import deepcopy |
|
68 |
from snf_django.lib.api import faults |
|
67 | 69 |
|
70 |
Location = namedtuple("ObjectLocation", ["account", "container", "path"]) |
|
68 | 71 |
|
69 | 72 |
logger = logging.getLogger(__name__) |
70 | 73 |
|
... | ... | |
94 | 97 |
return _pithos_backend_pool.pool_get() |
95 | 98 |
|
96 | 99 |
|
97 |
def create_url(account, container, name): |
|
98 |
assert "/" not in account, "Invalid account" |
|
99 |
assert "/" not in container, "Invalid container" |
|
100 |
return "pithos://%s/%s/%s" % (account, container, name) |
|
101 |
|
|
102 |
|
|
103 |
def split_url(url): |
|
104 |
"""Returns (accout, container, object) from a url string""" |
|
105 |
try: |
|
106 |
assert(isinstance(url, basestring)) |
|
107 |
t = url.split('/', 4) |
|
108 |
assert t[0] == "pithos:", "Invalid url" |
|
109 |
assert len(t) == 5, "Invalid url" |
|
110 |
return t[2:5] |
|
111 |
except AssertionError: |
|
112 |
raise InvalidLocation("Invalid location '%s" % url) |
|
113 |
|
|
114 |
|
|
115 | 100 |
def format_timestamp(t): |
116 | 101 |
return strftime('%Y-%m-%d %H:%M:%S', gmtime(t)) |
117 | 102 |
|
118 | 103 |
|
119 |
def handle_backend_exceptions(func):
|
|
104 |
def handle_pithos_backend(func):
|
|
120 | 105 |
@wraps(func) |
121 |
def wrapper(*args, **kwargs): |
|
122 |
try: |
|
123 |
return func(*args, **kwargs) |
|
124 |
except NotAllowedError: |
|
125 |
raise Forbidden |
|
126 |
except NameError: |
|
127 |
raise ImageNotFound |
|
128 |
except VersionNotExists: |
|
129 |
raise ImageNotFound |
|
130 |
return wrapper |
|
131 |
|
|
132 |
|
|
133 |
def commit_on_success(func): |
|
134 | 106 |
def wrapper(self, *args, **kwargs): |
135 | 107 |
backend = self.backend |
136 | 108 |
backend.pre_exec() |
109 |
commit = False |
|
137 | 110 |
try: |
138 | 111 |
ret = func(self, *args, **kwargs) |
139 |
except: |
|
140 |
backend.post_exec(False) |
|
141 |
raise |
|
112 |
except NotAllowedError: |
|
113 |
raise faults.Forbidden |
|
114 |
except (NameError, VersionNotExists): |
|
115 |
raise faults.ItemNotFound |
|
116 |
except (AssertionError, ValueError): |
|
117 |
raise faults.BadRequest |
|
118 |
except QuotaError: |
|
119 |
raise faults.OverLimit |
|
142 | 120 |
else: |
143 |
backend.post_exec(True) |
|
121 |
commit = True |
|
122 |
finally: |
|
123 |
backend.post_exec(commit) |
|
144 | 124 |
return ret |
145 | 125 |
return wrapper |
146 | 126 |
|
147 | 127 |
|
148 |
class ImageBackend(object):
|
|
128 |
class PlanktonBackend(object):
|
|
149 | 129 |
"""A wrapper arround the pithos backend to simplify image handling.""" |
150 | 130 |
|
151 | 131 |
def __init__(self, user): |
152 | 132 |
self.user = user |
153 |
|
|
154 |
original_filters = warnings.filters |
|
155 |
warnings.simplefilter('ignore') # Suppress SQLAlchemy warnings |
|
156 | 133 |
self.backend = get_pithos_backend() |
157 |
warnings.filters = original_filters # Restore warnings |
|
158 | 134 |
|
159 | 135 |
def close(self): |
160 | 136 |
"""Close PithosBackend(return to pool)""" |
161 | 137 |
self.backend.close() |
162 | 138 |
|
163 |
@handle_backend_exceptions |
|
164 |
@commit_on_success |
|
165 |
def get_image(self, image_uuid): |
|
166 |
"""Retrieve information about an image.""" |
|
167 |
image_url = self._get_image_url(image_uuid) |
|
168 |
return self._get_image(image_url) |
|
139 |
def __enter__(self): |
|
140 |
return self |
|
141 |
|
|
142 |
def __exit__(self, exc_type, exc_val, exc_tb): |
|
143 |
self.close() |
|
144 |
self.backend = None |
|
145 |
return False |
|
146 |
|
|
147 |
@handle_pithos_backend |
|
148 |
def get_image(self, uuid): |
|
149 |
return self._get_image(uuid) |
|
150 |
|
|
151 |
def _get_image(self, uuid): |
|
152 |
location, metadata = self._get_raw_metadata(uuid) |
|
153 |
permissions = self._get_raw_permissions(uuid, location) |
|
154 |
return image_to_dict(location, metadata, permissions) |
|
155 |
|
|
156 |
@handle_pithos_backend |
|
157 |
def add_property(self, uuid, key, value): |
|
158 |
location, _ = self._get_raw_metadata(uuid) |
|
159 |
properties = self._prefix_properties({key: value}) |
|
160 |
self._update_metadata(uuid, location, properties, replace=False) |
|
161 |
|
|
162 |
@handle_pithos_backend |
|
163 |
def remove_property(self, uuid, key): |
|
164 |
location, _ = self._get_raw_metadata(uuid) |
|
165 |
# Use empty string to delete a property |
|
166 |
properties = self._prefix_properties({key: ""}) |
|
167 |
self._update_metadata(uuid, location, properties, replace=False) |
|
168 |
|
|
169 |
@handle_pithos_backend |
|
170 |
def update_properties(self, uuid, properties): |
|
171 |
location, _ = self._get_raw_metadata(uuid) |
|
172 |
properties = self._prefix_properties(properties) |
|
173 |
self._update_metadata(uuid, location, properties, replace=False) |
|
174 |
|
|
175 |
@staticmethod |
|
176 |
def _prefix_properties(properties): |
|
177 |
"""Add property prefix to properties.""" |
|
178 |
return dict([(PROPERTY_PREFIX + k, v) for k, v in properties.items()]) |
|
179 |
|
|
180 |
@staticmethod |
|
181 |
def _unprefix_properties(properties): |
|
182 |
"""Remove property prefix from properties.""" |
|
183 |
return dict([(k.replace(PROPERTY_PREFIX, "", 1), v) |
|
184 |
for k, v in properties.items()]) |
|
185 |
|
|
186 |
@staticmethod |
|
187 |
def _prefix_metadata(metadata): |
|
188 |
"""Add plankton prefix to metadata.""" |
|
189 |
return dict([(PLANKTON_PREFIX + k, v) for k, v in metadata.items()]) |
|
190 |
|
|
191 |
@staticmethod |
|
192 |
def _unprefix_metadata(metadata): |
|
193 |
"""Remove plankton prefix from metadata.""" |
|
194 |
return dict([(k.replace(PLANKTON_PREFIX, "", 1), v) |
|
195 |
for k, v in metadata.items()]) |
|
196 |
|
|
197 |
@handle_pithos_backend |
|
198 |
def update_metadata(self, uuid, metadata): |
|
199 |
location, _ = self._get_raw_metadata(uuid) |
|
169 | 200 |
|
170 |
def _get_image_url(self, image_uuid): |
|
171 |
"""Get the Pithos url that corresponds to an image UUID.""" |
|
172 |
account, container, name = self.backend.get_uuid(self.user, image_uuid) |
|
173 |
return create_url(account, container, name) |
|
201 |
is_public = metadata.pop("is_public", None) |
|
202 |
if is_public is not None: |
|
203 |
self._set_public(uuid, location, public=is_public) |
|
174 | 204 |
|
175 |
def _get_image(self, image_url): |
|
176 |
"""Get information about an Image. |
|
205 |
# Each property is stored as a separate prefixed metadata |
|
206 |
meta = deepcopy(metadata) |
|
207 |
properties = meta.pop("properties", {}) |
|
208 |
meta.update(self._prefix_properties(properties)) |
|
177 | 209 |
|
178 |
Get all available information about an Image. |
|
179 |
""" |
|
180 |
account, container, name = split_url(image_url) |
|
181 |
try: |
|
182 |
meta = self._get_meta(image_url) |
|
183 |
meta["deleted"] = "" |
|
184 |
except NameError: |
|
185 |
versions = self.backend.list_versions(self.user, account, |
|
186 |
container, name) |
|
187 |
if not versions: |
|
188 |
raise Exception("Image without versions %s" % image_url) |
|
189 |
# Object was deleted, use the latest version |
|
190 |
version, timestamp = versions[-1] |
|
191 |
meta = self._get_meta(image_url, version) |
|
192 |
meta["deleted"] = timestamp |
|
193 |
|
|
194 |
# XXX: Check that an object is a plankton image! PithosBackend will |
|
195 |
# return common metadata for an object, even if it has no metadata in |
|
196 |
# plankton domain. All images must have a name, so we check if a file |
|
197 |
# is an image by checking if they are having an image name. |
|
198 |
if PLANKTON_PREFIX + 'name' not in meta: |
|
199 |
raise ImageNotFound |
|
200 |
|
|
201 |
permissions = self._get_permissions(image_url) |
|
202 |
return image_to_dict(image_url, meta, permissions) |
|
203 |
|
|
204 |
def _get_meta(self, image_url, version=None): |
|
205 |
"""Get object's metadata.""" |
|
206 |
account, container, name = split_url(image_url) |
|
207 |
return self.backend.get_object_meta(self.user, account, container, |
|
208 |
name, PLANKTON_DOMAIN, version) |
|
209 |
|
|
210 |
def _update_meta(self, image_url, meta, replace=False): |
|
211 |
"""Update object's metadata.""" |
|
212 |
account, container, name = split_url(image_url) |
|
213 |
|
|
214 |
prefixed = [(PLANKTON_PREFIX + uenc(k), uenc(v)) |
|
215 |
for k, v in meta.items() |
|
216 |
if k in PLANKTON_META or k.startswith(PROPERTY_PREFIX)] |
|
217 |
prefixed = dict(prefixed) |
|
218 |
|
|
219 |
for k, v in prefixed.items(): |
|
210 |
self._update_metadata(uuid, location, metadata=meta, replace=False) |
|
211 |
|
|
212 |
return self._get_image(uuid) |
|
213 |
|
|
214 |
def _update_metadata(self, uuid, location, metadata, replace=False): |
|
215 |
_prefixed_metadata = self._prefix_metadata(metadata) |
|
216 |
prefixed = {} |
|
217 |
for k, v in _prefixed_metadata.items(): |
|
218 |
# Encode to UTF-8 |
|
219 |
k, v = uenc(k), uenc(v) |
|
220 |
# Check the length of key/value |
|
220 | 221 |
if len(k) > 128: |
221 |
raise InvalidMetadata('Metadata keys should be less than %s '
|
|
222 |
'characters' % MAX_META_KEY_LENGTH)
|
|
222 |
raise faults.BadRequest('Metadata keys should be less than %s'
|
|
223 |
' characters' % MAX_META_KEY_LENGTH)
|
|
223 | 224 |
if len(v) > 256: |
224 |
raise InvalidMetadata('Metadata values should be less than %s ' |
|
225 |
'characters.' % MAX_META_VALUE_LENGTH) |
|
225 |
raise faults.BadRequest('Metadata values should be less than' |
|
226 |
' %scharacters.' |
|
227 |
% MAX_META_VALUE_LENGTH) |
|
228 |
prefixed[k] = v |
|
226 | 229 |
|
227 |
self.backend.update_object_meta(self.user, account, container, name, |
|
230 |
account, container, path = location |
|
231 |
self.backend.update_object_meta(self.user, account, container, path, |
|
228 | 232 |
PLANKTON_DOMAIN, prefixed, replace) |
229 |
logger.debug("User '%s' updated image '%s', meta: '%s'", self.user, |
|
230 |
image_url, prefixed) |
|
231 |
|
|
232 |
def _get_permissions(self, image_url): |
|
233 |
"""Get object's permissions.""" |
|
234 |
account, container, name = split_url(image_url) |
|
235 |
_a, path, permissions = \ |
|
236 |
self.backend.get_object_permissions(self.user, account, container, |
|
237 |
name) |
|
238 |
|
|
239 |
if path is None and permissions != {}: |
|
240 |
logger.warning("Image '%s' got permissions '%s' from 'None' path.", |
|
241 |
image_url, permissions) |
|
242 |
raise Exception("Database Inconsistency Error:" |
|
243 |
" Image '%s' got permissions from 'None' path." % |
|
244 |
image_url) |
|
245 |
|
|
246 |
return permissions |
|
247 |
|
|
248 |
def _update_permissions(self, image_url, permissions): |
|
249 |
"""Update object's permissions.""" |
|
250 |
account, container, name = split_url(image_url) |
|
251 |
self.backend.update_object_permissions(self.user, account, container, |
|
252 |
name, permissions) |
|
253 |
logger.debug("User '%s' updated image '%s', permissions: '%s'", |
|
254 |
self.user, image_url, permissions) |
|
233 |
logger.debug("User '%s' updated image '%s', metadata: '%s'", self.user, |
|
234 |
uuid, prefixed) |
|
255 | 235 |
|
256 |
@handle_backend_exceptions |
|
257 |
@commit_on_success |
|
258 |
def unregister(self, image_uuid): |
|
259 |
"""Unregister an image. |
|
236 |
def _get_raw_metadata(self, uuid, version=None, check_image=True): |
|
237 |
"""Get info and metadata in Plankton doamin for the Pithos object. |
|
260 | 238 |
|
261 |
Unregister an image, by removing all metadata from the Pithos |
|
262 |
file that exist in the PLANKTON_DOMAIN. |
|
239 |
Return the location and the metadata of the Pithos object. |
|
240 |
If 'check_image' is set, check that the Pithos object is a registered |
|
241 |
Plankton Image. |
|
263 | 242 |
|
264 | 243 |
""" |
265 |
image_url = self._get_image_url(image_uuid) |
|
266 |
self._get_image(image_url) # Assert that it is an image |
|
267 |
# Unregister the image by removing all metadata from domain |
|
268 |
# 'PLANKTON_DOMAIN' |
|
269 |
meta = {} |
|
270 |
self._update_meta(image_url, meta, True) |
|
271 |
logger.debug("User '%s' deleted image '%s'", self.user, image_url) |
|
272 |
|
|
273 |
@handle_backend_exceptions |
|
274 |
@commit_on_success |
|
275 |
def add_user(self, image_uuid, add_user): |
|
276 |
"""Add a user as an image member. |
|
277 |
|
|
278 |
Update read permissions of Pithos file, to include the specified user. |
|
279 |
|
|
280 |
""" |
|
281 |
image_url = self._get_image_url(image_uuid) |
|
282 |
self._get_image(image_url) # Assert that it is an image |
|
283 |
permissions = self._get_permissions(image_url) |
|
244 |
# Convert uuid to location |
|
245 |
account, container, path = self.backend.get_uuid(self.user, uuid) |
|
246 |
try: |
|
247 |
meta = self.backend.get_object_meta(self.user, account, container, |
|
248 |
path, PLANKTON_DOMAIN, version) |
|
249 |
meta["deleted"] = False |
|
250 |
except NameError: |
|
251 |
if version is not None: |
|
252 |
raise |
|
253 |
versions = self.backend.list_versions(self.user, account, |
|
254 |
container, path) |
|
255 |
assert(versions), ("Object without versions: %s/%s/%s" % |
|
256 |
(account, container, path)) |
|
257 |
# Object was deleted, use the latest version |
|
258 |
version, timestamp = versions[-1] |
|
259 |
meta = self.backend.get_object_meta(self.user, account, container, |
|
260 |
path, PLANKTON_DOMAIN, version) |
|
261 |
meta["deleted"] = True |
|
262 |
|
|
263 |
if check_image and PLANKTON_PREFIX + "name" not in meta: |
|
264 |
# Check that object is an image by checking if it has an Image name |
|
265 |
# in Plankton metadata |
|
266 |
raise faults.ItemNotFound("Image '%s' does not exist." % uuid) |
|
267 |
|
|
268 |
return Location(account, container, path), meta |
|
269 |
|
|
270 |
# Users and Permissions |
|
271 |
@handle_pithos_backend |
|
272 |
def add_user(self, uuid, user): |
|
273 |
assert(isinstance(user, basestring)) |
|
274 |
location, _ = self._get_raw_metadata(uuid) |
|
275 |
permissions = self._get_raw_permissions(uuid, location) |
|
284 | 276 |
read = set(permissions.get("read", [])) |
285 |
assert(isinstance(add_user, (str, unicode)))
|
|
286 |
read.add(add_user)
|
|
287 |
permissions["read"] = list(read) |
|
288 |
self._update_permissions(image_url, permissions)
|
|
277 |
if not user in read:
|
|
278 |
read.add(user)
|
|
279 |
permissions["read"] = list(read)
|
|
280 |
self._update_permissions(uuid, location, permissions)
|
|
289 | 281 |
|
290 |
@handle_backend_exceptions |
|
291 |
@commit_on_success |
|
292 |
def remove_user(self, image_uuid, remove_user): |
|
293 |
"""Remove the user from image members. |
|
282 |
@handle_pithos_backend |
|
283 |
def remove_user(self, uuid, user): |
|
284 |
assert(isinstance(user, basestring)) |
|
285 |
location, _ = self._get_raw_metadata(uuid) |
|
286 |
permissions = self._get_raw_permissions(uuid, location) |
|
287 |
read = set(permissions.get("read", [])) |
|
288 |
if user in read: |
|
289 |
read.remove(user) |
|
290 |
permissions["read"] = list(read) |
|
291 |
self._update_permissions(uuid, location, permissions) |
|
294 | 292 |
|
295 |
Remove the specified user from the read permissions of the Pithos file. |
|
293 |
@handle_pithos_backend |
|
294 |
def replace_users(self, uuid, users): |
|
295 |
assert(isinstance(users, list)) |
|
296 |
location, _ = self._get_raw_metadata(uuid) |
|
297 |
permissions = self._get_raw_permissions(uuid, location) |
|
298 |
read = set(permissions.get("read", [])) |
|
299 |
if "*" in read: # Retain public permissions |
|
300 |
users.append("*") |
|
301 |
permissions["read"] = list(users) |
|
302 |
self._update_permissions(uuid, location, permissions) |
|
303 |
|
|
304 |
@handle_pithos_backend |
|
305 |
def list_users(self, uuid): |
|
306 |
location, _ = self._get_raw_metadata(uuid) |
|
307 |
permissions = self._get_raw_permissions(uuid, location) |
|
308 |
return [user for user in permissions.get('read', []) if user != '*'] |
|
296 | 309 |
|
297 |
""" |
|
298 |
image_url = self._get_image_url(image_uuid) |
|
299 |
self._get_image(image_url) # Assert that it is an image |
|
300 |
permissions = self._get_permissions(image_url) |
|
310 |
def _set_public(self, uuid, location, public): |
|
311 |
permissions = self._get_raw_permissions(uuid, location) |
|
312 |
assert(isinstance(public, bool)) |
|
301 | 313 |
read = set(permissions.get("read", [])) |
302 |
assert(isinstance(remove_user, (str, unicode))) |
|
303 |
try: |
|
304 |
read.remove(remove_user) |
|
305 |
except ValueError: |
|
306 |
return # TODO: User did not have access |
|
314 |
if public and "*" not in read: |
|
315 |
read.add("*") |
|
316 |
elif not public and "*" in read: |
|
317 |
read.discard("*") |
|
307 | 318 |
permissions["read"] = list(read) |
308 |
self._update_permissions(image_url, permissions) |
|
309 |
|
|
310 |
@handle_backend_exceptions |
|
311 |
@commit_on_success |
|
312 |
def replace_users(self, image_uuid, replace_users): |
|
313 |
"""Replace image members. |
|
314 |
|
|
315 |
Replace the read permissions of the Pithos files with the specified |
|
316 |
users. If image is specified as public, we must preserve * permission. |
|
317 |
|
|
318 |
""" |
|
319 |
image_url = self._get_image_url(image_uuid) |
|
320 |
image = self._get_image(image_url) |
|
321 |
permissions = self._get_permissions(image_url) |
|
322 |
assert(isinstance(replace_users, list)) |
|
323 |
permissions["read"] = replace_users |
|
324 |
if image.get("is_public", False): |
|
325 |
permissions["read"].append("*") |
|
326 |
self._update_permissions(image_url, permissions) |
|
327 |
|
|
328 |
@handle_backend_exceptions |
|
329 |
@commit_on_success |
|
330 |
def list_users(self, image_uuid): |
|
331 |
"""List the image members. |
|
332 |
|
|
333 |
List the image members, by listing all users that have read permission |
|
334 |
to the corresponding Pithos file. |
|
335 |
|
|
336 |
""" |
|
337 |
image_url = self._get_image_url(image_uuid) |
|
338 |
self._get_image(image_url) # Assert that it is an image |
|
339 |
permissions = self._get_permissions(image_url) |
|
340 |
return [user for user in permissions.get('read', []) if user != '*'] |
|
319 |
self._update_permissions(uuid, location, permissions) |
|
320 |
return permissions |
|
341 | 321 |
|
342 |
@handle_backend_exceptions |
|
343 |
@commit_on_success |
|
344 |
def update_metadata(self, image_uuid, metadata): |
|
345 |
"""Update Image metadata.""" |
|
346 |
image_url = self._get_image_url(image_uuid) |
|
347 |
self._get_image(image_url) # Assert that it is an image |
|
322 |
def _get_raw_permissions(self, uuid, location): |
|
323 |
account, container, path = location |
|
324 |
_a, path, permissions = \ |
|
325 |
self.backend.get_object_permissions(self.user, account, container, |
|
326 |
path) |
|
348 | 327 |
|
349 |
# 'is_public' metadata is translated in proper file permissions |
|
350 |
is_public = metadata.pop("is_public", None) |
|
351 |
if is_public is not None: |
|
352 |
permissions = self._get_permissions(image_url) |
|
353 |
read = set(permissions.get("read", [])) |
|
354 |
if is_public: |
|
355 |
read.add("*") |
|
356 |
else: |
|
357 |
read.discard("*") |
|
358 |
permissions["read"] = list(read) |
|
359 |
self._update_permissions(image_url, permissions) |
|
328 |
if path is None and permissions != {}: |
|
329 |
raise Exception("Database Inconsistency Error:" |
|
330 |
" Image '%s' got permissions from 'None' path." % |
|
331 |
uuid) |
|
360 | 332 |
|
361 |
# Extract the properties dictionary from metadata, and store each |
|
362 |
# property as a separeted, prefixed metadata |
|
363 |
properties = metadata.pop("properties", {}) |
|
364 |
meta = dict([(PROPERTY_PREFIX + k, v) for k, v in properties.items()]) |
|
365 |
# Also add the following metadata |
|
366 |
meta.update(**metadata) |
|
333 |
return permissions |
|
367 | 334 |
|
368 |
self._update_meta(image_url, meta) |
|
369 |
image_url = self._get_image_url(image_uuid) |
|
370 |
return self._get_image(image_url) |
|
335 |
def _update_permissions(self, uuid, location, permissions): |
|
336 |
account, container, path = location |
|
337 |
self.backend.update_object_permissions(self.user, account, container, |
|
338 |
path, permissions) |
|
339 |
logger.debug("User '%s' updated image '%s' permissions: '%s'", |
|
340 |
self.user, uuid, permissions) |
|
371 | 341 |
|
372 |
@handle_backend_exceptions |
|
373 |
@commit_on_success |
|
342 |
@handle_pithos_backend |
|
374 | 343 |
def register(self, name, image_url, metadata): |
375 | 344 |
# Validate that metadata are allowed |
376 | 345 |
if "id" in metadata: |
377 |
raise ValueError("Passing an ID is not supported")
|
|
346 |
raise faults.BadRequest("Passing an ID is not supported")
|
|
378 | 347 |
store = metadata.pop("store", "pithos") |
379 | 348 |
if store != "pithos": |
380 |
raise ValueError("Invalid store '%s'. Only 'pithos' store is"
|
|
381 |
"supported" % store)
|
|
349 |
raise faults.BadRequest("Invalid store '%s'. Only 'pithos' store"
|
|
350 |
" is supported" % store)
|
|
382 | 351 |
disk_format = metadata.setdefault("disk_format", |
383 | 352 |
settings.DEFAULT_DISK_FORMAT) |
384 | 353 |
if disk_format not in settings.ALLOWED_DISK_FORMATS: |
385 |
raise ValueError("Invalid disk format '%s'" % disk_format)
|
|
354 |
raise faults.BadRequest("Invalid disk format '%s'" % disk_format)
|
|
386 | 355 |
container_format =\ |
387 | 356 |
metadata.setdefault("container_format", |
388 | 357 |
settings.DEFAULT_CONTAINER_FORMAT) |
389 | 358 |
if container_format not in settings.ALLOWED_CONTAINER_FORMATS: |
390 |
raise ValueError("Invalid container format '%s'" % |
|
391 |
container_format) |
|
392 |
|
|
393 |
# Validate that 'size' and 'checksum' are valid |
|
394 |
account, container, object = split_url(image_url) |
|
359 |
raise faults.BadRequest("Invalid container format '%s'" % |
|
360 |
container_format) |
|
395 | 361 |
|
396 |
meta = self._get_meta(image_url) |
|
362 |
account, container, path = split_url(image_url) |
|
363 |
location = Location(account, container, path) |
|
364 |
meta = self.backend.get_object_meta(self.user, account, container, |
|
365 |
path, PLANKTON_DOMAIN, None) |
|
366 |
uuid = meta["uuid"] |
|
397 | 367 |
|
398 |
size = int(metadata.pop('size', meta['bytes'])) |
|
399 |
if size != meta['bytes']: |
|
400 |
raise ValueError("Invalid size") |
|
368 |
# Validate that 'size' and 'checksum' |
|
369 |
size = metadata.pop('size', int(meta['bytes'])) |
|
370 |
if not isinstance(size, int) or int(size) != int(meta["bytes"]): |
|
371 |
raise faults.BadRequest("Invalid 'size' field") |
|
401 | 372 |
|
402 | 373 |
checksum = metadata.pop('checksum', meta['hash']) |
403 |
if checksum != meta['hash']: |
|
404 |
raise ValueError("Invalid checksum") |
|
374 |
if not isinstance(checksum, basestring) or checksum != meta['hash']: |
|
375 |
raise faults.BadRequest("Invalid checksum field") |
|
376 |
|
|
377 |
users = [self.user] |
|
378 |
public = metadata.pop("is_public", False) |
|
379 |
if not isinstance(public, bool): |
|
380 |
raise faults.BadRequest("Invalid value for 'is_public' metadata") |
|
381 |
if public: |
|
382 |
users.append("*") |
|
383 |
permissions = {'read': users} |
|
384 |
self._update_permissions(uuid, location, permissions) |
|
385 |
|
|
386 |
# Each property is stored as a separate prefixed metadata |
|
387 |
meta = deepcopy(metadata) |
|
388 |
properties = meta.pop("properties", {}) |
|
389 |
meta.update(self._prefix_properties(properties)) |
|
390 |
# Add extra metadata |
|
391 |
meta["name"] = name |
|
392 |
meta["status"] = "AVAILABLE" |
|
393 |
meta['created_at'] = str(time()) |
|
394 |
#meta["is_snapshot"] = False |
|
395 |
self._update_metadata(uuid, location, metadata=meta, replace=False) |
|
396 |
|
|
397 |
logger.debug("User '%s' registered image '%s'('%s')", self.user, |
|
398 |
uuid, location) |
|
399 |
return self._get_image(uuid) |
|
400 |
|
|
401 |
@handle_pithos_backend |
|
402 |
def unregister(self, uuid): |
|
403 |
"""Unregister an Image. |
|
404 |
|
|
405 |
Unregister an Image by removing all the metadata in the Plankton |
|
406 |
domain. The Pithos file is not deleted. |
|
405 | 407 |
|
406 |
# Fix permissions |
|
407 |
is_public = metadata.pop('is_public', False) |
|
408 |
if is_public: |
|
409 |
permissions = {'read': ['*']} |
|
410 |
else: |
|
411 |
permissions = {'read': [self.user]} |
|
412 |
|
|
413 |
# Extract the properties dictionary from metadata, and store each |
|
414 |
# property as a separeted, prefixed metadata |
|
415 |
properties = metadata.pop("properties", {}) |
|
416 |
meta = dict([(PROPERTY_PREFIX + k, v) for k, v in properties.items()]) |
|
417 |
# Add creation(register) timestamp as a metadata, to avoid extra |
|
418 |
# queries when retrieving the list of images. |
|
419 |
meta['created_at'] = time() |
|
420 |
# Update rest metadata |
|
421 |
meta.update(name=name, status='available', **metadata) |
|
422 |
|
|
423 |
# Do the actualy update in the Pithos backend |
|
424 |
self._update_meta(image_url, meta) |
|
425 |
self._update_permissions(image_url, permissions) |
|
426 |
logger.debug("User '%s' created image '%s'('%s')", self.user, |
|
427 |
image_url, name) |
|
428 |
return self._get_image(image_url) |
|
408 |
""" |
|
409 |
location, _ = self._get_raw_metadata(uuid) |
|
410 |
self._update_metadata(uuid, location, metadata={}, replace=True) |
|
411 |
logger.debug("User '%s' unregistered image '%s'", self.user, uuid) |
|
429 | 412 |
|
413 |
# List functions |
|
430 | 414 |
def _list_images(self, user=None, filters=None, params=None): |
431 | 415 |
filters = filters or {} |
432 | 416 |
|
... | ... | |
445 | 429 |
user=user) |
446 | 430 |
|
447 | 431 |
images = [] |
448 |
for (location, meta, permissions) in _images: |
|
449 |
image_url = "pithos://" + location |
|
450 |
meta["modified"] = meta["version_timestamp"] |
|
451 |
images.append(image_to_dict(image_url, meta, permissions)) |
|
432 |
for (location, metadata, permissions) in _images: |
|
433 |
location = Location(*location.split("/", 2)) |
|
434 |
images.append(image_to_dict(location, metadata, permissions)) |
|
452 | 435 |
|
453 | 436 |
if params is None: |
454 | 437 |
params = {} |
438 |
|
|
455 | 439 |
key = itemgetter(params.get('sort_key', 'created_at')) |
456 | 440 |
reverse = params.get('sort_dir', 'desc') == 'desc' |
457 | 441 |
images.sort(key=key, reverse=reverse) |
458 | 442 |
return images |
459 | 443 |
|
460 |
@commit_on_success
|
|
444 |
@handle_pithos_backend
|
|
461 | 445 |
def list_images(self, filters=None, params=None): |
462 | 446 |
return self._list_images(user=self.user, filters=filters, |
463 | 447 |
params=params) |
464 | 448 |
|
465 |
@commit_on_success
|
|
449 |
@handle_pithos_backend
|
|
466 | 450 |
def list_shared_images(self, member, filters=None, params=None): |
467 | 451 |
images = self._list_images(user=self.user, filters=filters, |
468 | 452 |
params=params) |
469 | 453 |
is_shared = lambda img: not img["is_public"] and img["owner"] == member |
470 | 454 |
return filter(is_shared, images) |
471 | 455 |
|
472 |
@commit_on_success
|
|
456 |
@handle_pithos_backend
|
|
473 | 457 |
def list_public_images(self, filters=None, params=None): |
474 | 458 |
images = self._list_images(user=None, filters=filters, params=params) |
475 | 459 |
return filter(lambda img: img["is_public"], images) |
476 | 460 |
|
461 |
# # Snapshots |
|
462 |
# def list_snapshots(self, user=None): |
|
463 |
# _snapshots = self.list_images() |
|
464 |
# return [s for s in _snapshots if s["is_snapshot"]] |
|
465 |
|
|
466 |
# @handle_pithos_backend |
|
467 |
# def get_snapshot(self, user, snapshot_uuid): |
|
468 |
# snap = self._get_image(snapshot_uuid) |
|
469 |
# if snap.get("is_snapshot", False) is False: |
|
470 |
# raise faults.ItemNotFound("Snapshots '%s' does not exist" % |
|
471 |
# snapshot_uuid) |
|
472 |
# return snap |
|
473 |
|
|
474 |
# @handle_pithos_backend |
|
475 |
# def delete_snapshot(self, snapshot_uuid): |
|
476 |
# self.backend.delete_object_for_uuid(self.user, snapshot_uuid) |
|
477 |
|
|
478 |
# @handle_pithos_backend |
|
479 |
# def update_status(self, image_uuid, status): |
|
480 |
# """Update status of snapshot""" |
|
481 |
# location, _ = self._get_raw_metadata(image_uuid) |
|
482 |
# properties = {"status": status.upper()} |
|
483 |
# self._update_metadata(image_uuid, location, properties, |
|
484 |
# replace=False) |
|
485 |
# return self._get_image(image_uuid) |
|
477 | 486 |
|
478 |
class ImageBackendError(Exception): |
|
479 |
pass |
|
480 | 487 |
|
481 |
|
|
482 |
class ImageNotFound(ImageBackendError): |
|
483 |
pass |
|
484 |
|
|
485 |
|
|
486 |
class Forbidden(ImageBackendError): |
|
487 |
pass |
|
488 |
|
|
489 |
|
|
490 |
class InvalidMetadata(ImageBackendError): |
|
491 |
pass |
|
488 |
def create_url(account, container, name): |
|
489 |
"""Create a Pithos URL from the object info""" |
|
490 |
assert "/" not in account, "Invalid account" |
|
491 |
assert "/" not in container, "Invalid container" |
|
492 |
return "pithos://%s/%s/%s" % (account, container, name) |
|
492 | 493 |
|
493 | 494 |
|
494 |
class InvalidLocation(ImageBackendError): |
|
495 |
pass |
|
495 |
def split_url(url): |
|
496 |
"""Get object info from the Pithos URL""" |
|
497 |
assert(isinstance(url, basestring)) |
|
498 |
t = url.split('/', 4) |
|
499 |
assert t[0] == "pithos:", "Invalid url" |
|
500 |
assert len(t) == 5, "Invalid url" |
|
501 |
return t[2:5] |
|
496 | 502 |
|
497 | 503 |
|
498 |
def image_to_dict(image_url, meta, permissions):
|
|
504 |
def image_to_dict(location, metadata, permissions):
|
|
499 | 505 |
"""Render an image to a dictionary""" |
500 |
account, container, name = split_url(image_url)
|
|
506 |
account, container, name = location
|
|
501 | 507 |
|
502 | 508 |
image = {} |
503 |
if PLANKTON_PREFIX + 'name' not in meta: |
|
504 |
logger.warning("Image without Plankton name!! url %s meta %s", |
|
505 |
image_url, meta) |
|
506 |
image[PLANKTON_PREFIX + "name"] = "" |
|
507 |
|
|
508 |
image["id"] = meta["uuid"] |
|
509 |
image["location"] = image_url |
|
510 |
image["checksum"] = meta["hash"] |
|
511 |
created = meta.get("created_at", meta["modified"]) |
|
512 |
image["created_at"] = format_timestamp(created) |
|
513 |
deleted = meta.get("deleted", None) |
|
514 |
image["deleted_at"] = format_timestamp(deleted) if deleted else "" |
|
515 |
image["updated_at"] = format_timestamp(meta["modified"]) |
|
516 |
image["size"] = meta["bytes"] |
|
517 |
image["store"] = "pithos" |
|
509 |
image["id"] = metadata["uuid"] |
|
510 |
image["mapfile"] = metadata["hash"] |
|
511 |
image["checksum"] = metadata["hash"] |
|
512 |
image["location"] = create_url(account, container, name) |
|
513 |
image["size"] = metadata["bytes"] |
|
518 | 514 |
image['owner'] = account |
519 |
|
|
515 |
image["store"] = u"pithos" |
|
516 |
#image["is_snapshot"] = metadata.pop(PLANKTON_PREFIX + "is_snapshot", |
|
517 |
#False) |
|
520 | 518 |
# Permissions |
521 |
image["is_public"] = "*" in permissions.get('read', []) |
|
519 |
users = list(permissions.get("read", [])) |
|
520 |
image["is_public"] = "*" in users |
|
521 |
image["users"] = [u for u in users if u != "*"] |
|
522 |
# Timestamps |
|
523 |
updated_at = metadata["version_timestamp"] |
|
524 |
created_at = metadata.get("created_at", updated_at) |
|
525 |
image["created_at"] = format_timestamp(created_at) |
|
526 |
image["updated_at"] = format_timestamp(updated_at) |
|
527 |
if metadata.get("deleted", False): |
|
528 |
image["deleted_at"] = image["updated_at"] |
|
529 |
else: |
|
530 |
image["deleted_at"] = "" |
|
522 | 531 |
|
523 | 532 |
properties = {} |
524 |
for key, val in meta.items(): |
|
533 |
for key, val in metadata.items():
|
|
525 | 534 |
# Get plankton properties |
526 | 535 |
if key.startswith(PLANKTON_PREFIX): |
527 | 536 |
# Remove plankton prefix |
528 | 537 |
key = key.replace(PLANKTON_PREFIX, "") |
529 |
# Keep only those in plankton meta |
|
538 |
# Keep only those in plankton metadata
|
|
530 | 539 |
if key in PLANKTON_META: |
531 | 540 |
if key != "created_at": |
532 | 541 |
# created timestamp is return in 'created_at' field |
... | ... | |
583 | 592 |
backend_module = getattr(settings, 'PLANKTON_BACKEND_MODULE', None) |
584 | 593 |
if not backend_module: |
585 | 594 |
# no setting set |
586 |
return ImageBackend
|
|
595 |
return PlanktonBackend
|
|
587 | 596 |
|
588 | 597 |
parts = backend_module.split(".") |
589 | 598 |
module = ".".join(parts[:-1]) |
Also available in: Unified diff