Revision bfd9f988 snf-app/synnefo/plankton/backend.py
| b/snf-app/synnefo/plankton/backend.py | ||
|---|---|---|
| 32 | 32 |
# or implied, of GRNET S.A. |
| 33 | 33 |
|
| 34 | 34 |
""" |
| 35 |
Plankton attributes are divided in 3 categories: |
|
| 36 |
- generated: They are dynamically generated and not stored anywhere. |
|
| 37 |
- user: Stored as user accessible metadata and can be modified from within |
|
| 38 |
Pithos apps. They are visible as prefixed with PLANKTON_PREFIX. |
|
| 39 |
- system: Stored as metadata that can not be modified through Pithos. |
|
| 40 |
|
|
| 41 |
In more detail, Plankton attributes are the following: |
|
| 35 |
The Plankton attributes are the following: |
|
| 42 | 36 |
- checksum: the 'hash' meta |
| 43 | 37 |
- container_format: stored as a user meta |
| 44 | 38 |
- created_at: the 'modified' meta of the first version |
| ... | ... | |
| 59 | 53 |
import json |
| 60 | 54 |
import warnings |
| 61 | 55 |
|
| 62 |
from binascii import hexlify |
|
| 63 |
from functools import partial |
|
| 64 |
from hashlib import md5 |
|
| 65 | 56 |
from operator import itemgetter |
| 66 | 57 |
from time import gmtime, strftime, time |
| 67 |
from uuid import UUID |
|
| 68 | 58 |
|
| 69 | 59 |
from django.conf import settings |
| 70 | 60 |
|
| ... | ... | |
| 72 | 62 |
from pithos.backends.base import NotAllowedError |
| 73 | 63 |
|
| 74 | 64 |
|
| 75 |
PITHOS_DOMAIN = 'pithos' |
|
| 76 | 65 |
PLANKTON_DOMAIN = 'plankton' |
| 77 |
|
|
| 78 |
PLANKTON_PREFIX = 'X-Object-Meta-plankton:' |
|
| 66 |
PLANKTON_PREFIX = 'plankton:' |
|
| 79 | 67 |
PROPERTY_PREFIX = 'property:' |
| 80 | 68 |
|
| 81 |
SYSTEM_META = set(['status'])
|
|
| 82 |
USER_META = set(['name', 'container_format', 'disk_format'])
|
|
| 69 |
PLANKTON_META = ('container_format', 'disk_format', 'name', 'properties',
|
|
| 70 |
'status')
|
|
| 83 | 71 |
|
| 84 | 72 |
|
| 85 | 73 |
def get_location(account, container, object): |
| ... | ... | |
| 103 | 91 |
|
| 104 | 92 |
def __init__(self, user): |
| 105 | 93 |
self.user = user |
| 106 |
self.container = settings.PITHOS_IMAGE_CONTAINER |
|
| 107 | 94 |
|
| 108 | 95 |
original_filters = warnings.filters |
| 109 | 96 |
warnings.simplefilter('ignore') # Suppress SQLAlchemy warnings
|
| 110 |
self.backend = connect_backend() |
|
| 97 |
db_connection = settings.BACKEND_DB_CONNECTION |
|
| 98 |
block_path = settings.BACKEND_BLOCK_PATH |
|
| 99 |
self.backend = connect_backend(db_connection=db_connection, |
|
| 100 |
block_path=block_path) |
|
| 111 | 101 |
warnings.filters = original_filters # Restore warnings |
| 112 |
|
|
| 113 |
try: |
|
| 114 |
self.backend.put_container(self.user, self.user, self.container) |
|
| 115 |
except NameError: |
|
| 116 |
pass # Container already exists |
|
| 117 | 102 |
|
| 118 | 103 |
def _get_image(self, location): |
| 119 | 104 |
def format_timestamp(t): |
| ... | ... | |
| 155 | 140 |
image['properties'] = {}
|
| 156 | 141 |
|
| 157 | 142 |
for key, val in meta.items(): |
| 158 |
if key.startswith(PLANKTON_PREFIX): |
|
| 159 |
key = key[len(PLANKTON_PREFIX):] |
|
| 160 |
|
|
| 161 |
if key in SYSTEM_META | USER_META: |
|
| 143 |
if not key.startswith(PLANKTON_PREFIX): |
|
| 144 |
continue |
|
| 145 |
key = key[len(PLANKTON_PREFIX):] |
|
| 146 |
if key == 'properties': |
|
| 147 |
val = json.loads(val) |
|
| 148 |
if key in PLANKTON_META: |
|
| 162 | 149 |
image[key] = val |
| 163 |
elif key.startswith(PROPERTY_PREFIX): |
|
| 164 |
key = key[len(PROPERTY_PREFIX):] |
|
| 165 |
image['properties'][key] = val |
|
| 166 | 150 |
|
| 167 | 151 |
return image |
| 168 | 152 |
|
| 169 |
def _get_meta(self, location, version=None, user=None): |
|
| 170 |
user = user or self.user |
|
| 153 |
def _get_meta(self, location, version=None): |
|
| 171 | 154 |
account, container, object = split_location(location) |
| 172 | 155 |
try: |
| 173 |
meta = self.backend.get_object_meta(user, account, container, |
|
| 174 |
object, PITHOS_DOMAIN, version) |
|
| 175 |
meta.update(self.backend.get_object_meta(user, account, container, |
|
| 176 |
object, PLANKTON_DOMAIN, version)) |
|
| 156 |
return self.backend.get_object_meta(self.user, account, container, |
|
| 157 |
object, PLANKTON_DOMAIN, version) |
|
| 177 | 158 |
except NameError: |
| 178 | 159 |
return None |
| 179 |
|
|
| 180 |
return meta |
|
| 181 | 160 |
|
| 182 | 161 |
def _get_permissions(self, location): |
| 183 | 162 |
account, container, object = split_location(location) |
| ... | ... | |
| 213 | 192 |
self._update_meta(location, meta, replace=True) |
| 214 | 193 |
|
| 215 | 194 |
def _update_meta(self, location, meta, replace=False): |
| 216 |
user = self.user |
|
| 217 | 195 |
account, container, object = split_location(location) |
| 218 |
|
|
| 219 |
user_meta = {}
|
|
| 220 |
system_meta = {}
|
|
| 196 |
|
|
| 197 |
prefixed = {}
|
|
| 221 | 198 |
for key, val in meta.items(): |
| 222 |
if key in SYSTEM_META: |
|
| 223 |
system_meta[key] = val |
|
| 224 |
elif key in USER_META: |
|
| 225 |
user_meta[PLANKTON_PREFIX + key] = val |
|
| 226 |
elif key == 'properties': |
|
| 227 |
for k, v in val.items(): |
|
| 228 |
user_meta[PLANKTON_PREFIX + PROPERTY_PREFIX + k] = v |
|
| 229 |
|
|
| 230 |
if user_meta: |
|
| 231 |
self.backend.update_object_meta(user, account, container, object, |
|
| 232 |
PITHOS_DOMAIN, user_meta, replace) |
|
| 233 |
replace = False |
|
| 234 |
|
|
| 235 |
if system_meta: |
|
| 236 |
self.backend.update_object_meta(user, account, container, object, |
|
| 237 |
PLANKTON_DOMAIN, system_meta, replace) |
|
| 199 |
if key == 'properties': |
|
| 200 |
val = json.dumps(val) |
|
| 201 |
if key in PLANKTON_META: |
|
| 202 |
prefixed[PLANKTON_PREFIX + key] = val |
|
| 203 |
|
|
| 204 |
self.backend.update_object_meta(self.user, account, container, object, |
|
| 205 |
PLANKTON_DOMAIN, prefixed, replace) |
|
| 238 | 206 |
|
| 239 | 207 |
def _update_permissions(self, location, permissions): |
| 240 | 208 |
account, container, object = split_location(location) |
| ... | ... | |
| 242 | 210 |
object, permissions) |
| 243 | 211 |
|
| 244 | 212 |
def add_user(self, image_id, user): |
| 245 |
image = self.get_meta(image_id)
|
|
| 213 |
image = self.get_image(image_id)
|
|
| 246 | 214 |
assert image, "Image not found" |
| 247 | 215 |
|
| 248 | 216 |
location = image['location'] |
| ... | ... | |
| 263 | 231 |
assert len(data) == size |
| 264 | 232 |
return data |
| 265 | 233 |
|
| 266 |
def get_meta(self, image_id):
|
|
| 234 |
def get_image(self, image_id):
|
|
| 267 | 235 |
try: |
| 268 | 236 |
account, container, object = self.backend.get_uuid(self.user, |
| 269 | 237 |
image_id) |
| ... | ... | |
| 276 | 244 |
def iter_public(self, filters): |
| 277 | 245 |
backend = self.backend |
| 278 | 246 |
|
| 279 |
keys = set()
|
|
| 247 |
keys = [PLANKTON_PREFIX + 'name']
|
|
| 280 | 248 |
for key, val in filters.items(): |
| 281 | 249 |
if key == 'size_min': |
| 282 |
filter = 'bytes >= %d' % size_min
|
|
| 250 |
filter = 'bytes >= %d' % int(val)
|
|
| 283 | 251 |
elif key == 'size_max': |
| 284 |
filter = 'bytes <= %d' % size_max
|
|
| 252 |
filter = 'bytes <= %d' % int(val)
|
|
| 285 | 253 |
else: |
| 286 |
# XXX Only filters for user meta supported |
|
| 287 | 254 |
filter = '%s = %s' % (PLANKTON_PREFIX + key, val) |
| 288 |
keys.add(filter) |
|
| 289 |
|
|
| 290 |
container = self.container |
|
| 255 |
keys.append(filter) |
|
| 291 | 256 |
|
| 292 | 257 |
for account in backend.list_accounts(None): |
| 293 | 258 |
for container in backend.list_containers(None, account, |
| 294 | 259 |
shared=True): |
| 295 | 260 |
for path, version_id in backend.list_objects(None, account, |
| 296 | 261 |
container, prefix='', delimiter='/', |
| 297 |
domain=PITHOS_DOMAIN,
|
|
| 262 |
domain=PLANKTON_DOMAIN,
|
|
| 298 | 263 |
keys=keys, shared=True): |
| 299 | 264 |
location = get_location(account, container, path) |
| 300 | 265 |
image = self._get_image(location) |
| ... | ... | |
| 308 | 273 |
|
| 309 | 274 |
# To get the list we connect as member and get the list shared by us |
| 310 | 275 |
for container in backend.list_containers(member, self.user): |
| 311 |
for path, version_id in backend.list_objects(member, self.user, |
|
| 312 |
container, prefix='', delimiter='/', domain=PITHOS_DOMAIN): |
|
| 276 |
for object, version_id in backend.list_objects(member, self.user, |
|
| 277 |
container, prefix='', delimiter='/', |
|
| 278 |
domain=PLANKTON_DOMAIN): |
|
| 313 | 279 |
try: |
| 314 |
location = get_location(self.user, container, path) |
|
| 315 |
meta = self._get_meta(location, user=member) |
|
| 280 |
location = get_location(self.user, container, object) |
|
| 281 |
meta = backend.get_object_meta(member, self.user, |
|
| 282 |
container, object, PLANKTON_DOMAIN) |
|
| 316 | 283 |
if PLANKTON_PREFIX + 'name' in meta: |
| 317 | 284 |
yield meta['uuid'] |
| 318 | 285 |
except NotAllowedError: |
| ... | ... | |
| 326 | 293 |
return images |
| 327 | 294 |
|
| 328 | 295 |
def list_users(self, image_id): |
| 329 |
image = self.get_meta(image_id)
|
|
| 296 |
image = self.get_image(image_id)
|
|
| 330 | 297 |
assert image, "Image not found" |
| 331 | 298 |
|
| 332 | 299 |
permissions = self._get_permissions(image['location']) |
| ... | ... | |
| 343 | 310 |
settings.DEFAULT_CONTAINER_FORMAT) in \ |
| 344 | 311 |
settings.ALLOWED_CONTAINER_FORMATS, "Invalid container_format" |
| 345 | 312 |
|
| 313 |
container = settings.DEFAULT_PLANKTON_CONTAINER |
|
| 346 | 314 |
filename = params.pop('filename', name)
|
| 347 |
location = 'pithos://%s/%s/%s' % (self.user, self.container, filename)
|
|
| 315 |
location = 'pithos://%s/%s/%s' % (self.user, container, filename) |
|
| 348 | 316 |
is_public = params.pop('is_public', False)
|
| 349 | 317 |
permissions = {'read': ['*']} if is_public else {}
|
| 350 | 318 |
size = params.pop('size', None)
|
| ... | ... | |
| 375 | 343 |
meta = self._get_meta(location) |
| 376 | 344 |
assert meta, "File not found" |
| 377 | 345 |
|
| 378 |
size = params.pop('size', meta['bytes'])
|
|
| 346 |
size = int(params.pop('size', meta['bytes']))
|
|
| 379 | 347 |
if size != meta['bytes']: |
| 348 |
print repr(size) |
|
| 349 |
print repr(meta['bytes']) |
|
| 380 | 350 |
raise BackendException("Invalid size")
|
| 381 | 351 |
|
| 382 | 352 |
checksum = params.pop('checksum', meta['hash'])
|
| ... | ... | |
| 395 | 365 |
return self._get_image(location) |
| 396 | 366 |
|
| 397 | 367 |
def remove_user(self, image_id, user): |
| 398 |
image = self.get_meta(image_id)
|
|
| 368 |
image = self.get_image(image_id)
|
|
| 399 | 369 |
assert image, "Image not found" |
| 400 | 370 |
|
| 401 | 371 |
location = image['location'] |
| ... | ... | |
| 407 | 377 |
self._update_permissions(location, permissions) |
| 408 | 378 |
|
| 409 | 379 |
def replace_users(self, image_id, users): |
| 410 |
image = self.get_meta(image_id)
|
|
| 380 |
image = self.get_image(image_id)
|
|
| 411 | 381 |
assert image, "Image not found" |
| 412 | 382 |
|
| 413 | 383 |
location = image['location'] |
| ... | ... | |
| 418 | 388 |
self._update_permissions(location, permissions) |
| 419 | 389 |
|
| 420 | 390 |
def update(self, image_id, params): |
| 421 |
image = self.get_meta(image_id)
|
|
| 391 |
image = self.get_image(image_id)
|
|
| 422 | 392 |
assert image, "Image not found" |
| 423 | 393 |
|
| 424 | 394 |
location = image['location'] |
| ... | ... | |
| 438 | 408 |
meta.update(**params) |
| 439 | 409 |
|
| 440 | 410 |
self._update_meta(location, meta) |
| 441 |
return self.get_meta(image_id) |
|
| 411 |
return self.get_image(image_id) |
|
Also available in: Unified diff