Revision c6dda6cd

b/snf-cyclades-app/synnefo/app_settings/default/api.py
136 136
MAX_PERSONALITY_SIZE = 10240
137 137

  
138 138
# Available storage types to be used as disk templates
139
# Use ext_<provider_name> to map specific provider for `ext` disk template.
139 140
GANETI_DISK_TEMPLATES = ('blockdev', 'diskless', 'drbd', 'file', 'plain',
140 141
                         'rbd',  'sharedfile')
141 142
DEFAULT_GANETI_DISK_TEMPLATE = 'drbd'
b/snf-cyclades-app/synnefo/plankton/backend.py
86 86
    pass
87 87

  
88 88

  
89
class ImageBackend(object):
89
class PithosImageBackend(object):
90 90
    """A wrapper arround the pithos backend to simplify image handling."""
91
    
91

  
92 92
    def __init__(self, user):
93 93
        self.user = user
94
        
94

  
95 95
        original_filters = warnings.filters
96 96
        warnings.simplefilter('ignore')         # Suppress SQLAlchemy warnings
97 97
        db_connection = settings.BACKEND_DB_CONNECTION
......
99 99
        self.backend = connect_backend(db_connection=db_connection,
100 100
                                       block_path=block_path)
101 101
        warnings.filters = original_filters     # Restore warnings
102
    
102

  
103 103
    def _get_image(self, location):
104 104
        def format_timestamp(t):
105 105
            return strftime('%Y-%m-%d %H:%M:%S', gmtime(t))
106
        
106

  
107 107
        account, container, object = split_location(location)
108
        
108

  
109 109
        try:
110 110
            versions = self.backend.list_versions(self.user, account,
111 111
                    container, object)
112 112
        except NameError:
113 113
            return None
114
        
114

  
115 115
        image = {}
116
        
116

  
117 117
        meta = self._get_meta(location)
118 118
        if meta:
119 119
            image['deleted_at'] = ''
......
122 122
            version, timestamp = versions[-1]
123 123
            meta = self._get_meta(location, version)
124 124
            image['deleted_at'] = format_timestamp(timestamp)
125
        
125

  
126 126
        if PLANKTON_PREFIX + 'name' not in meta:
127 127
            return None     # Not a Plankton image
128
        
128

  
129 129
        permissions = self._get_permissions(location)
130
        
130

  
131 131
        image['checksum'] = meta['hash']
132 132
        image['created_at'] = format_timestamp(versions[0][1])
133 133
        image['id'] = meta['uuid']
......
138 138
        image['store'] = 'pithos'
139 139
        image['updated_at'] = format_timestamp(meta['modified'])
140 140
        image['properties'] = {}
141
        
141

  
142 142
        for key, val in meta.items():
143 143
            if not key.startswith(PLANKTON_PREFIX):
144 144
                continue
......
147 147
                val = json.loads(val)
148 148
            if key in PLANKTON_META:
149 149
                image[key] = val
150
        
150

  
151 151
        return image
152
    
152

  
153 153
    def _get_meta(self, location, version=None):
154 154
        account, container, object = split_location(location)
155 155
        try:
......
157 157
                    object, PLANKTON_DOMAIN, version)
158 158
        except NameError:
159 159
            return None
160
    
160

  
161 161
    def _get_permissions(self, location):
162 162
        account, container, object = split_location(location)
163 163
        action, path, permissions = self.backend.get_object_permissions(
164 164
                self.user, account, container, object)
165 165
        return permissions
166
    
166

  
167 167
    def _store(self, f, size=None):
168 168
        """Breaks data into blocks and stores them in the backend"""
169
        
169

  
170 170
        bytes = 0
171 171
        hashmap = []
172 172
        backend = self.backend
173 173
        blocksize = backend.block_size
174
        
174

  
175 175
        data = f.read(blocksize)
176 176
        while data:
177 177
            hash = backend.put_block(data)
178 178
            hashmap.append(hash)
179 179
            bytes += len(data)
180 180
            data = f.read(blocksize)
181
        
181

  
182 182
        if size and size != bytes:
183 183
            raise BackendException("Invalid size")
184
        
184

  
185 185
        return hashmap, bytes
186
    
186

  
187 187
    def _update(self, location, size, hashmap, meta, permissions):
188 188
        account, container, object = split_location(location)
189 189
        self.backend.update_object_hashmap(self.user, account, container,
190 190
                object, size, hashmap, '', PLANKTON_DOMAIN,
191 191
                permissions=permissions)
192 192
        self._update_meta(location, meta, replace=True)
193
    
193

  
194 194
    def _update_meta(self, location, meta, replace=False):
195 195
        account, container, object = split_location(location)
196
        
196

  
197 197
        prefixed = {}
198 198
        for key, val in meta.items():
199 199
            if key == 'properties':
200 200
                val = json.dumps(val)
201 201
            if key in PLANKTON_META:
202 202
                prefixed[PLANKTON_PREFIX + key] = val
203
        
203

  
204 204
        self.backend.update_object_meta(self.user, account, container, object,
205 205
                PLANKTON_DOMAIN, prefixed, replace)
206
    
206

  
207 207
    def _update_permissions(self, location, permissions):
208 208
        account, container, object = split_location(location)
209 209
        self.backend.update_object_permissions(self.user, account, container,
210 210
                object, permissions)
211
    
211

  
212 212
    def add_user(self, image_id, user):
213 213
        image = self.get_image(image_id)
214 214
        assert image, "Image not found"
215
        
215

  
216 216
        location = image['location']
217 217
        permissions = self._get_permissions(location)
218 218
        read = set(permissions.get('read', []))
219 219
        read.add(user)
220 220
        permissions['read'] = list(read)
221 221
        self._update_permissions(location, permissions)
222
    
222

  
223 223
    def close(self):
224 224
        self.backend.close()
225
    
225

  
226 226
    def delete(self, image_id):
227 227
        image = self.get_image(image_id)
228 228
        account, container, object = split_location(image['location'])
229 229
        self.backend.delete_object(self.user, account, container, object)
230
    
230

  
231 231
    def get_data(self, location):
232 232
        account, container, object = split_location(location)
233 233
        size, hashmap = self.backend.get_object_hashmap(self.user, account,
......
235 235
        data = ''.join(self.backend.get_block(hash) for hash in hashmap)
236 236
        assert len(data) == size
237 237
        return data
238
    
238

  
239 239
    def get_image(self, image_id):
240 240
        try:
241 241
            account, container, object = self.backend.get_uuid(self.user,
242 242
                    image_id)
243 243
        except NameError:
244 244
            return None
245
        
245

  
246 246
        location = get_location(account, container, object)
247 247
        return self._get_image(location)
248
    
248

  
249 249
    def iter(self):
250 250
        """Iter over all images available to the user"""
251
        
251

  
252 252
        backend = self.backend
253 253
        for account in backend.list_accounts(self.user):
254 254
            for container in backend.list_containers(self.user, account,
......
259 259
                    image = self._get_image(location)
260 260
                    if image:
261 261
                        yield image
262
    
262

  
263 263
    def iter_public(self, filters=None):
264 264
        filters = filters or {}
265 265
        backend = self.backend
266
        
266

  
267 267
        keys = [PLANKTON_PREFIX + 'name']
268 268
        size_range = (None, None)
269
        
269

  
270 270
        for key, val in filters.items():
271 271
            if key == 'size_min':
272 272
                size_range = (int(val), size_range[1])
......
274 274
                size_range = (size_range[0], int(val))
275 275
            else:
276 276
                keys.append('%s = %s' % (PLANKTON_PREFIX + key, val))
277
        
277

  
278 278
        for account in backend.list_accounts(None):
279 279
            for container in backend.list_containers(None, account,
280 280
                                                     shared=True):
......
285 285
                    image = self._get_image(location)
286 286
                    if image:
287 287
                        yield image
288
    
288

  
289 289
    def iter_shared(self, member):
290 290
        """Iterate over image ids shared to this member"""
291
        
291

  
292 292
        backend = self.backend
293
        
293

  
294 294
        # To get the list we connect as member and get the list shared by us
295 295
        for container in  backend.list_containers(member, self.user):
296 296
            for object, version_id in backend.list_objects(member, self.user,
......
303 303
                        yield meta['uuid']
304 304
                except (NameError, NotAllowedError):
305 305
                    continue
306
    
306

  
307 307
    def list(self):
308 308
        """Iter over all images available to the user"""
309
        
309

  
310 310
        return list(self.iter())
311
    
311

  
312 312
    def list_public(self, filters, params):
313 313
        images = list(self.iter_public(filters))
314 314
        key = itemgetter(params.get('sort_key', 'created_at'))
315 315
        reverse = params.get('sort_dir', 'desc') == 'desc'
316 316
        images.sort(key=key, reverse=reverse)
317 317
        return images
318
    
318

  
319 319
    def list_users(self, image_id):
320 320
        image = self.get_image(image_id)
321 321
        assert image, "Image not found"
322
        
322

  
323 323
        permissions = self._get_permissions(image['location'])
324 324
        return [user for user in permissions.get('read', []) if user != '*']
325
    
325

  
326 326
    def put(self, name, f, params):
327 327
        assert 'checksum' not in params, "Passing a checksum is not supported"
328 328
        assert 'id' not in params, "Passing an ID is not supported"
......
333 333
        assert params.setdefault('container_format',
334 334
                settings.DEFAULT_CONTAINER_FORMAT) in \
335 335
                settings.ALLOWED_CONTAINER_FORMATS, "Invalid container_format"
336
        
336

  
337 337
        container = settings.DEFAULT_PLANKTON_CONTAINER
338 338
        filename = params.pop('filename', name)
339 339
        location = 'pithos://%s/%s/%s' % (self.user, container, filename)
340 340
        is_public = params.pop('is_public', False)
341 341
        permissions = {'read': ['*']} if is_public else {}
342 342
        size = params.pop('size', None)
343
        
343

  
344 344
        hashmap, size = self._store(f, size)
345
        
345

  
346 346
        meta = {}
347 347
        meta['properties'] = params.pop('properties', {})
348 348
        meta.update(name=name, status='available', **params)
349
        
349

  
350 350
        self._update(location, size, hashmap, meta, permissions)
351 351
        return self._get_image(location)
352
    
352

  
353 353
    def register(self, name, location, params):
354 354
        assert 'id' not in params, "Passing an ID is not supported"
355 355
        assert location.startswith('pithos://'), "Invalid location"
......
360 360
        assert params.setdefault('container_format',
361 361
                settings.DEFAULT_CONTAINER_FORMAT) in \
362 362
                settings.ALLOWED_CONTAINER_FORMATS, "Invalid container_format"
363
        
363

  
364 364
        user = self.user
365 365
        account, container, object = split_location(location)
366
        
366

  
367 367
        meta = self._get_meta(location)
368 368
        assert meta, "File not found"
369
        
369

  
370 370
        size = int(params.pop('size', meta['bytes']))
371 371
        if size != meta['bytes']:
372 372
            raise BackendException("Invalid size")
373
        
373

  
374 374
        checksum = params.pop('checksum', meta['hash'])
375 375
        if checksum != meta['hash']:
376 376
            raise BackendException("Invalid checksum")
377
        
377

  
378 378
        is_public = params.pop('is_public', False)
379 379
        permissions = {'read': ['*']} if is_public else {}
380
        
380

  
381 381
        meta = {}
382 382
        meta['properties'] = params.pop('properties', {})
383 383
        meta.update(name=name, status='available', **params)
384
        
384

  
385 385
        self._update_meta(location, meta)
386 386
        self._update_permissions(location, permissions)
387 387
        return self._get_image(location)
388
    
388

  
389 389
    def remove_user(self, image_id, user):
390 390
        image = self.get_image(image_id)
391 391
        assert image, "Image not found"
392
        
392

  
393 393
        location = image['location']
394 394
        permissions = self._get_permissions(location)
395 395
        try:
......
397 397
        except ValueError:
398 398
            return      # User did not have access anyway
399 399
        self._update_permissions(location, permissions)
400
    
400

  
401 401
    def replace_users(self, image_id, users):
402 402
        image = self.get_image(image_id)
403 403
        assert image, "Image not found"
404
        
404

  
405 405
        location = image['location']
406 406
        permissions = self._get_permissions(location)
407 407
        permissions['read'] = users
408 408
        if image.get('is_public', False):
409 409
            permissions['read'].append('*')
410 410
        self._update_permissions(location, permissions)
411
    
411

  
412 412
    def update(self, image_id, params):
413 413
        image = self.get_image(image_id)
414 414
        assert image, "Image not found"
415
        
415

  
416 416
        location = image['location']
417 417
        is_public = params.pop('is_public', None)
418 418
        if is_public is not None:
......
424 424
                read.discard('*')
425 425
            permissions['read'] = list(read)
426 426
            self.backend._update_permissions(location, permissions)
427
        
427

  
428 428
        meta = {}
429 429
        meta['properties'] = params.pop('properties', {})
430 430
        meta.update(**params)
431
        
431

  
432 432
        self._update_meta(location, meta)
433 433
        return self.get_image(image_id)
434

  
435

  
436

  
437
IMAGES = [
438
{
439
    "status": "available",
440
    "name": "Local test image",
441
    "checksum": "a149289f512d70c8f9f6acb0636d2ea9a5b5c3ec0b83e4398aed4a5678da6848",
442
    "created_at": "2012-03-28 15:05:52",
443
    "disk_format": "diskdump",
444
    "updated_at": "2012-03-28 16:56:31",
445
    "properties": {
446
        "kernel": "3.0.0",
447
        "osfamily": "linux",
448
        "users": "user",
449
        "gui": "KDE 4.7.4",
450
        "sortorder": "4",
451
        "size": "2850",
452
        "os": "kpap",
453
        "root_partition": "1",
454
        "description": "Kubuntu 11.10"
455
    },
456
    "location": "debian_base-6.0-7-x86_64",
457
    "container_format": "bare",
458
    "owner": "images@okeanos.grnet.gr",
459
    "is_public": True,
460
    "deleted_at": "",
461
    "id": "79d24739-af8f-436b-8f6e-eb2d908e0b7e",
462
    "size": 2985041920
463
},
464
{
465
    "status": "available",
466
    "name": "Local test image",
467
    "checksum": "a149289f512d70c8f9f6acb0636d2ea9a5b5c3ec0b83e4398aed4a5678da6848",
468
    "created_at": "2012-03-28 15:05:52",
469
    "disk_format": "diskdump",
470
    "updated_at": "2012-03-28 16:56:31",
471
    "properties": {
472
        "kernel": "3.0.0",
473
        "osfamily": "linux",
474
        "users": "user",
475
        "gui": "KDE 4.7.4",
476
        "sortorder": "4",
477
        "size": "2850",
478
        "os": "kpap",
479
        "root_partition": "1",
480
        "description": "Kubuntu 11.10"
481
    },
482
    "location": "debian_base-6.0-7-x86_64",
483
    "container_format": "bare",
484
    "owner": "admin",
485
    "is_public": True,
486
    "deleted_at": "",
487
    "id": "79d24739-af8f-436b-8f6e-eb2d908e0b74",
488
    "size": 2985041920
489
},
490
{
491
    "status": "available",
492
    "name": "Test image (extra metadata)",
493
    "checksum": "a149289f512d70c8f9f6acb0636d2ea9a5b5c3ec0b83e4398aed4a5678da6848",
494
    "created_at": "2012-03-28 15:05:52",
495
    "disk_format": "diskdump",
496
    "updated_at": "2012-03-28 16:56:31",
497
    "properties": {
498
        "kernel": "3.0.0",
499
        "osfamily": "linux",
500
        "users": "user takis",
501
        "gui": "KDE 4.7.4",
502
        "sortorder": "4",
503
        "size": "2850",
504
        "root_partition": "1",
505
        "metadata_key": "lal alal",
506
        "metadata_key2": "test llalalalala",
507
    },
508
    "location": "debian_base-6.0-7-x86_64",
509
    "container_format": "bare",
510
    "owner": "admin",
511
    "is_public": True,
512
    "deleted_at": "",
513
    "id": "79d24739-af8f-436b-8f6e-eb2d908e0b72",
514
    "size": 2985041920
515
},
516
{
517
    "status": "available",
518
    "name": "Test image (no os)",
519
    "checksum": "a149289f512d70c8f9f6acb0636d2ea9a5b5c3ec0b83e4398aed4a5678da6848",
520
    "created_at": "2012-03-28 15:05:52",
521
    "disk_format": "diskdump",
522
    "updated_at": "2012-03-28 16:56:31",
523
    "properties": {
524
        "kernel": "3.0.0",
525
        "osfamily": "linux",
526
        "users": "user",
527
        "gui": "KDE 4.7.4",
528
        "sortorder": "4",
529
        "size": "2850",
530
        "root_partition": "1",
531
        "description": "Kubuntu 11.10"
532
    },
533
    "location": "debian_base-6.0-7-x86_64",
534
    "container_format": "bare",
535
    "owner": "admin",
536
    "is_public": True,
537
    "deleted_at": "",
538
    "id": "79d24739-af8f-436b-8f6e-eb2d908e0b71",
539
    "size": 30000000000
540
},
541
{
542
    "status": "available",
543
    "name": "Test image (no os)",
544
    "checksum": "a149289f512d70c8f9f6acb0636d2ea9a5b5c3ec0b83e4398aed4a5678da6848",
545
    "created_at": "2012-03-28 15:05:52",
546
    "disk_format": "diskdump",
547
    "updated_at": "2012-03-28 16:56:31",
548
    "properties": {
549
        "kernel": "3.0.0",
550
        "osfamily": "linux",
551
        "users": "user root",
552
        "gui": "KDE 4.7.4",
553
        "sortorder": "4",
554
        "size": "2850",
555
        "root_partition": "1",
556
        "description": "Kubuntu 11.10"
557
    },
558
    "location": "debian_base-6.0-7-x86_64",
559
    "container_format": "bare",
560
    "owner": "admin@admin.com",
561
    "is_public": True,
562
    "deleted_at": "",
563
    "id": "79d24739-af8f-436b-8f6e-eb2d908e0b55",
564
    "size": 49850419200
565
},
566
{
567
    "status": "available",
568
    "name": "Test image",
569
    "checksum": "a149289f512d70c8f9f6acb0636d2ea9a5b5c3ec0b83e4398aed4a5678da6848",
570
    "created_at": "2012-03-28 15:05:52",
571
    "disk_format": "diskdump",
572
    "updated_at": "2012-03-28 16:56:31",
573
    "properties": {
574
        "kernel": "3.0.0",
575
        "osfamily": "linux",
576
        "users": "user root",
577
        "gui": "KDE 4.7.4",
578
        "os": "ubuntu",
579
        "sortorder": "4",
580
        "size": "2850",
581
        "root_partition": "1",
582
        "description": "Kubuntu 11.10 <h1>TEST</h1>"
583
    },
584
    "location": "debian_base-6.0-7-x86_64",
585
    "container_format": "bare",
586
    "owner": "admin",
587
    "is_public": True,
588
    "deleted_at": "",
589
    "id": "79d24739-af8f-436b-8f6e-eb2d908e0b79",
590
    "size": 49850419200
591
},
592

  
593
]
594

  
595
class DummyImageBackend():
596

  
597
    def __init__(self, user, images=None):
598
        self.user = user
599
        self.images = images or IMAGES
600

  
601

  
602
    def iter(self):
603
        return self.images
604

  
605
    def get_image(self, image_id):
606
        for i in self.images:
607
            if i['id'] == image_id:
608
                return i
609
        return None
610

  
611
    def close(self):
612
        pass
613

  
614
    def list_public(self, filters, params):
615
        return self.images
616

  
617

  
618
ImageBackend = PithosImageBackend
619
ImageBackend = DummyImageBackend

Also available in: Unified diff