Revision 5f01e1e6 snf-cyclades-app/synnefo/plankton/backend.py

b/snf-cyclades-app/synnefo/plankton/backend.py
86 86
    pass
87 87

  
88 88

  
89
class PithosImageBackend(object):
89
class ImageBackend(object):
90 90
    """A wrapper arround the pithos backend to simplify image handling."""
91

  
91
    
92 92
    def __init__(self, user):
93 93
        self.user = user
94

  
94
        
95 95
        original_filters = warnings.filters
96 96
        warnings.simplefilter('ignore')         # Suppress SQLAlchemy warnings
97 97
        db_connection = settings.BACKEND_DB_CONNECTION
......
99 99
        self.backend = connect_backend(db_connection=db_connection,
100 100
                                       block_path=block_path)
101 101
        warnings.filters = original_filters     # Restore warnings
102

  
102
    
103 103
    def _get_image(self, location):
104 104
        def format_timestamp(t):
105 105
            return strftime('%Y-%m-%d %H:%M:%S', gmtime(t))
106

  
106
        
107 107
        account, container, object = split_location(location)
108

  
108
        
109 109
        try:
110 110
            versions = self.backend.list_versions(self.user, account,
111 111
                    container, object)
112 112
        except NameError:
113 113
            return None
114

  
114
        
115 115
        image = {}
116

  
116
        
117 117
        meta = self._get_meta(location)
118 118
        if meta:
119 119
            image['deleted_at'] = ''
......
122 122
            version, timestamp = versions[-1]
123 123
            meta = self._get_meta(location, version)
124 124
            image['deleted_at'] = format_timestamp(timestamp)
125

  
125
        
126 126
        if PLANKTON_PREFIX + 'name' not in meta:
127 127
            return None     # Not a Plankton image
128

  
128
        
129 129
        permissions = self._get_permissions(location)
130

  
130
        
131 131
        image['checksum'] = meta['hash']
132 132
        image['created_at'] = format_timestamp(versions[0][1])
133 133
        image['id'] = meta['uuid']
......
138 138
        image['store'] = 'pithos'
139 139
        image['updated_at'] = format_timestamp(meta['modified'])
140 140
        image['properties'] = {}
141

  
141
        
142 142
        for key, val in meta.items():
143 143
            if not key.startswith(PLANKTON_PREFIX):
144 144
                continue
......
147 147
                val = json.loads(val)
148 148
            if key in PLANKTON_META:
149 149
                image[key] = val
150

  
150
        
151 151
        return image
152

  
152
    
153 153
    def _get_meta(self, location, version=None):
154 154
        account, container, object = split_location(location)
155 155
        try:
......
157 157
                    object, PLANKTON_DOMAIN, version)
158 158
        except NameError:
159 159
            return None
160

  
160
    
161 161
    def _get_permissions(self, location):
162 162
        account, container, object = split_location(location)
163 163
        action, path, permissions = self.backend.get_object_permissions(
164 164
                self.user, account, container, object)
165 165
        return permissions
166

  
166
    
167 167
    def _store(self, f, size=None):
168 168
        """Breaks data into blocks and stores them in the backend"""
169

  
169
        
170 170
        bytes = 0
171 171
        hashmap = []
172 172
        backend = self.backend
173 173
        blocksize = backend.block_size
174

  
174
        
175 175
        data = f.read(blocksize)
176 176
        while data:
177 177
            hash = backend.put_block(data)
178 178
            hashmap.append(hash)
179 179
            bytes += len(data)
180 180
            data = f.read(blocksize)
181

  
181
        
182 182
        if size and size != bytes:
183 183
            raise BackendException("Invalid size")
184

  
184
        
185 185
        return hashmap, bytes
186

  
186
    
187 187
    def _update(self, location, size, hashmap, meta, permissions):
188 188
        account, container, object = split_location(location)
189 189
        self.backend.update_object_hashmap(self.user, account, container,
190 190
                object, size, hashmap, '', PLANKTON_DOMAIN,
191 191
                permissions=permissions)
192 192
        self._update_meta(location, meta, replace=True)
193

  
193
    
194 194
    def _update_meta(self, location, meta, replace=False):
195 195
        account, container, object = split_location(location)
196

  
196
        
197 197
        prefixed = {}
198 198
        for key, val in meta.items():
199 199
            if key == 'properties':
200 200
                val = json.dumps(val)
201 201
            if key in PLANKTON_META:
202 202
                prefixed[PLANKTON_PREFIX + key] = val
203

  
203
        
204 204
        self.backend.update_object_meta(self.user, account, container, object,
205 205
                PLANKTON_DOMAIN, prefixed, replace)
206

  
206
    
207 207
    def _update_permissions(self, location, permissions):
208 208
        account, container, object = split_location(location)
209 209
        self.backend.update_object_permissions(self.user, account, container,
210 210
                object, permissions)
211

  
211
    
212 212
    def add_user(self, image_id, user):
213 213
        image = self.get_image(image_id)
214 214
        assert image, "Image not found"
215

  
215
        
216 216
        location = image['location']
217 217
        permissions = self._get_permissions(location)
218 218
        read = set(permissions.get('read', []))
219 219
        read.add(user)
220 220
        permissions['read'] = list(read)
221 221
        self._update_permissions(location, permissions)
222

  
222
    
223 223
    def close(self):
224 224
        self.backend.close()
225

  
225
    
226 226
    def delete(self, image_id):
227 227
        image = self.get_image(image_id)
228 228
        account, container, object = split_location(image['location'])
229 229
        self.backend.delete_object(self.user, account, container, object)
230

  
230
    
231 231
    def get_data(self, location):
232 232
        account, container, object = split_location(location)
233 233
        size, hashmap = self.backend.get_object_hashmap(self.user, account,
......
235 235
        data = ''.join(self.backend.get_block(hash) for hash in hashmap)
236 236
        assert len(data) == size
237 237
        return data
238

  
238
    
239 239
    def get_image(self, image_id):
240 240
        try:
241 241
            account, container, object = self.backend.get_uuid(self.user,
242 242
                    image_id)
243 243
        except NameError:
244 244
            return None
245

  
245
        
246 246
        location = get_location(account, container, object)
247 247
        return self._get_image(location)
248

  
248
    
249 249
    def iter(self):
250 250
        """Iter over all images available to the user"""
251

  
251
        
252 252
        backend = self.backend
253 253
        for account in backend.list_accounts(self.user):
254 254
            for container in backend.list_containers(self.user, account,
......
259 259
                    image = self._get_image(location)
260 260
                    if image:
261 261
                        yield image
262

  
262
    
263 263
    def iter_public(self, filters=None):
264 264
        filters = filters or {}
265 265
        backend = self.backend
266

  
266
        
267 267
        keys = [PLANKTON_PREFIX + 'name']
268 268
        size_range = (None, None)
269

  
269
        
270 270
        for key, val in filters.items():
271 271
            if key == 'size_min':
272 272
                size_range = (int(val), size_range[1])
......
274 274
                size_range = (size_range[0], int(val))
275 275
            else:
276 276
                keys.append('%s = %s' % (PLANKTON_PREFIX + key, val))
277

  
277
        
278 278
        for account in backend.list_accounts(None):
279 279
            for container in backend.list_containers(None, account,
280 280
                                                     shared=True):
......
285 285
                    image = self._get_image(location)
286 286
                    if image:
287 287
                        yield image
288

  
288
    
289 289
    def iter_shared(self, member):
290 290
        """Iterate over image ids shared to this member"""
291

  
291
        
292 292
        backend = self.backend
293

  
293
        
294 294
        # To get the list we connect as member and get the list shared by us
295 295
        for container in  backend.list_containers(member, self.user):
296 296
            for object, version_id in backend.list_objects(member, self.user,
......
303 303
                        yield meta['uuid']
304 304
                except (NameError, NotAllowedError):
305 305
                    continue
306

  
306
    
307 307
    def list(self):
308 308
        """Iter over all images available to the user"""
309

  
309
        
310 310
        return list(self.iter())
311

  
311
    
312 312
    def list_public(self, filters, params):
313 313
        images = list(self.iter_public(filters))
314 314
        key = itemgetter(params.get('sort_key', 'created_at'))
315 315
        reverse = params.get('sort_dir', 'desc') == 'desc'
316 316
        images.sort(key=key, reverse=reverse)
317 317
        return images
318

  
318
    
319 319
    def list_users(self, image_id):
320 320
        image = self.get_image(image_id)
321 321
        assert image, "Image not found"
322

  
322
        
323 323
        permissions = self._get_permissions(image['location'])
324 324
        return [user for user in permissions.get('read', []) if user != '*']
325

  
325
    
326 326
    def put(self, name, f, params):
327 327
        assert 'checksum' not in params, "Passing a checksum is not supported"
328 328
        assert 'id' not in params, "Passing an ID is not supported"
......
333 333
        assert params.setdefault('container_format',
334 334
                settings.DEFAULT_CONTAINER_FORMAT) in \
335 335
                settings.ALLOWED_CONTAINER_FORMATS, "Invalid container_format"
336

  
336
        
337 337
        container = settings.DEFAULT_PLANKTON_CONTAINER
338 338
        filename = params.pop('filename', name)
339 339
        location = 'pithos://%s/%s/%s' % (self.user, container, filename)
340 340
        is_public = params.pop('is_public', False)
341 341
        permissions = {'read': ['*']} if is_public else {}
342 342
        size = params.pop('size', None)
343

  
343
        
344 344
        hashmap, size = self._store(f, size)
345

  
345
        
346 346
        meta = {}
347 347
        meta['properties'] = params.pop('properties', {})
348 348
        meta.update(name=name, status='available', **params)
349

  
349
        
350 350
        self._update(location, size, hashmap, meta, permissions)
351 351
        return self._get_image(location)
352

  
352
    
353 353
    def register(self, name, location, params):
354 354
        assert 'id' not in params, "Passing an ID is not supported"
355 355
        assert location.startswith('pithos://'), "Invalid location"
......
360 360
        assert params.setdefault('container_format',
361 361
                settings.DEFAULT_CONTAINER_FORMAT) in \
362 362
                settings.ALLOWED_CONTAINER_FORMATS, "Invalid container_format"
363

  
363
        
364 364
        user = self.user
365 365
        account, container, object = split_location(location)
366

  
366
        
367 367
        meta = self._get_meta(location)
368 368
        assert meta, "File not found"
369

  
369
        
370 370
        size = int(params.pop('size', meta['bytes']))
371 371
        if size != meta['bytes']:
372 372
            raise BackendException("Invalid size")
373

  
373
        
374 374
        checksum = params.pop('checksum', meta['hash'])
375 375
        if checksum != meta['hash']:
376 376
            raise BackendException("Invalid checksum")
377

  
377
        
378 378
        is_public = params.pop('is_public', False)
379 379
        permissions = {'read': ['*']} if is_public else {}
380

  
380
        
381 381
        meta = {}
382 382
        meta['properties'] = params.pop('properties', {})
383 383
        meta.update(name=name, status='available', **params)
384

  
384
        
385 385
        self._update_meta(location, meta)
386 386
        self._update_permissions(location, permissions)
387 387
        return self._get_image(location)
388

  
388
    
389 389
    def remove_user(self, image_id, user):
390 390
        image = self.get_image(image_id)
391 391
        assert image, "Image not found"
392

  
392
        
393 393
        location = image['location']
394 394
        permissions = self._get_permissions(location)
395 395
        try:
......
397 397
        except ValueError:
398 398
            return      # User did not have access anyway
399 399
        self._update_permissions(location, permissions)
400

  
400
    
401 401
    def replace_users(self, image_id, users):
402 402
        image = self.get_image(image_id)
403 403
        assert image, "Image not found"
404

  
404
        
405 405
        location = image['location']
406 406
        permissions = self._get_permissions(location)
407 407
        permissions['read'] = users
408 408
        if image.get('is_public', False):
409 409
            permissions['read'].append('*')
410 410
        self._update_permissions(location, permissions)
411

  
411
    
412 412
    def update(self, image_id, params):
413 413
        image = self.get_image(image_id)
414 414
        assert image, "Image not found"
415

  
415
        
416 416
        location = image['location']
417 417
        is_public = params.pop('is_public', None)
418 418
        if is_public is not None:
......
424 424
                read.discard('*')
425 425
            permissions['read'] = list(read)
426 426
            self.backend._update_permissions(location, permissions)
427

  
427
        
428 428
        meta = {}
429 429
        meta['properties'] = params.pop('properties', {})
430 430
        meta.update(**params)
431

  
431
        
432 432
        self._update_meta(location, meta)
433 433
        return self.get_image(image_id)
434

  
435

  
436
class DummyImageBackend():
437

  
438
    def __init__(self, user, images=None):
439
        self.user = user
440
        self.images = images or [{"status":"available","name":"Windows","checksum":"c4524d6f6e4f04ecf0212e50bb36f70114f459b223f932f1a3fa6a995e1334f7","created_at":"2012-03-28 14:39:43","disk_format":"diskdump","updated_at":"2012-03-28 16:56:34","properties":{"kernel":"Windows NT Kernel","osfamily":"windows","users":"Administrator","gui":"Windows, Aero Theme","sortorder":"7","size":"10537","os":"windows","root_partition":"2","description":"Windows 2008 R2, Aero Desktop Experience","OS":"windows"},"location":"pithos://images@okeanos.grnet.gr/pithos/windows-2008R2-7-x86_64.diskdump","container_format":"bare","owner":"images@okeanos.grnet.gr","is_public":True,"deleted_at":"","id":"34138811-299e-45db-9a53-acd5f1f7693c","size":11037310976,"metadata":{"values":{"kernel":"Windows NT Kernel","osfamily":"windows","users":"Administrator","gui":"Windows, Aero Theme","sortorder":"7","size":"10537","os":"windows","root_partition":"2","description":"Windows 2008 R2, Aero Desktop Experience","OS":"windows"}},"OS":"windows","description":"Windows 2008 R2, Aero Desktop Experience","kernel":"Windows NT Kernel","GUI":""},{"status":"available","name":"CentOS","checksum":"a7517d876f2387527f35e9d8c19cba4a37419279005354ca03bd625db1cf410e","created_at":"2012-03-28 13:37:28","disk_format":"diskdump","updated_at":"2012-03-28 16:56:33","properties":{"kernel":"2.6.32","osfamily":"linux","users":"root","gui":"No GUI","sortorder":"6","size":"601","os":"centos","root_partition":"1","description":"CentOS 6.0","OS":"centos"},"location":"pithos://images@okeanos.grnet.gr/pithos/centos-6.0-8-x86_64.diskdump","container_format":"bare","owner":"images@okeanos.grnet.gr","is_public":True,"deleted_at":"","id":"8f06e736-d890-4aed-8448-aa0e978695a9","size":628834304,"metadata":{"values":{"kernel":"2.6.32","osfamily":"linux","users":"root","gui":"No GUI","sortorder":"6","size":"601","os":"centos","root_partition":"1","description":"CentOS 6.0","OS":"centos"}},"OS":"centos","description":"CentOS 6.0","kernel":"2.6.32","GUI":""},{"status":"available","name":"Fedora","checksum":"407c7fa05c14937e213503783149aa74ad9bcfc8783ccb653a419fb86bffe0d9","created_at":"2012-03-28 13:52:45","disk_format":"diskdump","updated_at":"2012-03-28 16:56:32","properties":{"kernel":"3.1.9","osfamily":"linux","users":"root user","gui":"GNOME 3.2","sortorder":"5","size":"2641","os":"fedora","root_partition":"1","description":"Fedora 16 Desktop Edition","OS":"fedora"},"location":"debian_base-6.0-7-x86_64","container_format":"bare","owner":"images@okeanos.grnet.gr","is_public":True,"deleted_at":"","id":"1ba3666b-6e57-4d52-813b-e2a33185d12d","size":2765684736,"metadata":{"values":{"kernel":"3.1.9","osfamily":"linux","users":"root user","gui":"GNOME 3.2","sortorder":"5","size":"2641","os":"fedora","root_partition":"1","description":"Fedora 16 Desktop Edition","OS":"fedora"}},"OS":"fedora","description":"Fedora 16 Desktop Edition","kernel":"3.1.9","GUI":""},{"status":"available","name":"Kubuntu","checksum":"a149289f512d70c8f9f6acb0636d2ea9a5b5c3ec0b83e4398aed4a5678da6848","created_at":"2012-03-28 15:05:52","disk_format":"diskdump","updated_at":"2012-03-28 16:56:31","properties":{"kernel":"3.0.0","osfamily":"linux","users":"user","gui":"KDE 4.7.4","sortorder":"4","size":"2850","os":"kubuntu","root_partition":"1","description":"Kubuntu 11.10","OS":"kubuntu"},"location":"pithos://images@okeanos.grnet.gr/pithos/kubuntu-11.10-1-x86_64.diskdump","container_format":"bare","owner":"images@okeanos.grnet.gr","is_public":True,"deleted_at":"","id":"79d24739-af8f-436b-8f6e-eb2d908e0b7e","size":2985041920,"metadata":{"values":{"kernel":"3.0.0","osfamily":"linux","users":"user","gui":"KDE 4.7.4","sortorder":"4","size":"2850","os":"kubuntu","root_partition":"1","description":"Kubuntu 11.10","OS":"kubuntu"}},"OS":"kubuntu","description":"Kubuntu 11.10","kernel":"3.0.0","GUI":""},{"status":"available","name":"Ubuntu","checksum":"f508e1fc8d9cbbd360a9cfc3a68e475933063c77691dac652cb7a5c824791e1b","created_at":"2012-03-28 14:08:35","disk_format":"diskdump","updated_at":"2012-03-28 16:58:10","properties":{"kernel":"3.0.0","osfamily":"linux","users":"user","gui":"Unity 4.22","sortorder":"3","size":"2540","os":"ubuntu","root_partition":"1","description":"Ubuntu 11.10","OS":"ubuntu"},"location":"pithos://images@okeanos.grnet.gr/pithos/ubuntu-11.10-1-x86_64.diskdump","container_format":"bare","owner":"images@okeanos.grnet.gr","is_public":True,"deleted_at":"","id":"d8317451-820a-4c41-917b-665492ab0f81","size":2660171776,"metadata":{"values":{"kernel":"3.0.0","osfamily":"linux","users":"user","gui":"Unity 4.22","sortorder":"3","size":"2540","os":"ubuntu","root_partition":"1","description":"Ubuntu 11.10","OS":"ubuntu"}},"OS":"ubuntu","description":"Ubuntu 11.10","kernel":"3.0.0","GUI":""},{"status":"available","name":"Debian Desktop","checksum":"a7bea0bf6815168a281b505454cd9b2f07ffb9cad0d92e08e950a633c0f05bd2","created_at":"2012-03-28 14:55:37","disk_format":"diskdump","updated_at":"2012-04-03 15:52:09","properties":{"kernel":"2.6.32","osfamily":"linux","users":"root user","gui":"GNOME 2.30","sortorder":"2","size":"3314","os":"debian","root_partition":"1","description":"Debian Squeeze Desktop","OS":"debian"},"location":"pithos://images@okeanos.grnet.gr/pithos/debian_desktop-6.0-6-x86_64.diskdump","container_format":"bare","owner":"images@okeanos.grnet.gr","is_public":True,"deleted_at":"","id":"d7970a84-99b9-40be-b790-ce61001a5b9a","size":3482439680,"metadata":{"values":{"kernel":"2.6.32","osfamily":"linux","users":"root user","gui":"GNOME 2.30","sortorder":"2","size":"3314","os":"debian","root_partition":"1","description":"Debian Squeeze Desktop","OS":"debian"}},"OS":"debian","description":"Debian Squeeze Desktop","kernel":"2.6.32","GUI":""},{"status":"available","name":"Debian Base","checksum":"65352163c9842d6fbc5717437811495f163da2338c807819a1d4d7a50766e56c","created_at":"2012-03-28 13:39:38","disk_format":"diskdump","updated_at":"2012-03-28 16:56:29","properties":{"kernel":"2.6.32","osfamily":"linux","users":"root","gui":"No GUI","sortorder":"1","size":"451","os":"debian","root_partition":"1","description":"Debian Squeeze Base System","OS":"debian"},"location":"pithos://images@okeanos.grnet.gr/pithos/debian_base-6.0-7-x86_64.diskdump","container_format":"bare","owner":"images@okeanos.grnet.gr","is_public":True,"deleted_at":"","id":"49971ade-3bbc-4700-9a84-b3ca00133850","size":471891968,"metadata":{"values":{"kernel":"2.6.32","osfamily":"linux","users":"root","gui":"No GUI","sortorder":"1","size":"451","os":"debian","root_partition":"1","description":"Debian Squeeze Base System","OS":"debian"}},"OS":"debian","description":"Debian Squeeze Base System","kernel":"2.6.32","GUI":""},{"id":"20","name":"(deleted image)","size":-1,"progress":100,"status":"DELETED"},{"id":"21","name":"(deleted image)","size":-1,"progress":100,"status":"DELETED"}]
441

  
442

  
443
    def iter(self):
444
        return self.images
445

  
446
    def get_image(self, image_id):
447
        for i in self.images:
448
            if i['id'] == image_id:
449
                return i
450
        return None
451

  
452
    def close(self):
453
        pass
454

  
455
    def list_public(self, filters, params):
456
        return self.images
457

  
458

  
459
ImageBackend = PithosImageBackend
460
ImageBackend = DummyImageBackend
461

  

Also available in: Unified diff