Revision 1f96b68d snf-pithos-backend/pithos/backends/modular.py

b/snf-pithos-backend/pithos/backends/modular.py
37 37
import hashlib
38 38
import binascii
39 39

  
40
from functools import wraps
41
from traceback import format_exc
42

  
40 43
try:
41 44
    from astakosclient import AstakosClient
42 45
except ImportError:
......
120 123
logger = logging.getLogger(__name__)
121 124

  
122 125

  
126
def debug_method(func):
127
    @wraps(func)
128
    def wrapper(self, *args, **kw):
129
        try:
130
            result = func(self, *args, **kw)
131
            return result
132
        except:
133
            result = format_exc()
134
            raise
135
        finally:
136
            all_args = [str(i) for i in args]
137
            map(all_args.append, ('%s=%s' % (k, v) for k, v in kw.iteritems()))
138
            logger.debug(">>> %s(%s) <<< %s" % (
139
                func.__name__, ', '.join(all_args).rstrip(', '), result))
140
    return wrapper
141

  
142

  
123 143
class ModularBackend(BaseBackend):
124 144
    """A modular backend.
125 145

  
......
276 296
    def using_external_quotaholder(self):
277 297
        return not isinstance(self.astakosclient, DisabledAstakosClient)
278 298

  
299
    @debug_method
279 300
    def list_accounts(self, user, marker=None, limit=10000):
280 301
        """Return a list of accounts the user can access."""
281 302

  
282
        logger.debug("list_accounts: %s %s %s", user, marker, limit)
283 303
        allowed = self._allowed_accounts(user)
284 304
        start, limit = self._list_limits(allowed, marker, limit)
285 305
        return allowed[start:start + limit]
286 306

  
307
    @debug_method
287 308
    def get_account_meta(
288 309
            self, user, account, domain, until=None, include_user_defined=True,
289 310
            external_quota=None):
290 311
        """Return a dictionary with the account metadata for the domain."""
291 312

  
292
        logger.debug(
293
            "get_account_meta: %s %s %s %s", user, account, domain, until)
294 313
        path, node = self._lookup_account(account, user == account)
295 314
        if user != account:
296 315
            if until or (node is None) or (account not
......
327 346
        meta.update({'modified': modified})
328 347
        return meta
329 348

  
349
    @debug_method
330 350
    def update_account_meta(self, user, account, domain, meta, replace=False):
331 351
        """Update the metadata associated with the account for the domain."""
332 352

  
333
        logger.debug("update_account_meta: %s %s %s %s %s", user,
334
                     account, domain, meta, replace)
335 353
        if user != account:
336 354
            raise NotAllowedError
337 355
        path, node = self._lookup_account(account, True)
338 356
        self._put_metadata(user, node, domain, meta, replace,
339 357
                           update_statistics_ancestors_depth=-1)
340 358

  
359
    @debug_method
341 360
    def get_account_groups(self, user, account):
342 361
        """Return a dictionary with the user groups defined for the account."""
343 362

  
344
        logger.debug("get_account_groups: %s %s", user, account)
345 363
        if user != account:
346 364
            if account not in self._allowed_accounts(user):
347 365
                raise NotAllowedError
......
349 367
        self._lookup_account(account, True)
350 368
        return self.permissions.group_dict(account)
351 369

  
370
    @debug_method
352 371
    def update_account_groups(self, user, account, groups, replace=False):
353 372
        """Update the groups associated with the account."""
354 373

  
355
        logger.debug("update_account_groups: %s %s %s %s", user,
356
                     account, groups, replace)
357 374
        if user != account:
358 375
            raise NotAllowedError
359 376
        self._lookup_account(account, True)
......
366 383
            if v:
367 384
                self.permissions.group_addmany(account, k, v)
368 385

  
386
    @debug_method
369 387
    def get_account_policy(self, user, account, external_quota=None):
370 388
        """Return a dictionary with the account policy."""
371 389

  
372
        logger.debug("get_account_policy: %s %s", user, account)
373 390
        if user != account:
374 391
            if account not in self._allowed_accounts(user):
375 392
                raise NotAllowedError
......
381 398
            policy['quota'] = external_quota.get('limit', 0)
382 399
        return policy
383 400

  
401
    @debug_method
384 402
    def update_account_policy(self, user, account, policy, replace=False):
385 403
        """Update the policy associated with the account."""
386 404

  
387
        logger.debug("update_account_policy: %s %s %s %s", user,
388
                     account, policy, replace)
389 405
        if user != account:
390 406
            raise NotAllowedError
391 407
        path, node = self._lookup_account(account, True)
392 408
        self._check_policy(policy, is_account_policy=True)
393 409
        self._put_policy(node, policy, replace, is_account_policy=True)
394 410

  
411
    @debug_method
395 412
    def put_account(self, user, account, policy=None):
396 413
        """Create a new account with the given name."""
397 414

  
398
        logger.debug("put_account: %s %s %s", user, account, policy)
399 415
        policy = policy or {}
400 416
        if user != account:
401 417
            raise NotAllowedError
......
408 424
                              update_statistics_ancestors_depth=-1)
409 425
        self._put_policy(node, policy, True, is_account_policy=True)
410 426

  
427
    @debug_method
411 428
    def delete_account(self, user, account):
412 429
        """Delete the account with the given name."""
413 430

  
414
        logger.debug("delete_account: %s %s", user, account)
415 431
        if user != account:
416 432
            raise NotAllowedError
417 433
        node = self.node.node_lookup(account)
......
422 438
            raise AccountNotEmpty('Account is not empty')
423 439
        self.permissions.group_destroy(account)
424 440

  
441
    @debug_method
425 442
    def list_containers(self, user, account, marker=None, limit=10000,
426 443
                        shared=False, until=None, public=False):
427 444
        """Return a list of containers existing under an account."""
428 445

  
429
        logger.debug("list_containers: %s %s %s %s %s %s %s", user,
430
                     account, marker, limit, shared, until, public)
431 446
        if user != account:
432 447
            if until or account not in self._allowed_accounts(user):
433 448
                raise NotAllowedError
......
452 467
            [x[0] for x in containers], marker, limit)
453 468
        return containers[start:start + limit]
454 469

  
470
    @debug_method
455 471
    def list_container_meta(self, user, account, container, domain,
456 472
                            until=None):
457 473
        """Return a list of the container's object meta keys for a domain."""
458 474

  
459
        logger.debug("list_container_meta: %s %s %s %s %s", user,
460
                     account, container, domain, until)
461 475
        allowed = []
462 476
        if user != account:
463 477
            if until:
......
472 486
        return self.node.latest_attribute_keys(node, domain, before,
473 487
                                               CLUSTER_DELETED, allowed)
474 488

  
489
    @debug_method
475 490
    def get_container_meta(self, user, account, container, domain, until=None,
476 491
                           include_user_defined=True):
477 492
        """Return a dictionary with the container metadata for the domain."""
478 493

  
479
        logger.debug("get_container_meta: %s %s %s %s %s", user,
480
                     account, container, domain, until)
481 494
        if user != account:
482 495
            if until or container not in self._allowed_containers(user,
483 496
                                                                  account):
......
507 520
        meta.update({'modified': modified})
508 521
        return meta
509 522

  
523
    @debug_method
510 524
    def update_container_meta(self, user, account, container, domain, meta,
511 525
                              replace=False):
512 526
        """Update the metadata associated with the container for the domain."""
513 527

  
514
        logger.debug("update_container_meta: %s %s %s %s %s %s",
515
                     user, account, container, domain, meta, replace)
516 528
        if user != account:
517 529
            raise NotAllowedError
518 530
        path, node = self._lookup_container(account, container)
......
526 538
                self.node.version_remove(src_version_id,
527 539
                                         update_statistics_ancestors_depth=0)
528 540

  
541
    @debug_method
529 542
    def get_container_policy(self, user, account, container):
530 543
        """Return a dictionary with the container policy."""
531 544

  
532
        logger.debug(
533
            "get_container_policy: %s %s %s", user, account, container)
534 545
        if user != account:
535 546
            if container not in self._allowed_containers(user, account):
536 547
                raise NotAllowedError
......
538 549
        path, node = self._lookup_container(account, container)
539 550
        return self._get_policy(node, is_account_policy=False)
540 551

  
552
    @debug_method
541 553
    def update_container_policy(self, user, account, container, policy,
542 554
                                replace=False):
543 555
        """Update the policy associated with the container."""
544 556

  
545
        logger.debug("update_container_policy: %s %s %s %s %s",
546
                     user, account, container, policy, replace)
547 557
        if user != account:
548 558
            raise NotAllowedError
549 559
        path, node = self._lookup_container(account, container)
550 560
        self._check_policy(policy, is_account_policy=False)
551 561
        self._put_policy(node, policy, replace, is_account_policy=False)
552 562

  
563
    @debug_method
553 564
    def put_container(self, user, account, container, policy=None):
554 565
        """Create a new container with the given name."""
555 566

  
556
        logger.debug(
557
            "put_container: %s %s %s %s", user, account, container, policy)
558 567
        policy = policy or {}
559 568
        if user != account:
560 569
            raise NotAllowedError
......
572 581
            update_statistics_ancestors_depth=-1)
573 582
        self._put_policy(node, policy, True, is_account_policy=False)
574 583

  
584
    @debug_method
575 585
    def delete_container(self, user, account, container, until=None, prefix='',
576 586
                         delimiter=None):
577 587
        """Delete/purge the container with the given name."""
578 588

  
579
        logger.debug("delete_container: %s %s %s %s %s %s", user,
580
                     account, container, until, prefix, delimiter)
581 589
        if user != account:
582 590
            raise NotAllowedError
583 591
        path, node = self._lookup_container(account, container)
......
741 749
                return []
742 750
        return allowed
743 751

  
752
    @debug_method
744 753
    def list_objects(self, user, account, container, prefix='', delimiter=None,
745 754
                     marker=None, limit=10000, virtual=True, domain=None,
746 755
                     keys=None, shared=False, until=None, size_range=None,
747 756
                     public=False):
748 757
        """List (object name, object version_id) under a container."""
749 758

  
750
        logger.debug("list_objects: %s %s %s %s %s %s %s %s %s %s %s %s %s %s",
751
                     user, account, container, prefix, delimiter, marker,
752
                     limit, virtual, domain, keys, shared, until, size_range,
753
                     public)
754 759
        keys = keys or []
755 760
        return self._list_objects(
756 761
            user, account, container, prefix, delimiter, marker, limit,
757 762
            virtual, domain, keys, shared, until, size_range, False, public)
758 763

  
764
    @debug_method
759 765
    def list_object_meta(self, user, account, container, prefix='',
760 766
                         delimiter=None, marker=None, limit=10000,
761 767
                         virtual=True, domain=None, keys=None, shared=False,
762 768
                         until=None, size_range=None, public=False):
763 769
        """Return a list of metadata dicts of objects under a container."""
764 770

  
765
        logger.debug(
766
            "list_object_meta: %s %s %s %s %s %s %s %s %s %s %s %s %s %s",
767
            user, account, container, prefix, delimiter, marker, limit,
768
            virtual, domain, keys, shared, until, size_range, public)
769 771
        keys = keys or []
770 772
        props = self._list_objects(
771 773
            user, account, container, prefix, delimiter, marker, limit,
......
788 790
                    'checksum': p[self.CHECKSUM + 1]})
789 791
        return objects
790 792

  
793
    @debug_method
791 794
    def list_object_permissions(self, user, account, container, prefix=''):
792 795
        """Return a list of paths enforce permissions under a container."""
793 796

  
794
        logger.debug("list_object_permissions: %s %s %s %s", user,
795
                     account, container, prefix)
796 797
        return self._list_object_permissions(user, account, container, prefix,
797 798
                                             True, False)
798 799

  
800
    @debug_method
799 801
    def list_object_public(self, user, account, container, prefix=''):
800 802
        """Return a mapping of object paths to public ids under a container."""
801 803

  
802
        logger.debug("list_object_public: %s %s %s %s", user,
803
                     account, container, prefix)
804 804
        public = {}
805 805
        for path, p in self.permissions.public_list('/'.join((account,
806 806
                                                              container,
......
808 808
            public[path] = p
809 809
        return public
810 810

  
811
    @debug_method
811 812
    def get_object_meta(self, user, account, container, name, domain,
812 813
                        version=None, include_user_defined=True):
813 814
        """Return a dictionary with the object metadata for the domain."""
814 815

  
815
        logger.debug("get_object_meta: %s %s %s %s %s %s", user,
816
                     account, container, name, domain, version)
817 816
        self._can_read(user, account, container, name)
818 817
        path, node = self._lookup_object(account, container, name)
819 818
        props = self._get_version(node, version)
......
846 845
                     'checksum': props[self.CHECKSUM]})
847 846
        return meta
848 847

  
848
    @debug_method
849 849
    def update_object_meta(self, user, account, container, name, domain, meta,
850 850
                           replace=False):
851 851
        """Update object metadata for a domain and return the new version."""
852 852

  
853
        logger.debug("update_object_meta: %s %s %s %s %s %s %s",
854
                     user, account, container, name, domain, meta, replace)
855 853
        self._can_write(user, account, container, name)
856 854
        path, node = self._lookup_object(account, container, name)
857 855
        src_version_id, dest_version_id = self._put_metadata(
......
861 859
                               update_statistics_ancestors_depth=1)
862 860
        return dest_version_id
863 861

  
862
    @debug_method
864 863
    def get_object_permissions(self, user, account, container, name):
865 864
        """Return the action allowed on the object, the path
866 865
        from which the object gets its permissions from,
867 866
        along with a dictionary containing the permissions."""
868 867

  
869
        logger.debug("get_object_permissions: %s %s %s %s", user,
870
                     account, container, name)
871 868
        allowed = 'write'
872 869
        permissions_path = self._get_permissions_path(account, container, name)
873 870
        if user != account:
......
884 881
                permissions_path,
885 882
                self.permissions.access_get(permissions_path))
886 883

  
884
    @debug_method
887 885
    def update_object_permissions(self, user, account, container, name,
888 886
                                  permissions):
889 887
        """Update the permissions associated with the object."""
890 888

  
891
        logger.debug("update_object_permissions: %s %s %s %s %s",
892
                     user, account, container, name, permissions)
893 889
        if user != account:
894 890
            raise NotAllowedError
895 891
        path = self._lookup_object(account, container, name)[0]
......
898 894
        self._report_sharing_change(user, account, path, {'members':
899 895
                                    self.permissions.access_members(path)})
900 896

  
897
    @debug_method
901 898
    def get_object_public(self, user, account, container, name):
902 899
        """Return the public id of the object if applicable."""
903 900

  
904
        logger.debug(
905
            "get_object_public: %s %s %s %s", user, account, container, name)
906 901
        self._can_read(user, account, container, name)
907 902
        path = self._lookup_object(account, container, name)[0]
908 903
        p = self.permissions.public_get(path)
909 904
        return p
910 905

  
906
    @debug_method
911 907
    def update_object_public(self, user, account, container, name, public):
912 908
        """Update the public status of the object."""
913 909

  
914
        logger.debug("update_object_public: %s %s %s %s %s", user,
915
                     account, container, name, public)
916 910
        self._can_write(user, account, container, name)
917 911
        path = self._lookup_object(account, container, name)[0]
918 912
        if not public:
......
922 916
                path, self.public_url_security, self.public_url_alphabet
923 917
            )
924 918

  
919
    @debug_method
925 920
    def get_object_hashmap(self, user, account, container, name, version=None):
926 921
        """Return the object's size and a list with partial hashes."""
927 922

  
928
        logger.debug("get_object_hashmap: %s %s %s %s %s", user,
929
                     account, container, name, version)
930 923
        self._can_read(user, account, container, name)
931 924
        path, node = self._lookup_object(account, container, name)
932 925
        props = self._get_version(node, version)
......
1004 997
            details={'version': dest_version_id, 'action': 'object update'})
1005 998
        return dest_version_id
1006 999

  
1000
    @debug_method
1007 1001
    def update_object_hashmap(self, user, account, container, name, size, type,
1008 1002
                              hashmap, checksum, domain, meta=None,
1009 1003
                              replace_meta=False, permissions=None):
1010 1004
        """Create/update an object's hashmap and return the new version."""
1011 1005

  
1012
        logger.debug("update_object_hashmap: %s %s %s %s %s %s %s %s", user,
1013
                     account, container, name, size, type, hashmap, checksum)
1014 1006
        meta = meta or {}
1015 1007
        if size == 0:  # No such thing as an empty hashmap.
1016 1008
            hashmap = [self.put_block('')]
......
1030 1022
        self.store.map_put(hash, map)
1031 1023
        return dest_version_id, hexlified
1032 1024

  
1025
    @debug_method
1033 1026
    def update_object_checksum(self, user, account, container, name, version,
1034 1027
                               checksum):
1035 1028
        """Update an object's checksum."""
1036 1029

  
1037
        logger.debug("update_object_checksum: %s %s %s %s %s %s",
1038
                     user, account, container, name, version, checksum)
1039 1030
        # Update objects with greater version and same hashmap
1040 1031
        # and size (fix metadata updates).
1041 1032
        self._can_write(user, account, container, name)
......
1107 1098
        return (dest_version_ids[0] if len(dest_version_ids) == 1 else
1108 1099
                dest_version_ids)
1109 1100

  
1101
    @debug_method
1110 1102
    def copy_object(self, user, src_account, src_container, src_name,
1111 1103
                    dest_account, dest_container, dest_name, type, domain,
1112 1104
                    meta=None, replace_meta=False, permissions=None,
1113 1105
                    src_version=None, delimiter=None):
1114 1106
        """Copy an object's data and metadata."""
1115 1107

  
1116
        logger.debug("copy_object: %s %s %s %s %s %s %s %s %s %s %s %s %s %s",
1117
                     user, src_account, src_container, src_name, dest_account,
1118
                     dest_container, dest_name, type, domain, meta,
1119
                     replace_meta, permissions, src_version, delimiter)
1120 1108
        meta = meta or {}
1121 1109
        dest_version_id = self._copy_object(
1122 1110
            user, src_account, src_container, src_name, dest_account,
......
1124 1112
            permissions, src_version, False, delimiter)
1125 1113
        return dest_version_id
1126 1114

  
1115
    @debug_method
1127 1116
    def move_object(self, user, src_account, src_container, src_name,
1128 1117
                    dest_account, dest_container, dest_name, type, domain,
1129 1118
                    meta=None, replace_meta=False, permissions=None,
1130 1119
                    delimiter=None):
1131 1120
        """Move an object's data and metadata."""
1132 1121

  
1133
        logger.debug("move_object: %s %s %s %s %s %s %s %s %s %s %s %s %s",
1134
                     user, src_account, src_container, src_name, dest_account,
1135
                     dest_container, dest_name, type, domain, meta,
1136
                     replace_meta, permissions, delimiter)
1137 1122
        meta = meta or {}
1138 1123
        if user != src_account:
1139 1124
            raise NotAllowedError
......
1226 1211
                paths.append(path)
1227 1212
            self.permissions.access_clear_bulk(paths)
1228 1213

  
1214
    @debug_method
1229 1215
    def delete_object(self, user, account, container, name, until=None,
1230 1216
                      prefix='', delimiter=None):
1231 1217
        """Delete/purge an object."""
1232 1218

  
1233
        logger.debug("delete_object: %s %s %s %s %s %s %s", user,
1234
                     account, container, name, until, prefix, delimiter)
1235 1219
        self._delete_object(user, account, container, name, until, delimiter)
1236 1220

  
1221
    @debug_method
1237 1222
    def list_versions(self, user, account, container, name):
1238 1223
        """Return a list of all object (version, version_timestamp) tuples."""
1239 1224

  
1240
        logger.debug(
1241
            "list_versions: %s %s %s %s", user, account, container, name)
1242 1225
        self._can_read(user, account, container, name)
1243 1226
        path, node = self._lookup_object(account, container, name)
1244 1227
        versions = self.node.node_get_versions(node)
1245 1228
        return [[x[self.SERIAL], x[self.MTIME]] for x in versions if
1246 1229
                x[self.CLUSTER] != CLUSTER_DELETED]
1247 1230

  
1231
    @debug_method
1248 1232
    def get_uuid(self, user, uuid):
1249 1233
        """Return the (account, container, name) for the UUID given."""
1250 1234

  
1251
        logger.debug("get_uuid: %s %s", user, uuid)
1252 1235
        info = self.node.latest_uuid(uuid, CLUSTER_NORMAL)
1253 1236
        if info is None:
1254 1237
            raise NameError
......
1257 1240
        self._can_read(user, account, container, name)
1258 1241
        return (account, container, name)
1259 1242

  
1243
    @debug_method
1260 1244
    def get_public(self, user, public):
1261 1245
        """Return the (account, container, name) for the public id given."""
1262 1246

  
1263
        logger.debug("get_public: %s %s", user, public)
1264 1247
        path = self.permissions.public_path(public)
1265 1248
        if path is None:
1266 1249
            raise NameError
......
1268 1251
        self._can_read(user, account, container, name)
1269 1252
        return (account, container, name)
1270 1253

  
1254
    @debug_method
1271 1255
    def get_block(self, hash):
1272 1256
        """Return a block's data."""
1273 1257

  
1274
        logger.debug("get_block: %s", hash)
1275 1258
        block = self.store.block_get(binascii.unhexlify(hash))
1276 1259
        if not block:
1277 1260
            raise ItemNotExists('Block does not exist')
......
1494 1477

  
1495 1478
    # Reporting functions.
1496 1479

  
1480
    @debug_method
1497 1481
    def _report_size_change(self, user, account, size, details=None):
1498 1482
        details = details or {}
1499 1483

  
......
1503 1487
        account_node = self._lookup_account(account, True)[1]
1504 1488
        total = self._get_statistics(account_node, compute=True)[1]
1505 1489
        details.update({'user': user, 'total': total})
1506
        logger.debug(
1507
            "_report_size_change: %s %s %s %s", user, account, size, details)
1508 1490
        self.messages.append(
1509 1491
            (QUEUE_MESSAGE_KEY_PREFIX % ('resource.diskspace',),
1510 1492
             account, QUEUE_INSTANCE_ID, 'diskspace', float(size), details))
......
1525 1507
        else:
1526 1508
            self.serials.append(serial)
1527 1509

  
1510
    @debug_method
1528 1511
    def _report_object_change(self, user, account, path, details=None):
1529 1512
        details = details or {}
1530 1513
        details.update({'user': user})
1531
        logger.debug("_report_object_change: %s %s %s %s", user,
1532
                     account, path, details)
1533 1514
        self.messages.append((QUEUE_MESSAGE_KEY_PREFIX % ('object',),
1534 1515
                              account, QUEUE_INSTANCE_ID, 'object', path,
1535 1516
                              details))
1536 1517

  
1518
    @debug_method
1537 1519
    def _report_sharing_change(self, user, account, path, details=None):
1538
        logger.debug("_report_permissions_change: %s %s %s %s",
1539
                     user, account, path, details)
1540 1520
        details = details or {}
1541 1521
        details.update({'user': user})
1542 1522
        self.messages.append((QUEUE_MESSAGE_KEY_PREFIX % ('sharing',),
......
1681 1661

  
1682 1662
    # Domain functions
1683 1663

  
1664
    @debug_method
1684 1665
    def get_domain_objects(self, domain, user=None):
1685 1666
        allowed_paths = self.permissions.access_list_paths(
1686 1667
            user, include_owned=user is not None, include_containers=False)

Also available in: Unified diff