Revision 9f135224 snf-pithos-backend/pithos/backends/modular.py
b/snf-pithos-backend/pithos/backends/modular.py | ||
---|---|---|
118 | 118 |
logger = logging.getLogger(__name__) |
119 | 119 |
|
120 | 120 |
|
121 |
def backend_method(func=None, autocommit=1): |
|
122 |
if func is None: |
|
123 |
def fn(func): |
|
124 |
return backend_method(func, autocommit) |
|
125 |
return fn |
|
126 |
|
|
127 |
if not autocommit: |
|
128 |
return func |
|
129 |
|
|
130 |
def fn(self, *args, **kw): |
|
131 |
self.wrapper.execute() |
|
132 |
serials = [] |
|
133 |
self.serials = serials |
|
134 |
self.messages = [] |
|
135 |
|
|
136 |
try: |
|
137 |
ret = func(self, *args, **kw) |
|
138 |
for m in self.messages: |
|
139 |
self.queue.send(*m) |
|
140 |
if self.serials: |
|
141 |
self.commission_serials.insert_many(self.serials) |
|
142 |
|
|
143 |
# commit to ensure that the serials are registered |
|
144 |
# even if accept commission fails |
|
145 |
self.wrapper.commit() |
|
146 |
self.wrapper.execute() |
|
147 |
|
|
148 |
r = self.astakosclient.resolve_commissions( |
|
149 |
token=self.service_token, |
|
150 |
accept_serials=self.serials, |
|
151 |
reject_serials=[]) |
|
152 |
self.commission_serials.delete_many(r['accepted']) |
|
153 |
|
|
154 |
self.wrapper.commit() |
|
155 |
return ret |
|
156 |
except: |
|
157 |
if self.serials: |
|
158 |
self.astakosclient.resolve_commissions( |
|
159 |
token=self.service_token, |
|
160 |
accept_serials=[], |
|
161 |
reject_serials=self.serials) |
|
162 |
self.wrapper.rollback() |
|
163 |
raise |
|
164 |
return fn |
|
165 |
|
|
166 |
|
|
167 | 121 |
class ModularBackend(BaseBackend): |
168 | 122 |
"""A modular backend. |
169 | 123 |
|
... | ... | |
275 | 229 |
def using_external_quotaholder(self): |
276 | 230 |
return not isinstance(self.astakosclient, DisabledAstakosClient) |
277 | 231 |
|
278 |
@backend_method |
|
279 | 232 |
def list_accounts(self, user, marker=None, limit=10000): |
280 | 233 |
"""Return a list of accounts the user can access.""" |
281 | 234 |
|
... | ... | |
284 | 237 |
start, limit = self._list_limits(allowed, marker, limit) |
285 | 238 |
return allowed[start:start + limit] |
286 | 239 |
|
287 |
@backend_method |
|
288 | 240 |
def get_account_meta( |
289 | 241 |
self, user, account, domain, until=None, include_user_defined=True, |
290 | 242 |
external_quota=None): |
... | ... | |
327 | 279 |
meta.update({'modified': modified}) |
328 | 280 |
return meta |
329 | 281 |
|
330 |
@backend_method |
|
331 | 282 |
def update_account_meta(self, user, account, domain, meta, replace=False): |
332 | 283 |
"""Update the metadata associated with the account for the domain.""" |
333 | 284 |
|
... | ... | |
339 | 290 |
self._put_metadata(user, node, domain, meta, replace, |
340 | 291 |
update_statistics_ancestors_depth=-1) |
341 | 292 |
|
342 |
@backend_method |
|
343 | 293 |
def get_account_groups(self, user, account): |
344 | 294 |
"""Return a dictionary with the user groups defined for this account.""" |
345 | 295 |
|
... | ... | |
351 | 301 |
self._lookup_account(account, True) |
352 | 302 |
return self.permissions.group_dict(account) |
353 | 303 |
|
354 |
@backend_method |
|
355 | 304 |
def update_account_groups(self, user, account, groups, replace=False): |
356 | 305 |
"""Update the groups associated with the account.""" |
357 | 306 |
|
... | ... | |
369 | 318 |
if v: |
370 | 319 |
self.permissions.group_addmany(account, k, v) |
371 | 320 |
|
372 |
@backend_method |
|
373 | 321 |
def get_account_policy(self, user, account, external_quota=None): |
374 | 322 |
"""Return a dictionary with the account policy.""" |
375 | 323 |
|
... | ... | |
385 | 333 |
policy['quota'] = external_quota.get('limit', 0) |
386 | 334 |
return policy |
387 | 335 |
|
388 |
@backend_method |
|
389 | 336 |
def update_account_policy(self, user, account, policy, replace=False): |
390 | 337 |
"""Update the policy associated with the account.""" |
391 | 338 |
|
... | ... | |
397 | 344 |
self._check_policy(policy, is_account_policy=True) |
398 | 345 |
self._put_policy(node, policy, replace, is_account_policy=True) |
399 | 346 |
|
400 |
@backend_method |
|
401 | 347 |
def put_account(self, user, account, policy=None): |
402 | 348 |
"""Create a new account with the given name.""" |
403 | 349 |
|
... | ... | |
414 | 360 |
update_statistics_ancestors_depth=-1) |
415 | 361 |
self._put_policy(node, policy, True, is_account_policy=True) |
416 | 362 |
|
417 |
@backend_method |
|
418 | 363 |
def delete_account(self, user, account): |
419 | 364 |
"""Delete the account with the given name.""" |
420 | 365 |
|
... | ... | |
429 | 374 |
raise AccountNotEmpty('Account is not empty') |
430 | 375 |
self.permissions.group_destroy(account) |
431 | 376 |
|
432 |
@backend_method |
|
433 | 377 |
def list_containers(self, user, account, marker=None, limit=10000, shared=False, until=None, public=False): |
434 | 378 |
"""Return a list of containers existing under an account.""" |
435 | 379 |
|
... | ... | |
457 | 401 |
[x[0] for x in containers], marker, limit) |
458 | 402 |
return containers[start:start + limit] |
459 | 403 |
|
460 |
@backend_method |
|
461 | 404 |
def list_container_meta(self, user, account, container, domain, until=None): |
462 | 405 |
"""Return a list with all the container's object meta keys for the domain.""" |
463 | 406 |
|
... | ... | |
476 | 419 |
allowed = self._get_formatted_paths(allowed) |
477 | 420 |
return self.node.latest_attribute_keys(node, domain, before, CLUSTER_DELETED, allowed) |
478 | 421 |
|
479 |
@backend_method |
|
480 | 422 |
def get_container_meta(self, user, account, container, domain, until=None, include_user_defined=True): |
481 | 423 |
"""Return a dictionary with the container metadata for the domain.""" |
482 | 424 |
|
... | ... | |
510 | 452 |
meta.update({'modified': modified}) |
511 | 453 |
return meta |
512 | 454 |
|
513 |
@backend_method |
|
514 | 455 |
def update_container_meta(self, user, account, container, domain, meta, replace=False): |
515 | 456 |
"""Update the metadata associated with the container for the domain.""" |
516 | 457 |
|
... | ... | |
529 | 470 |
self.node.version_remove(src_version_id, |
530 | 471 |
update_statistics_ancestors_depth=0) |
531 | 472 |
|
532 |
@backend_method |
|
533 | 473 |
def get_container_policy(self, user, account, container): |
534 | 474 |
"""Return a dictionary with the container policy.""" |
535 | 475 |
|
... | ... | |
542 | 482 |
path, node = self._lookup_container(account, container) |
543 | 483 |
return self._get_policy(node, is_account_policy=False) |
544 | 484 |
|
545 |
@backend_method |
|
546 | 485 |
def update_container_policy(self, user, account, container, policy, replace=False): |
547 | 486 |
"""Update the policy associated with the container.""" |
548 | 487 |
|
... | ... | |
554 | 493 |
self._check_policy(policy, is_account_policy=False) |
555 | 494 |
self._put_policy(node, policy, replace, is_account_policy=False) |
556 | 495 |
|
557 |
@backend_method |
|
558 | 496 |
def put_container(self, user, account, container, policy=None): |
559 | 497 |
"""Create a new container with the given name.""" |
560 | 498 |
|
... | ... | |
577 | 515 |
update_statistics_ancestors_depth=-1) |
578 | 516 |
self._put_policy(node, policy, True, is_account_policy=False) |
579 | 517 |
|
580 |
@backend_method |
|
581 | 518 |
def delete_container(self, user, account, container, until=None, prefix='', delimiter=None): |
582 | 519 |
"""Delete/purge the container with the given name.""" |
583 | 520 |
|
... | ... | |
731 | 668 |
return [] |
732 | 669 |
return allowed |
733 | 670 |
|
734 |
@backend_method |
|
735 | 671 |
def list_objects(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=None, shared=False, until=None, size_range=None, public=False): |
736 | 672 |
"""Return a list of object (name, version_id) tuples existing under a container.""" |
737 | 673 |
|
... | ... | |
739 | 675 |
keys = keys or [] |
740 | 676 |
return self._list_objects(user, account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range, False, public) |
741 | 677 |
|
742 |
@backend_method |
|
743 | 678 |
def list_object_meta(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=None, shared=False, until=None, size_range=None, public=False): |
744 | 679 |
"""Return a list of object metadata dicts existing under a container.""" |
745 | 680 |
|
... | ... | |
763 | 698 |
'checksum': p[self.CHECKSUM + 1]}) |
764 | 699 |
return objects |
765 | 700 |
|
766 |
@backend_method |
|
767 | 701 |
def list_object_permissions(self, user, account, container, prefix=''): |
768 | 702 |
"""Return a list of paths that enforce permissions under a container.""" |
769 | 703 |
|
... | ... | |
771 | 705 |
account, container, prefix) |
772 | 706 |
return self._list_object_permissions(user, account, container, prefix, True, False) |
773 | 707 |
|
774 |
@backend_method |
|
775 | 708 |
def list_object_public(self, user, account, container, prefix=''): |
776 | 709 |
"""Return a dict mapping paths to public ids for objects that are public under a container.""" |
777 | 710 |
|
... | ... | |
782 | 715 |
public[path] = p |
783 | 716 |
return public |
784 | 717 |
|
785 |
@backend_method |
|
786 | 718 |
def get_object_meta(self, user, account, container, name, domain, version=None, include_user_defined=True): |
787 | 719 |
"""Return a dictionary with the object metadata for the domain.""" |
788 | 720 |
|
... | ... | |
820 | 752 |
'checksum': props[self.CHECKSUM]}) |
821 | 753 |
return meta |
822 | 754 |
|
823 |
@backend_method |
|
824 | 755 |
def update_object_meta(self, user, account, container, name, domain, meta, replace=False): |
825 | 756 |
"""Update the metadata associated with the object for the domain and return the new version.""" |
826 | 757 |
|
... | ... | |
835 | 766 |
update_statistics_ancestors_depth=1) |
836 | 767 |
return dest_version_id |
837 | 768 |
|
838 |
@backend_method |
|
839 | 769 |
def get_object_permissions(self, user, account, container, name): |
840 | 770 |
"""Return the action allowed on the object, the path |
841 | 771 |
from which the object gets its permissions from, |
... | ... | |
855 | 785 |
self._lookup_object(account, container, name) |
856 | 786 |
return (allowed, permissions_path, self.permissions.access_get(permissions_path)) |
857 | 787 |
|
858 |
@backend_method |
|
859 | 788 |
def update_object_permissions(self, user, account, container, name, permissions): |
860 | 789 |
"""Update the permissions associated with the object.""" |
861 | 790 |
|
... | ... | |
869 | 798 |
self._report_sharing_change(user, account, path, {'members': |
870 | 799 |
self.permissions.access_members(path)}) |
871 | 800 |
|
872 |
@backend_method |
|
873 | 801 |
def get_object_public(self, user, account, container, name): |
874 | 802 |
"""Return the public id of the object if applicable.""" |
875 | 803 |
|
... | ... | |
880 | 808 |
p = self.permissions.public_get(path) |
881 | 809 |
return p |
882 | 810 |
|
883 |
@backend_method |
|
884 | 811 |
def update_object_public(self, user, account, container, name, public): |
885 | 812 |
"""Update the public status of the object.""" |
886 | 813 |
|
... | ... | |
895 | 822 |
path, self.public_url_security, self.public_url_alphabet |
896 | 823 |
) |
897 | 824 |
|
898 |
@backend_method |
|
899 | 825 |
def get_object_hashmap(self, user, account, container, name, version=None): |
900 | 826 |
"""Return the object's size and a list with partial hashes.""" |
901 | 827 |
|
... | ... | |
975 | 901 |
self._report_object_change(user, account, path, details={'version': dest_version_id, 'action': 'object update'}) |
976 | 902 |
return dest_version_id |
977 | 903 |
|
978 |
@backend_method |
|
979 | 904 |
def update_object_hashmap(self, user, account, container, name, size, type, hashmap, checksum, domain, meta=None, replace_meta=False, permissions=None): |
980 | 905 |
"""Create/update an object with the specified size and partial hashes.""" |
981 | 906 |
|
... | ... | |
997 | 922 |
self.store.map_put(hash, map) |
998 | 923 |
return dest_version_id |
999 | 924 |
|
1000 |
@backend_method |
|
1001 | 925 |
def update_object_checksum(self, user, account, container, name, version, checksum): |
1002 | 926 |
"""Update an object's checksum.""" |
1003 | 927 |
|
... | ... | |
1053 | 977 |
self._delete_object(user, src_account, src_container, path) |
1054 | 978 |
return dest_version_ids[0] if len(dest_version_ids) == 1 else dest_version_ids |
1055 | 979 |
|
1056 |
@backend_method |
|
1057 | 980 |
def copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta=None, replace_meta=False, permissions=None, src_version=None, delimiter=None): |
1058 | 981 |
"""Copy an object's data and metadata.""" |
1059 | 982 |
|
... | ... | |
1062 | 985 |
dest_version_id = self._copy_object(user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta, replace_meta, permissions, src_version, False, delimiter) |
1063 | 986 |
return dest_version_id |
1064 | 987 |
|
1065 |
@backend_method |
|
1066 | 988 |
def move_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, type, domain, meta=None, replace_meta=False, permissions=None, delimiter=None): |
1067 | 989 |
"""Move an object's data and metadata.""" |
1068 | 990 |
|
... | ... | |
1149 | 1071 |
paths.append(path) |
1150 | 1072 |
self.permissions.access_clear_bulk(paths) |
1151 | 1073 |
|
1152 |
@backend_method |
|
1153 | 1074 |
def delete_object(self, user, account, container, name, until=None, prefix='', delimiter=None): |
1154 | 1075 |
"""Delete/purge an object.""" |
1155 | 1076 |
|
... | ... | |
1157 | 1078 |
account, container, name, until, prefix, delimiter) |
1158 | 1079 |
self._delete_object(user, account, container, name, until, delimiter) |
1159 | 1080 |
|
1160 |
@backend_method |
|
1161 | 1081 |
def list_versions(self, user, account, container, name): |
1162 | 1082 |
"""Return a list of all (version, version_timestamp) tuples for an object.""" |
1163 | 1083 |
|
... | ... | |
1168 | 1088 |
versions = self.node.node_get_versions(node) |
1169 | 1089 |
return [[x[self.SERIAL], x[self.MTIME]] for x in versions if x[self.CLUSTER] != CLUSTER_DELETED] |
1170 | 1090 |
|
1171 |
@backend_method |
|
1172 | 1091 |
def get_uuid(self, user, uuid): |
1173 | 1092 |
"""Return the (account, container, name) for the UUID given.""" |
1174 | 1093 |
|
... | ... | |
1181 | 1100 |
self._can_read(user, account, container, name) |
1182 | 1101 |
return (account, container, name) |
1183 | 1102 |
|
1184 |
@backend_method |
|
1185 | 1103 |
def get_public(self, user, public): |
1186 | 1104 |
"""Return the (account, container, name) for the public id given.""" |
1187 | 1105 |
|
... | ... | |
1193 | 1111 |
self._can_read(user, account, container, name) |
1194 | 1112 |
return (account, container, name) |
1195 | 1113 |
|
1196 |
@backend_method(autocommit=0) |
|
1197 | 1114 |
def get_block(self, hash): |
1198 | 1115 |
"""Return a block's data.""" |
1199 | 1116 |
|
... | ... | |
1203 | 1120 |
raise ItemNotExists('Block does not exist') |
1204 | 1121 |
return block |
1205 | 1122 |
|
1206 |
@backend_method(autocommit=0) |
|
1207 | 1123 |
def put_block(self, data): |
1208 | 1124 |
"""Store a block and return the hash.""" |
1209 | 1125 |
|
1210 | 1126 |
logger.debug("put_block: %s", len(data)) |
1211 | 1127 |
return binascii.hexlify(self.store.block_put(data)) |
1212 | 1128 |
|
1213 |
@backend_method(autocommit=0) |
|
1214 | 1129 |
def update_block(self, hash, data, offset=0): |
1215 | 1130 |
"""Update a known block and return the hash.""" |
1216 | 1131 |
|
... | ... | |
1592 | 1507 |
|
1593 | 1508 |
# Domain functions |
1594 | 1509 |
|
1595 |
@backend_method |
|
1596 | 1510 |
def get_domain_objects(self, domain, user=None): |
1597 | 1511 |
obj_list = self.node.domain_object_list(domain, CLUSTER_NORMAL) |
1598 | 1512 |
if user != None: |
Also available in: Unified diff