Revision 9d4502a8
| b/snf-pithos-backend/pithos/backends/lib/sqlalchemy/node.py | ||
|---|---|---|
| 330 | 330 |
self.statistics_update(parent, -nr, size, mtime, cluster) |
| 331 | 331 |
self.statistics_update_ancestors(parent, -nr, size, mtime, cluster) |
| 332 | 332 |
|
| 333 |
s = select([self.versions.c.hash]) |
|
| 333 |
s = select([self.versions.c.hash, self.versions.c.serial])
|
|
| 334 | 334 |
s = s.where(where_clause) |
| 335 | 335 |
r = self.conn.execute(s) |
| 336 |
hashes = [row[0] for row in r.fetchall()] |
|
| 336 |
hashes = [] |
|
| 337 |
serials = [] |
|
| 338 |
for row in r.fetchall(): |
|
| 339 |
hashes += [row[0]] |
|
| 340 |
serials += [row[1]] |
|
| 337 | 341 |
r.close() |
| 338 | 342 |
|
| 339 | 343 |
#delete versions |
| ... | ... | |
| 352 | 356 |
s = self.nodes.delete().where(self.nodes.c.node.in_(nodes)) |
| 353 | 357 |
self.conn.execute(s).close() |
| 354 | 358 |
|
| 355 |
return hashes, size |
|
| 359 |
return hashes, size, serials
|
|
| 356 | 360 |
|
| 357 | 361 |
def node_purge(self, node, before=inf, cluster=0): |
| 358 | 362 |
"""Delete all versions with the specified |
| ... | ... | |
| 378 | 382 |
mtime = time() |
| 379 | 383 |
self.statistics_update_ancestors(node, -nr, -size, mtime, cluster) |
| 380 | 384 |
|
| 381 |
s = select([self.versions.c.hash]) |
|
| 385 |
s = select([self.versions.c.hash, self.versions.c.serial])
|
|
| 382 | 386 |
s = s.where(where_clause) |
| 383 | 387 |
r = self.conn.execute(s) |
| 384 |
hashes = [r[0] for r in r.fetchall()] |
|
| 388 |
hashes = [] |
|
| 389 |
serials = [] |
|
| 390 |
for row in r.fetchall(): |
|
| 391 |
hashes += [row[0]] |
|
| 392 |
serials += [row[1]] |
|
| 385 | 393 |
r.close() |
| 386 | 394 |
|
| 387 | 395 |
#delete versions |
| ... | ... | |
| 400 | 408 |
s = self.nodes.delete().where(self.nodes.c.node.in_(nodes)) |
| 401 | 409 |
self.conn.execute(s).close() |
| 402 | 410 |
|
| 403 |
return hashes, size |
|
| 411 |
return hashes, size, serials
|
|
| 404 | 412 |
|
| 405 | 413 |
def node_remove(self, node): |
| 406 | 414 |
"""Remove the node specified. |
| b/snf-pithos-backend/pithos/backends/lib/sqlite/node.py | ||
|---|---|---|
| 286 | 286 |
"and cluster = ? " |
| 287 | 287 |
"and mtime <= ?") |
| 288 | 288 |
execute(q, args) |
| 289 |
hashes = [r[0] for r in self.fetchall()] |
|
| 289 |
hashes = [] |
|
| 290 |
serials = [] |
|
| 291 |
for r in self.fetchall(): |
|
| 292 |
hashes += [r[0]] |
|
| 293 |
serials += [r[1]] |
|
| 294 |
|
|
| 290 | 295 |
q = ("delete from versions "
|
| 291 | 296 |
"where node in (select node " |
| 292 | 297 |
"from nodes " |
| ... | ... | |
| 301 | 306 |
"where node = n.node) = 0 " |
| 302 | 307 |
"and parent = ?)") |
| 303 | 308 |
execute(q, (parent,)) |
| 304 |
return hashes, size |
|
| 309 |
return hashes, size, serials
|
|
| 305 | 310 |
|
| 306 | 311 |
def node_purge(self, node, before=inf, cluster=0): |
| 307 | 312 |
"""Delete all versions with the specified |
| ... | ... | |
| 328 | 333 |
"and cluster = ? " |
| 329 | 334 |
"and mtime <= ?") |
| 330 | 335 |
execute(q, args) |
| 331 |
hashes = [r[0] for r in self.fetchall()] |
|
| 336 |
hashes = [] |
|
| 337 |
serials = [] |
|
| 338 |
for r in self.fetchall(): |
|
| 339 |
hashes += [r[0]] |
|
| 340 |
serials += [r[1]] |
|
| 341 |
|
|
| 332 | 342 |
q = ("delete from versions "
|
| 333 | 343 |
"where node = ? " |
| 334 | 344 |
"and cluster = ? " |
| ... | ... | |
| 341 | 351 |
"where node = n.node) = 0 " |
| 342 | 352 |
"and node = ?)") |
| 343 | 353 |
execute(q, (node,)) |
| 344 |
return hashes, size |
|
| 354 |
return hashes, size, serials
|
|
| 345 | 355 |
|
| 346 | 356 |
def node_remove(self, node): |
| 347 | 357 |
"""Remove the node specified. |
| b/snf-pithos-backend/pithos/backends/modular.py | ||
|---|---|---|
| 483 | 483 |
path, node = self._lookup_container(account, container) |
| 484 | 484 |
|
| 485 | 485 |
if until is not None: |
| 486 |
hashes, size = self.node.node_purge_children( |
|
| 486 |
hashes, size, serials = self.node.node_purge_children(
|
|
| 487 | 487 |
node, until, CLUSTER_HISTORY) |
| 488 | 488 |
for h in hashes: |
| 489 | 489 |
self.store.map_delete(h) |
| 490 | 490 |
self.node.node_purge_children(node, until, CLUSTER_DELETED) |
| 491 |
self._report_size_change(user, account, -size, {'action':
|
|
| 492 |
'container purge', 'path': path}) |
|
| 491 |
self._report_size_change(user, account, -size, |
|
| 492 |
{'action':'container purge', 'path': path,
|
|
| 493 |
'versions': serials}) |
|
| 493 | 494 |
return |
| 494 | 495 |
|
| 495 | 496 |
if not delimiter: |
| 496 | 497 |
if self._get_statistics(node)[0] > 0: |
| 497 | 498 |
raise ContainerNotEmpty('Container is not empty')
|
| 498 |
hashes, size = self.node.node_purge_children( |
|
| 499 |
hashes, size, serials = self.node.node_purge_children(
|
|
| 499 | 500 |
node, inf, CLUSTER_HISTORY) |
| 500 | 501 |
for h in hashes: |
| 501 | 502 |
self.store.map_delete(h) |
| 502 | 503 |
self.node.node_purge_children(node, inf, CLUSTER_DELETED) |
| 503 | 504 |
self.node.node_remove(node) |
| 504 |
self._report_size_change(user, account, -size, {'action':
|
|
| 505 |
'container delete', 'path': path}) |
|
| 505 |
self._report_size_change(user, account, -size, |
|
| 506 |
{'action': 'container delete',
|
|
| 507 |
'path': path, |
|
| 508 |
'versions': serials}) |
|
| 506 | 509 |
else: |
| 507 | 510 |
# remove only contents |
| 508 | 511 |
src_names = self._list_objects_no_limit(user, account, container, prefix='', delimiter=None, virtual=False, domain=None, keys=[], shared=False, until=None, size_range=None, all_props=True, public=False) |
| ... | ... | |
| 514 | 517 |
del_size = self._apply_versioning( |
| 515 | 518 |
account, container, src_version_id) |
| 516 | 519 |
if del_size: |
| 517 |
self._report_size_change(user, account, -del_size, {'action': 'object delete', 'path': path})
|
|
| 520 |
self._report_size_change(user, account, -del_size, |
|
| 521 |
{'action': 'object delete',
|
|
| 522 |
'path': path, 'versions': [dest_version_id]}) |
|
| 518 | 523 |
self._report_object_change( |
| 519 | 524 |
user, account, path, details={'action': 'object delete'})
|
| 520 | 525 |
paths.append(path) |
| ... | ... | |
| 804 | 809 |
(container_quota > 0 and self._get_statistics(container_node)[1] + size_delta > container_quota): |
| 805 | 810 |
# This must be executed in a transaction, so the version is never created if it fails. |
| 806 | 811 |
raise QuotaError |
| 807 |
self._report_size_change(user, account, size_delta, {
|
|
| 808 |
'action': 'object update', 'path': path}) |
|
| 812 |
self._report_size_change(user, account, size_delta, |
|
| 813 |
{'action': 'object update', 'path': path,
|
|
| 814 |
'versions': [dest_version_id]}) |
|
| 809 | 815 |
|
| 810 | 816 |
if permissions is not None: |
| 811 | 817 |
self.permissions.access_set(path, permissions) |
| ... | ... | |
| 919 | 925 |
return |
| 920 | 926 |
hashes = [] |
| 921 | 927 |
size = 0 |
| 922 |
h, s = self.node.node_purge(node, until, CLUSTER_NORMAL) |
|
| 928 |
serials = [] |
|
| 929 |
h, s, v = self.node.node_purge(node, until, CLUSTER_NORMAL) |
|
| 923 | 930 |
hashes += h |
| 924 | 931 |
size += s |
| 925 |
h, s = self.node.node_purge(node, until, CLUSTER_HISTORY) |
|
| 932 |
serials += v |
|
| 933 |
h, s, v = self.node.node_purge(node, until, CLUSTER_HISTORY) |
|
| 926 | 934 |
hashes += h |
| 927 | 935 |
size += s |
| 936 |
serials += v |
|
| 928 | 937 |
for h in hashes: |
| 929 | 938 |
self.store.map_delete(h) |
| 930 | 939 |
self.node.node_purge(node, until, CLUSTER_DELETED) |
| ... | ... | |
| 932 | 941 |
props = self._get_version(node) |
| 933 | 942 |
except NameError: |
| 934 | 943 |
self.permissions.access_clear(path) |
| 935 |
self._report_size_change(user, account, -size, {
|
|
| 936 |
'action': 'object purge', 'path': path}) |
|
| 944 |
self._report_size_change(user, account, -size, |
|
| 945 |
{'action': 'object purge', 'path': path,
|
|
| 946 |
'versions': serials}) |
|
| 937 | 947 |
return |
| 938 | 948 |
|
| 939 | 949 |
path, node = self._lookup_object(account, container, name) |
| 940 | 950 |
src_version_id, dest_version_id = self._put_version_duplicate(user, node, size=0, type='', hash=None, checksum='', cluster=CLUSTER_DELETED) |
| 941 | 951 |
del_size = self._apply_versioning(account, container, src_version_id) |
| 942 | 952 |
if del_size: |
| 943 |
self._report_size_change(user, account, -del_size, {
|
|
| 944 |
'action': 'object delete', 'path': path}) |
|
| 953 |
self._report_size_change(user, account, -del_size, |
|
| 954 |
{'action': 'object delete', 'path': path,
|
|
| 955 |
'versions': [dest_version_id]}) |
|
| 945 | 956 |
self._report_object_change( |
| 946 | 957 |
user, account, path, details={'action': 'object delete'})
|
| 947 | 958 |
self.permissions.access_clear(path) |
| ... | ... | |
| 957 | 968 |
del_size = self._apply_versioning( |
| 958 | 969 |
account, container, src_version_id) |
| 959 | 970 |
if del_size: |
| 960 |
self._report_size_change(user, account, -del_size, {'action': 'object delete', 'path': path})
|
|
| 971 |
self._report_size_change(user, account, -del_size, |
|
| 972 |
{'action': 'object delete',
|
|
| 973 |
'path': path, |
|
| 974 |
'versions': [dest_version_id]}) |
|
| 961 | 975 |
self._report_object_change( |
| 962 | 976 |
user, account, path, details={'action': 'object delete'})
|
| 963 | 977 |
paths.append(path) |
| ... | ... | |
| 1215 | 1229 |
logger.debug( |
| 1216 | 1230 |
"_report_size_change: %s %s %s %s", user, account, size, details) |
| 1217 | 1231 |
self.messages.append((QUEUE_MESSAGE_KEY_PREFIX % ('resource.diskspace',),
|
| 1218 |
account, |
|
| 1219 |
QUEUE_INSTANCE_ID, |
|
| 1220 |
'diskspace', |
|
| 1221 |
float(size), |
|
| 1222 |
details)) |
|
| 1232 |
account, QUEUE_INSTANCE_ID, 'diskspace', |
|
| 1233 |
float(size), details)) |
|
| 1223 | 1234 |
|
| 1224 | 1235 |
def _report_object_change(self, user, account, path, details={}):
|
| 1225 | 1236 |
details.update({'user': user})
|
| b/snf-pithos-tools/pithos/tools/test.py | ||
|---|---|---|
| 59 | 59 |
"%A, %d-%b-%y %H:%M:%S GMT", |
| 60 | 60 |
"%a, %d %b %Y %H:%M:%S GMT"] |
| 61 | 61 |
|
| 62 |
OTHER_ACCOUNTS = {
|
|
| 63 |
'0001': 'verigak', |
|
| 64 |
'0002': 'chazapis', |
|
| 65 |
'0003': 'gtsouk', |
|
| 66 |
'0004': 'papagian', |
|
| 67 |
'0005': 'louridas', |
|
| 68 |
'0006': 'chstath', |
|
| 69 |
'0007': 'pkanavos', |
|
| 70 |
'0008': 'mvasilak', |
|
| 71 |
'0009': 'διογένης'} |
|
| 62 |
from pithos.api.settings import AUTHENTICATION_USERS |
|
| 63 |
AUTHENTICATION_USERS = AUTHENTICATION_USERS or {}
|
|
| 64 |
OTHER_ACCOUNTS = AUTHENTICATION_USERS.copy() |
|
| 65 |
OTHER_ACCOUNTS.pop(get_auth()) |
|
| 72 | 66 |
|
| 73 | 67 |
class BaseTestCase(unittest.TestCase): |
| 74 | 68 |
#TODO unauthorized request |
| ... | ... | |
| 123 | 117 |
|
| 124 | 118 |
def _clean_account(self): |
| 125 | 119 |
for c in self.client.list_containers(): |
| 126 |
if c not in self.initial_containers: |
|
| 120 |
# if c not in self.initial_containers:
|
|
| 127 | 121 |
self.client.delete_container(c, delimiter='/') |
| 128 | 122 |
self.client.delete_container(c) |
| 129 | 123 |
|
| ... | ... | |
| 298 | 292 |
|
| 299 | 293 |
def test_get_account_meta_until(self): |
| 300 | 294 |
t = datetime.datetime.utcnow() |
| 301 |
past = t - datetime.timedelta(minutes=-15)
|
|
| 295 |
past = t - datetime.timedelta(minutes=15) |
|
| 302 | 296 |
past = int(_time.mktime(past.timetuple())) |
| 303 | 297 |
|
| 304 | 298 |
meta = {'premium':True}
|
| ... | ... | |
| 913 | 907 |
v_meta = self.client.retrieve_object_metadata(c, o['name'], |
| 914 | 908 |
restricted=True, |
| 915 | 909 |
version=v) |
| 916 |
for k in meta.keys(): |
|
| 917 |
self.assertTrue(k not in v_meta) |
|
| 918 |
|
|
| 910 |
(self.assertTrue(k not in v_meta) for k in meta.keys()) |
|
| 911 |
|
|
| 919 | 912 |
#update obejct |
| 920 | 913 |
data = get_random_data() |
| 921 | 914 |
self.client.update_object(c, o['name'], StringIO(data)) |
| 922 |
|
|
| 915 |
|
|
| 923 | 916 |
aa = self.client.retrieve_object_versionlist(c, o['name'])['versions'] |
| 924 | 917 |
self.assert_versionlist_structure(aa) |
| 925 | 918 |
self.assertEqual(len(a)+1, len(aa)) |
| ... | ... | |
| 1862 | 1855 |
for i in range(2): |
| 1863 | 1856 |
self.upload_random_data('c1', 'o%s' %i)
|
| 1864 | 1857 |
accounts = OTHER_ACCOUNTS.copy() |
| 1865 |
self.o1_sharing_with = accounts.popitem() |
|
| 1866 |
self.o1_sharing = [self.o1_sharing_with[1]] |
|
| 1867 |
self.client.share_object('c1', 'o1', self.o1_sharing, read=True)
|
|
| 1868 |
|
|
| 1858 |
self.o1_sharing = accounts.popitem() |
|
| 1859 |
self.client.share_object('c1', 'o1', (self.o1_sharing[1],), read=True)
|
|
| 1860 |
|
|
| 1869 | 1861 |
l = [] |
| 1870 |
for i in range(2):
|
|
| 1862 |
for i in range(len(OTHER_ACCOUNTS) - 1):
|
|
| 1871 | 1863 |
l.append(accounts.popitem()) |
| 1872 |
|
|
| 1864 |
|
|
| 1865 |
def tearDown(self): |
|
| 1866 |
pass |
|
| 1867 |
|
|
| 1873 | 1868 |
def test_list_other_shared(self): |
| 1874 | 1869 |
self.other = Pithos_Client(get_url(), |
| 1875 |
self.o1_sharing_with[0],
|
|
| 1876 |
self.o1_sharing_with[1])
|
|
| 1870 |
self.o1_sharing[0], |
|
| 1871 |
self.o1_sharing[1]) |
|
| 1877 | 1872 |
self.assertTrue(get_user() in self.other.list_shared_by_others()) |
| 1878 | 1873 |
|
| 1879 | 1874 |
def test_list_my_shared(self): |
Also available in: Unified diff