Revision 94bff756
b/snf-pithos-app/pithos/api/management/commands/pithos-manage-accounts.py | ||
---|---|---|
1 |
# Copyright 2012 GRNET S.A. All rights reserved. |
|
2 |
# |
|
3 |
# Redistribution and use in source and binary forms, with or |
|
4 |
# without modification, are permitted provided that the following |
|
5 |
# conditions are met: |
|
6 |
# |
|
7 |
# 1. Redistributions of source code must retain the above |
|
8 |
# copyright notice, this list of conditions and the following |
|
9 |
# disclaimer. |
|
10 |
# |
|
11 |
# 2. Redistributions in binary form must reproduce the above |
|
12 |
# copyright notice, this list of conditions and the following |
|
13 |
# disclaimer in the documentation and/or other materials |
|
14 |
# provided with the distribution. |
|
15 |
# |
|
16 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
|
17 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
|
18 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
|
19 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
|
20 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
21 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
22 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
|
23 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
|
24 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
|
25 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
|
26 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
|
27 |
# POSSIBILITY OF SUCH DAMAGE. |
|
28 |
# |
|
29 |
# The views and conclusions contained in the software and |
|
30 |
# documentation are those of the authors and should not be |
|
31 |
# interpreted as representing official policies, either expressed |
|
32 |
# or implied, of GRNET S.A. |
|
33 |
|
|
34 |
from optparse import make_option |
|
35 |
|
|
36 |
from django.core.management.base import NoArgsCommand, CommandError |
|
37 |
|
|
38 |
from pithos.api.swiss_army import SwissArmy |
|
39 |
|
|
40 |
|
|
41 |
class Command(NoArgsCommand): |
|
42 |
help = "Quotas migration helper" |
|
43 |
|
|
44 |
option_list = NoArgsCommand.option_list + ( |
|
45 |
make_option('--duplicate', |
|
46 |
dest='duplicate-accounts', |
|
47 |
action="store_true", |
|
48 |
default=True, |
|
49 |
help="Display case insensitive duplicate accounts."), |
|
50 |
make_option('--existing', |
|
51 |
dest='existing-accounts', |
|
52 |
action="store_true", |
|
53 |
default=False, |
|
54 |
help="Display existing accounts."), |
|
55 |
make_option('--merge-accounts', |
|
56 |
dest='merge_accounts', |
|
57 |
action='store_true', |
|
58 |
default=False, |
|
59 |
help="Merge SOURCE_ACCOUNT and DEST_ACCOUNT."), |
|
60 |
make_option('--delete-account', |
|
61 |
dest='delete_account', |
|
62 |
action='store', |
|
63 |
help="Account to be deleted."), |
|
64 |
make_option('--src-account', |
|
65 |
dest='src_account', |
|
66 |
action='store', |
|
67 |
help="Account to be merged and then deleted."), |
|
68 |
make_option('--dest-account', |
|
69 |
dest='dest_account', |
|
70 |
action='store', |
|
71 |
help="Account where SOURCE_ACCOUNT contents will move."), |
|
72 |
make_option('--dry', |
|
73 |
dest='dry', |
|
74 |
action="store_true", |
|
75 |
default=False, |
|
76 |
help="Do not commit database changes.") |
|
77 |
) |
|
78 |
|
|
79 |
def handle(self, *args, **options): |
|
80 |
try: |
|
81 |
utils = SwissArmy() |
|
82 |
self.strict = options.get('strict') |
|
83 |
self.dry = options.get('dry') |
|
84 |
|
|
85 |
if options.get('duplicate-accounts') and \ |
|
86 |
not options.get('existing-accounts') and \ |
|
87 |
not options.get('merge_accounts') and \ |
|
88 |
not options.get('delete_account'): |
|
89 |
duplicates = utils.duplicate_accounts() |
|
90 |
if duplicates: |
|
91 |
msg = "The following case insensitive duplicates found: %r" |
|
92 |
raise CommandError(msg % duplicates) |
|
93 |
else: |
|
94 |
print "No duplicate accounts are found." |
|
95 |
|
|
96 |
if options.get('existing-accounts') and \ |
|
97 |
not options.get('merge_accounts') and \ |
|
98 |
not options.get('delete_account'): |
|
99 |
accounts = utils.existing_accounts() |
|
100 |
print "The following accounts found:" |
|
101 |
print "%s" % '\n'.join(accounts) |
|
102 |
|
|
103 |
if options.get('merge_accounts'): |
|
104 |
src_account = options.get('src_account') |
|
105 |
dest_account = options.get('dest_account') |
|
106 |
if not src_account: |
|
107 |
raise CommandError('Please specify a source account') |
|
108 |
if not dest_account: |
|
109 |
raise CommandError('Please specify a destination account') |
|
110 |
utils.merge_account(src_account, dest_account, |
|
111 |
only_stats=True) |
|
112 |
|
|
113 |
confirm = raw_input("Type 'yes' if you are sure you want" |
|
114 |
" to remove those entries: ") |
|
115 |
if not confirm == 'yes': |
|
116 |
return |
|
117 |
else: |
|
118 |
utils.merge_account(options.get('src_account'), |
|
119 |
options.get('dest_account'), |
|
120 |
only_stats=False, |
|
121 |
dry=self.dry) |
|
122 |
return |
|
123 |
|
|
124 |
if options.get('delete_account'): |
|
125 |
utils.delete_account(options.get('delete_account'), |
|
126 |
only_stats=True) |
|
127 |
|
|
128 |
confirm = raw_input("Type 'yes' if you are sure you want" |
|
129 |
" to remove those entries: ") |
|
130 |
if not confirm == 'yes': |
|
131 |
return |
|
132 |
else: |
|
133 |
utils.delete_account(options.get('delete_account'), |
|
134 |
only_stats=False, |
|
135 |
dry=self.dry) |
|
136 |
return |
|
137 |
except Exception, e: |
|
138 |
raise CommandError(e) |
|
139 |
finally: |
|
140 |
utils.backend.close() |
b/snf-pithos-app/pithos/api/swiss_army/__init__.py | ||
---|---|---|
1 |
# Copyright 2012 GRNET S.A. All rights reserved. |
|
2 |
# |
|
3 |
# Redistribution and use in source and binary forms, with or |
|
4 |
# without modification, are permitted provided that the following |
|
5 |
# conditions are met: |
|
6 |
# |
|
7 |
# 1. Redistributions of source code must retain the above |
|
8 |
# copyright notice, this list of conditions and the following |
|
9 |
# disclaimer. |
|
10 |
# |
|
11 |
# 2. Redistributions in binary form must reproduce the above |
|
12 |
# copyright notice, this list of conditions and the following |
|
13 |
# disclaimer in the documentation and/or other materials |
|
14 |
# provided with the distribution. |
|
15 |
# |
|
16 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
|
17 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
|
18 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
|
19 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
|
20 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
21 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
22 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
|
23 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
|
24 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
|
25 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
|
26 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
|
27 |
# POSSIBILITY OF SUCH DAMAGE. |
|
28 |
# |
|
29 |
# The views and conclusions contained in the software and |
|
30 |
# documentation are those of the authors and should not be |
|
31 |
# interpreted as representing official policies, either expressed |
|
32 |
# or implied, of GRNET S.A. |
|
33 |
|
|
34 |
from pithos.api.util import get_backend, split_container_object_string |
|
35 |
|
|
36 |
import re |
|
37 |
import hashlib |
|
38 |
import os |
|
39 |
|
|
40 |
|
|
41 |
def data_read_iterator(str, size=1024): |
|
42 |
offset = 0 |
|
43 |
while True: |
|
44 |
data = str[offset:offset + size] |
|
45 |
offset = offset + size |
|
46 |
if not data: |
|
47 |
break |
|
48 |
yield data |
|
49 |
|
|
50 |
|
|
51 |
class SwissArmy(): |
|
52 |
def __init__(self): |
|
53 |
self.backend = get_backend() |
|
54 |
|
|
55 |
def cleanup(self): |
|
56 |
self.backend.close() |
|
57 |
|
|
58 |
def existing_accounts(self): |
|
59 |
return self.backend.node.node_accounts() |
|
60 |
|
|
61 |
def duplicate_accounts(self): |
|
62 |
accounts = self.existing_accounts() |
|
63 |
duplicates = [] |
|
64 |
for i in range(len(accounts)): |
|
65 |
account = accounts[i] |
|
66 |
matcher = re.compile(account, re.IGNORECASE) |
|
67 |
duplicate = filter(matcher.match, accounts[i + 1:]) |
|
68 |
if duplicate: |
|
69 |
duplicate.insert(0, account) |
|
70 |
duplicates.append(duplicate) |
|
71 |
return duplicates |
|
72 |
|
|
73 |
def list_all_containers(self, account, step=10): |
|
74 |
containers = [] |
|
75 |
marker = None |
|
76 |
while 1: |
|
77 |
more = self.backend.list_containers(account, account, limit=10, |
|
78 |
marker=marker) |
|
79 |
if not more: |
|
80 |
break |
|
81 |
containers.extend(more) |
|
82 |
marker = more[-1] |
|
83 |
return containers |
|
84 |
|
|
85 |
def list_all_container_objects(self, account, container, virtual=False): |
|
86 |
objects = [] |
|
87 |
marker = None |
|
88 |
while 1: |
|
89 |
more = self.backend.list_objects(account, account, container, |
|
90 |
marker=marker, virtual=virtual) |
|
91 |
if not more: |
|
92 |
break |
|
93 |
objects.extend((i[0] for i in more)) |
|
94 |
marker = more[-1][0] |
|
95 |
return objects |
|
96 |
|
|
97 |
def list_all_objects(self, account, virtual=False): |
|
98 |
containers = self.list_all_containers(account) |
|
99 |
objects = [] |
|
100 |
extend = objects.extend |
|
101 |
for c in containers: |
|
102 |
more = self.list_all_container_objects(account, c, virtual=virtual) |
|
103 |
extend([os.path.join(c, i) for i in more]) |
|
104 |
return objects |
|
105 |
|
|
106 |
def list_past_versions(self, account, container, name): |
|
107 |
versions = self.backend.list_versions(account, account, container, |
|
108 |
name) |
|
109 |
# do not return the current version |
|
110 |
return list(x[0] for x in versions[:-1]) |
|
111 |
|
|
112 |
def move_object(self, src_account, src_container, src_name, |
|
113 |
dest_account, dry=True, silent=False): |
|
114 |
|
|
115 |
trans = self.backend.wrapper.conn.begin() |
|
116 |
try: |
|
117 |
self._move_object(src_account, src_container, src_name, |
|
118 |
dest_account) |
|
119 |
|
|
120 |
if dry: |
|
121 |
if not silent: |
|
122 |
print "Skipping database commit." |
|
123 |
trans.rollback() |
|
124 |
else: |
|
125 |
trans.commit() |
|
126 |
if not silent: |
|
127 |
print "%s is deleted." % src_account |
|
128 |
except: |
|
129 |
trans.rollback() |
|
130 |
raise |
|
131 |
|
|
132 |
def _move_object(self, src_account, src_container, src_name, |
|
133 |
dest_account): |
|
134 |
path = os.path.join(src_container, src_name) |
|
135 |
fullpath = os.path.join(src_account, path) |
|
136 |
dest_container = src_container |
|
137 |
dest_name = src_name |
|
138 |
|
|
139 |
meta = self.backend.get_object_meta(src_account, src_account, |
|
140 |
src_container, src_name, 'pithos', |
|
141 |
version=None) |
|
142 |
content_type = meta.get('type') |
|
143 |
|
|
144 |
# get source object history |
|
145 |
versions = self.list_past_versions(src_account, src_container, |
|
146 |
src_name) |
|
147 |
|
|
148 |
# get source object permissions |
|
149 |
permissions = self.backend.permissions.access_get(fullpath) |
|
150 |
|
|
151 |
# get source object public |
|
152 |
public = self.backend.get_object_public(src_account, src_account, |
|
153 |
src_container, src_name) |
|
154 |
|
|
155 |
if dest_container in self.backend.list_containers(dest_account, |
|
156 |
dest_account): |
|
157 |
# Note: if dest_container contains an object with the same name |
|
158 |
# a new version with the contents of the source object will be |
|
159 |
# created and the one in the destination container will pass to |
|
160 |
# history |
|
161 |
self.backend.copy_object(dest_account, src_account, src_container, |
|
162 |
src_name, dest_account, dest_container, |
|
163 |
dest_name, content_type, 'pithos', |
|
164 |
meta={}, replace_meta=False, |
|
165 |
permissions=permissions) |
|
166 |
else: |
|
167 |
# create destination container and retry |
|
168 |
self.backend.put_container(dest_account, dest_account, |
|
169 |
dest_container) |
|
170 |
self.backend.copy_object(dest_account, src_account, src_container, |
|
171 |
src_name, dest_account, dest_container, |
|
172 |
dest_name, content_type, 'pithos', |
|
173 |
meta={}, replace_meta=False, |
|
174 |
permissions=permissions) |
|
175 |
|
|
176 |
self.backend.delete_object(src_account, src_account, src_container, |
|
177 |
src_name) |
|
178 |
|
|
179 |
dest_path, dest_node = self.backend._lookup_object(dest_account, |
|
180 |
dest_container, |
|
181 |
dest_name) |
|
182 |
assert dest_path == '/'.join([dest_account, path]) |
|
183 |
|
|
184 |
# turn history versions to point to the newly created node |
|
185 |
for serial in versions: |
|
186 |
self.backend.node.version_put_property(serial, 'node', dest_node) |
|
187 |
|
|
188 |
if public: |
|
189 |
# set destination object public |
|
190 |
fullpath = '/'.join([dest_account, dest_container, dest_name]) |
|
191 |
self.backend.permissions.public_set(fullpath) |
|
192 |
|
|
193 |
def _merge_account(self, src_account, dest_account): |
|
194 |
# TODO: handle exceptions |
|
195 |
# copy all source objects |
|
196 |
for path in self.list_all_objects(src_account): |
|
197 |
src_container, src_name = split_container_object_string( |
|
198 |
'/%s' % path) |
|
199 |
|
|
200 |
# give read permissions to the dest_account |
|
201 |
permissions = self.backend.get_object_permissions( |
|
202 |
src_account, src_account, src_container, src_name) |
|
203 |
if permissions: |
|
204 |
permissions = permissions[2] |
|
205 |
permissions['read'] = permissions.get('read', []) |
|
206 |
permissions['read'].append(dest_account) |
|
207 |
self.backend.update_object_permissions(src_account, |
|
208 |
src_account, |
|
209 |
src_container, |
|
210 |
src_name, |
|
211 |
permissions) |
|
212 |
|
|
213 |
self._move_object(src_account, src_container, src_name, |
|
214 |
dest_account) |
|
215 |
|
|
216 |
# move groups also |
|
217 |
groups = self.backend.get_account_groups(src_account, src_account) |
|
218 |
(v.replace(src_account, dest_account) for v in groups.values()) |
|
219 |
self.backend.update_account_groups(dest_account, dest_account, |
|
220 |
groups) |
|
221 |
self._delete_account(src_account) |
|
222 |
|
|
223 |
def merge_account(self, src_account, dest_account, only_stats=True, |
|
224 |
dry=True, silent=False): |
|
225 |
if only_stats: |
|
226 |
print "The following %s's entries will be moved to %s:" \ |
|
227 |
% (src_account, dest_account) |
|
228 |
print "Objects: %r" % self.list_all_objects(src_account) |
|
229 |
print "Groups: %r" \ |
|
230 |
% self.backend.get_account_groups(src_account, |
|
231 |
src_account).keys() |
|
232 |
return |
|
233 |
|
|
234 |
trans = self.backend.wrapper.conn.begin() |
|
235 |
try: |
|
236 |
self._merge_account(src_account, dest_account) |
|
237 |
|
|
238 |
if dry: |
|
239 |
if not silent: |
|
240 |
print "Skipping database commit." |
|
241 |
trans.rollback() |
|
242 |
else: |
|
243 |
trans.commit() |
|
244 |
if not silent: |
|
245 |
msg = "%s merged into %s and deleted." |
|
246 |
print msg % (src_account, dest_account) |
|
247 |
except: |
|
248 |
trans.rollback() |
|
249 |
raise |
|
250 |
|
|
251 |
def delete_container_contents(self, account, container): |
|
252 |
self.backend.delete_container(account, account, container, |
|
253 |
delimiter='/') |
|
254 |
|
|
255 |
def delete_container(self, account, container): |
|
256 |
self.backend.delete_container(account, account, container) |
|
257 |
|
|
258 |
def _delete_account(self, account): |
|
259 |
for c in self.list_all_containers(account): |
|
260 |
self.delete_container_contents(account, c) |
|
261 |
self.delete_container(account, c) |
|
262 |
self.backend.delete_account(account, account) |
|
263 |
|
|
264 |
def delete_account(self, account, only_stats=True, dry=True, silent=False): |
|
265 |
if only_stats: |
|
266 |
print "The following %s's entries will be removed:" % account |
|
267 |
print "Objects: %r" % self.list_all_objects(account) |
|
268 |
print "Groups: %r" \ |
|
269 |
% self.backend.get_account_groups(account, account).keys() |
|
270 |
return |
|
271 |
|
|
272 |
trans = self.backend.wrapper.conn.begin() |
|
273 |
try: |
|
274 |
self._delete_account(account) |
|
275 |
|
|
276 |
if dry: |
|
277 |
if not silent: |
|
278 |
print "Skipping database commit." |
|
279 |
trans.rollback() |
|
280 |
else: |
|
281 |
trans.commit() |
|
282 |
if not silent: |
|
283 |
print "%s is deleted." % account |
|
284 |
except: |
|
285 |
trans.rollback() |
|
286 |
raise |
|
287 |
|
|
288 |
def create_account(self, account): |
|
289 |
return self.backend._lookup_account(account, create=True) |
|
290 |
|
|
291 |
def create_update_object(self, account, container, name, content_type, |
|
292 |
data, meta={}, permissions={}, request_user=None): |
|
293 |
md5 = hashlib.md5() |
|
294 |
size = 0 |
|
295 |
hashmap = [] |
|
296 |
for block_data in data_read_iterator(data, self.backend.block_size): |
|
297 |
size += len(block_data) |
|
298 |
hashmap.append(self.backend.put_block(block_data)) |
|
299 |
md5.update(block_data) |
|
300 |
|
|
301 |
checksum = md5.hexdigest().lower() |
|
302 |
|
|
303 |
request_user = request_user or account |
|
304 |
return self.backend.update_object_hashmap(request_user, account, |
|
305 |
container, name, size, |
|
306 |
content_type, hashmap, |
|
307 |
checksum, 'pithos', meta, |
|
308 |
True, permissions) |
b/snf-pithos-app/pithos/api/swiss_army/tests.py | ||
---|---|---|
1 |
# Copyright 2011 GRNET S.A. All rights reserved. |
|
2 |
# |
|
3 |
# Redistribution and use in source and binary forms, with or |
|
4 |
# without modification, are permitted provided that the following |
|
5 |
# conditions are met: |
|
6 |
# |
|
7 |
# 1. Redistributions of source code must retain the above |
|
8 |
# copyright notice, this list of conditions and the following |
|
9 |
# disclaimer. |
|
10 |
# |
|
11 |
# 2. Redistributions in binary form must reproduce the above |
|
12 |
# copyright notice, this list of conditions and the following |
|
13 |
# disclaimer in the documentation and/or other materials |
|
14 |
# provided with the distribution. |
|
15 |
# |
|
16 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
|
17 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
|
18 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
|
19 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
|
20 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
21 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
22 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
|
23 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
|
24 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
|
25 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
|
26 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
|
27 |
# POSSIBILITY OF SUCH DAMAGE. |
|
28 |
# |
|
29 |
# The views and conclusions contained in the software and |
|
30 |
# documentation are those of the authors and should not be |
|
31 |
# interpreted as representing official policies, either expressed |
|
32 |
# or implied, of GRNET S.A. |
|
33 |
|
|
34 |
import unittest |
|
35 |
import uuid |
|
36 |
import random |
|
37 |
import string |
|
38 |
import os |
|
39 |
|
|
40 |
from collections import defaultdict |
|
41 |
|
|
42 |
from pithos.api.swiss_army import SwissArmy |
|
43 |
|
|
44 |
|
|
45 |
def get_random_data(length=500): |
|
46 |
char_set = string.ascii_uppercase + string.digits |
|
47 |
return ''.join(random.choice(char_set) for x in xrange(length)) |
|
48 |
|
|
49 |
|
|
50 |
class SwissArmyTests(unittest.TestCase): |
|
51 |
def setUp(self): |
|
52 |
self.utils = SwissArmy() |
|
53 |
self.accounts = ('account1', 'Account1', 'account2', 'account3') |
|
54 |
for i in self.accounts: |
|
55 |
self.utils.create_account(i) |
|
56 |
|
|
57 |
def tearDown(self): |
|
58 |
for i in self.accounts: |
|
59 |
self.utils._delete_account(i) |
|
60 |
self.utils.cleanup() |
|
61 |
|
|
62 |
def _verify_object(self, account, container, object, expected={}, |
|
63 |
strict=True): |
|
64 |
self._verify_object_metadata(account, container, object, |
|
65 |
expected.get('meta')) |
|
66 |
self._verify_object_history(account, container, object, |
|
67 |
expected.get('versions'), |
|
68 |
strict=strict) |
|
69 |
self._verify_object_permissions(account, container, object, |
|
70 |
expected.get('permissions')) |
|
71 |
|
|
72 |
def _verify_object_metadata(self, account, container, object, expected): |
|
73 |
object_meta = self.utils.backend.get_object_meta( |
|
74 |
account, account, container, object, 'pithos') |
|
75 |
for k in expected: |
|
76 |
self.assertTrue(k in object_meta) |
|
77 |
self.assertEquals(object_meta[k], expected[k]) |
|
78 |
|
|
79 |
def _verify_object_history(self, account, container, object, expected, |
|
80 |
strict=True): |
|
81 |
history = self.utils.list_past_versions(account, container, object) |
|
82 |
if strict: |
|
83 |
self.assertEquals(sorted(expected), history) |
|
84 |
else: |
|
85 |
self.assertTrue(set(expected) <= set(history)) |
|
86 |
|
|
87 |
def _verify_object_permissions(self, account, container, object, expected): |
|
88 |
expected = expected or {} |
|
89 |
perms_tuple = self.utils.backend.get_object_permissions( |
|
90 |
account, account, container, object) |
|
91 |
|
|
92 |
self.assertEqual(len(perms_tuple), 3) |
|
93 |
|
|
94 |
object_perms = perms_tuple[2] |
|
95 |
|
|
96 |
for k in expected: |
|
97 |
self.assertTrue(set(expected.get(k)) <= set(object_perms.get(k))) |
|
98 |
|
|
99 |
for holder in expected.get('read', []): |
|
100 |
if holder == '*': |
|
101 |
continue |
|
102 |
try: |
|
103 |
# check first for a group permission |
|
104 |
owner, group = holder.split(':') |
|
105 |
except ValueError: |
|
106 |
holders = [holder] |
|
107 |
else: |
|
108 |
holders = self.utils.backend.permissions.group_members(owner, |
|
109 |
group) |
|
110 |
|
|
111 |
for h in holders: |
|
112 |
try: |
|
113 |
self.utils.backend.get_object_meta( |
|
114 |
holder, account, container, object, 'pithos') |
|
115 |
except Exception, e: |
|
116 |
self.fail(e) |
|
117 |
|
|
118 |
def test_existing_accounts(self): |
|
119 |
accounts = self.utils.existing_accounts() |
|
120 |
self.assertEquals(sorted(accounts), accounts) |
|
121 |
self.assertTrue(set(['account1', 'account2']) <= set(accounts)) |
|
122 |
|
|
123 |
def test_duplicate_accounts(self): |
|
124 |
duplicates = self.utils.duplicate_accounts() |
|
125 |
self.assertTrue(['Account1', 'account1'] in duplicates) |
|
126 |
|
|
127 |
def test_list_all_containers(self): |
|
128 |
step = 10 |
|
129 |
containers = [] |
|
130 |
append = containers.append |
|
131 |
for i in range(3 * step + 1): |
|
132 |
while 1: |
|
133 |
cname = unicode(uuid.uuid4()) |
|
134 |
if cname not in containers: |
|
135 |
append(cname) |
|
136 |
break |
|
137 |
self.utils.backend.put_container('account1', 'account1', cname) |
|
138 |
self.assertEquals(sorted(containers), |
|
139 |
self.utils.list_all_containers('account1', |
|
140 |
step=step)) |
|
141 |
|
|
142 |
def test_list_all_container_objects(self): |
|
143 |
containers = ('container1', 'container2') |
|
144 |
objects = defaultdict(list) |
|
145 |
for c in containers: |
|
146 |
self.utils.backend.put_container('account1', 'account1', c) |
|
147 |
step = 10 |
|
148 |
append = objects[c].append |
|
149 |
content_type = 'application/octet-stream' |
|
150 |
for i in range(3 * step + 1): |
|
151 |
while 1: |
|
152 |
oname = unicode(uuid.uuid4()) |
|
153 |
if oname not in objects: |
|
154 |
append(oname) |
|
155 |
break |
|
156 |
data = get_random_data(int(random.random())) |
|
157 |
self.utils.create_update_object('account1', c, oname, |
|
158 |
content_type, data) |
|
159 |
|
|
160 |
(self.assertEquals(sorted(objects.get(c)), |
|
161 |
self.utils.list_all_container_objects('account1', c) |
|
162 |
) for c in containers) |
|
163 |
|
|
164 |
def test_list_all_objects(self): |
|
165 |
containers = ('container1', 'container2') |
|
166 |
objects = [] |
|
167 |
append = objects.append |
|
168 |
for c in containers: |
|
169 |
self.utils.backend.put_container('account1', 'account1', c) |
|
170 |
step = 10 |
|
171 |
content_type = 'application/octet-stream' |
|
172 |
for i in range(3 * step + 1): |
|
173 |
while 1: |
|
174 |
oname = unicode(uuid.uuid4()) |
|
175 |
if oname not in objects: |
|
176 |
append(os.path.join(c, oname)) |
|
177 |
break |
|
178 |
data = get_random_data(int(random.random())) |
|
179 |
self.utils.create_update_object('account1', c, oname, |
|
180 |
content_type, data) |
|
181 |
|
|
182 |
self.assertEquals(len(objects), |
|
183 |
len(self.utils.list_all_objects('account1'))) |
|
184 |
self.assertEquals(sorted(objects), |
|
185 |
self.utils.list_all_objects('account1')) |
|
186 |
|
|
187 |
def test_list_past_versions(self): |
|
188 |
self.utils.backend.put_container('account1', 'account1', 'container1') |
|
189 |
versions = [] |
|
190 |
append = versions.append |
|
191 |
for i in range(5): |
|
192 |
data = get_random_data(int(random.random())) |
|
193 |
append(self.utils.create_update_object('account1', 'container1', |
|
194 |
'object1', |
|
195 |
'application/octet-stream', |
|
196 |
data)) |
|
197 |
self.assertEquals(sorted(versions[:-1]), |
|
198 |
self.utils.list_past_versions('account1', |
|
199 |
'container1', |
|
200 |
'object1')) |
|
201 |
|
|
202 |
def test_move(self): |
|
203 |
# create containers |
|
204 |
self.utils.backend.put_container('account1', 'account1', 'container1') |
|
205 |
self.utils.backend.put_container('Account1', 'Account1', 'container1') |
|
206 |
|
|
207 |
# add group |
|
208 |
self.utils.backend.update_account_groups('Account1', 'Account1', |
|
209 |
{'test': ['account3']}) |
|
210 |
|
|
211 |
# upload object and update it several times |
|
212 |
versions = [] |
|
213 |
append = versions.append |
|
214 |
meta = {'foo': 'bar'} |
|
215 |
permissions = {'read': ['account1', 'account2', 'Account1:test'], |
|
216 |
'write': ['account2', 'Account1:test']} |
|
217 |
for i in range(5): |
|
218 |
data = get_random_data(int(random.random())) |
|
219 |
append(self.utils.create_update_object('Account1', 'container1', |
|
220 |
'object1', |
|
221 |
'application/octet-stream', |
|
222 |
data, meta, permissions)) |
|
223 |
|
|
224 |
self.utils.move_object('Account1', 'container1', 'object1', 'account1', |
|
225 |
dry=False, silent=True) |
|
226 |
|
|
227 |
expected = {'meta': meta, |
|
228 |
'versions': versions[:-1], |
|
229 |
'permissions': permissions} |
|
230 |
self._verify_object('account1', 'container1', 'object1', expected) |
|
231 |
|
|
232 |
def test_merge(self): |
|
233 |
# create container |
|
234 |
self.utils.backend.put_container('Account1', 'Account1', 'container0') |
|
235 |
self.utils.backend.put_container('Account1', 'Account1', 'container1') |
|
236 |
|
|
237 |
# add group |
|
238 |
self.utils.backend.update_account_groups('Account1', 'Account1', |
|
239 |
{'test': ['account3']}) |
|
240 |
|
|
241 |
# upload objects and update them several times |
|
242 |
versions = defaultdict(list) |
|
243 |
meta = {'foo': 'bar'} |
|
244 |
permissions = {'read': ['account2', 'Account1:test'], |
|
245 |
'write': ['account2', 'Account1:test']} |
|
246 |
|
|
247 |
for i in range(2): |
|
248 |
container = 'container%s' % i |
|
249 |
versions[container] = {} |
|
250 |
for j in range(3): |
|
251 |
object = 'object%s' % j |
|
252 |
versions[container][object] = [] |
|
253 |
append = versions[container][object].append |
|
254 |
for k in range(5): |
|
255 |
data = get_random_data(int(random.random())) |
|
256 |
append(self.utils.create_update_object( |
|
257 |
'Account1', container, object, |
|
258 |
'application/octet-stream', data, meta, permissions)) |
|
259 |
|
|
260 |
self.utils.merge_account('Account1', 'account1', only_stats=False, |
|
261 |
dry=False, silent=True) |
|
262 |
|
|
263 |
self.assertTrue('Account1' not in self.utils.existing_accounts()) |
|
264 |
self.assertTrue('account1' in self.utils.existing_accounts()) |
|
265 |
|
|
266 |
# assert container has been created |
|
267 |
try: |
|
268 |
self.utils.backend.get_container_meta('account1', 'account1', |
|
269 |
'container1', 'pithos') |
|
270 |
except NameError, e: |
|
271 |
self.fail(e) |
|
272 |
|
|
273 |
expected = {'meta': meta, |
|
274 |
'permissions': permissions} |
|
275 |
for c, o_dict in versions.iteritems(): |
|
276 |
for o, versions in o_dict.iteritems(): |
|
277 |
expected['versions'] = versions[:-1] |
|
278 |
self._verify_object('account1', c, o, expected) |
|
279 |
|
|
280 |
def test_merge_existing_dest_container(self): |
|
281 |
# create container |
|
282 |
self.utils.backend.put_container('Account1', 'Account1', 'container1') |
|
283 |
self.utils.backend.put_container('account1', 'account1', 'container1') |
|
284 |
|
|
285 |
# add group |
|
286 |
self.utils.backend.update_account_groups('Account1', 'Account1', |
|
287 |
{'test': ['account3']}) |
|
288 |
|
|
289 |
# upload objects and update them several times |
|
290 |
versions = defaultdict(list) |
|
291 |
meta = {'foo': 'bar'} |
|
292 |
permissions = {'read': ['account2', 'Account1:test'], |
|
293 |
'write': ['account2', 'Account1:test']} |
|
294 |
|
|
295 |
versions = [] |
|
296 |
append = versions.append |
|
297 |
for k in range(5): |
|
298 |
data = get_random_data(int(random.random())) |
|
299 |
append(self.utils.create_update_object( |
|
300 |
'Account1', 'container1', 'object1', |
|
301 |
'application/octet-stream', data, meta, permissions)) |
|
302 |
|
|
303 |
self.utils.merge_account('Account1', 'account1', only_stats=False, |
|
304 |
dry=False, silent=True) |
|
305 |
|
|
306 |
self.utils.merge_account('Account1', 'account1', only_stats=False, |
|
307 |
dry=False, silent=True) |
|
308 |
|
|
309 |
try: |
|
310 |
self.utils.backend.get_container_meta('Account1', 'Account1', |
|
311 |
'container1', 'pithos') |
|
312 |
except NameError: |
|
313 |
pass |
|
314 |
else: |
|
315 |
self.fail('Container not deleted') |
|
316 |
|
|
317 |
try: |
|
318 |
self.utils.backend.get_container_meta('account1', 'account1', |
|
319 |
'container1', 'pithos') |
|
320 |
except NameError, e: |
|
321 |
self.fail(e) |
|
322 |
|
|
323 |
expected = {'meta': meta, |
|
324 |
'versions': versions[:-1], |
|
325 |
'permissions': permissions} |
|
326 |
self._verify_object('account1', 'container1', 'object1', expected) |
|
327 |
|
|
328 |
def test_merge_existing_dest_object(self): |
|
329 |
# create container |
|
330 |
self.utils.backend.put_container('Account1', 'Account1', 'container1') |
|
331 |
self.utils.backend.put_container('account1', 'account1', 'container1') |
|
332 |
|
|
333 |
# add group |
|
334 |
self.utils.backend.update_account_groups('Account1', 'Account1', |
|
335 |
{'test': ['account3']}) |
|
336 |
|
|
337 |
# upload objects and update them several times |
|
338 |
versions = defaultdict(list) |
|
339 |
meta = {'foo': 'bar'} |
|
340 |
permissions = {'read': ['account2', 'Account1:test'], |
|
341 |
'write': ['account2', 'Account1:test']} |
|
342 |
|
|
343 |
container = 'container1' |
|
344 |
object = 'object1' |
|
345 |
versions = [] |
|
346 |
append = versions.append |
|
347 |
for k in range(5): |
|
348 |
data = get_random_data(int(random.random())) |
|
349 |
append(self.utils.create_update_object( |
|
350 |
'Account1', container, object, |
|
351 |
'application/octet-stream', data, meta, permissions)) |
|
352 |
data = get_random_data(int(random.random())) |
|
353 |
self.utils.create_update_object( |
|
354 |
'account1', container, object, |
|
355 |
'application/octet-stream', data, meta, permissions) |
|
356 |
|
|
357 |
self.utils.merge_account('Account1', 'account1', only_stats=False, |
|
358 |
dry=False, silent=True) |
|
359 |
|
|
360 |
try: |
|
361 |
self.utils.backend.get_container_meta('Account1', 'Account1', |
|
362 |
'container1', 'pithos') |
|
363 |
except NameError: |
|
364 |
pass |
|
365 |
else: |
|
366 |
self.fail('Container not deleted') |
|
367 |
|
|
368 |
try: |
|
369 |
self.utils.backend.get_container_meta('account1', 'account1', |
|
370 |
'container1', 'pithos') |
|
371 |
except NameError, e: |
|
372 |
self.fail(e) |
|
373 |
|
|
374 |
expected = {'meta': meta, |
|
375 |
'permissions': permissions, |
|
376 |
'versions': versions[:-1]} |
|
377 |
self._verify_object('account1', container, object, expected, |
|
378 |
strict=False) |
|
379 |
|
|
380 |
|
|
381 |
if __name__ == '__main__': |
|
382 |
unittest.main() |
b/snf-pithos-backend/pithos/backends/lib/sqlalchemy/node.py | ||
---|---|---|
438 | 438 |
self.conn.execute(s).close() |
439 | 439 |
return True |
440 | 440 |
|
441 |
def node_accounts(self): |
|
442 |
s = select([self.nodes.c.path]) |
|
443 |
s = s.where(and_(self.nodes.c.node != 0, self.nodes.c.parent == 0)) |
|
444 |
account_nodes = self.conn.execute(s).fetchall() |
|
445 |
return sorted(i[0] for i in account_nodes) |
|
446 |
|
|
441 | 447 |
def policy_get(self, node): |
442 | 448 |
s = select([self.policy.c.key, self.policy.c.value], |
443 | 449 |
self.policy.c.node == node) |
Also available in: Unified diff