root / snf-pithos-app / pithos / api / swiss_army / __init__.py @ 99c11993
History | View | Annotate | Download (13 kB)
1 |
# Copyright 2012 GRNET S.A. All rights reserved.
|
---|---|
2 |
#
|
3 |
# Redistribution and use in source and binary forms, with or
|
4 |
# without modification, are permitted provided that the following
|
5 |
# conditions are met:
|
6 |
#
|
7 |
# 1. Redistributions of source code must retain the above
|
8 |
# copyright notice, this list of conditions and the following
|
9 |
# disclaimer.
|
10 |
#
|
11 |
# 2. Redistributions in binary form must reproduce the above
|
12 |
# copyright notice, this list of conditions and the following
|
13 |
# disclaimer in the documentation and/or other materials
|
14 |
# provided with the distribution.
|
15 |
#
|
16 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
|
17 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
18 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
19 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
|
20 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
21 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
22 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
23 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
24 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
25 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
26 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
27 |
# POSSIBILITY OF SUCH DAMAGE.
|
28 |
#
|
29 |
# The views and conclusions contained in the software and
|
30 |
# documentation are those of the authors and should not be
|
31 |
# interpreted as representing official policies, either expressed
|
32 |
# or implied, of GRNET S.A.
|
33 |
|
34 |
from pithos.api.util import get_backend, split_container_object_string |
35 |
|
36 |
import re |
37 |
import hashlib |
38 |
import os |
39 |
|
40 |
|
41 |
def data_read_iterator(str, size=1024): |
42 |
offset = 0
|
43 |
while True: |
44 |
data = str[offset:offset + size]
|
45 |
offset = offset + size |
46 |
if not data: |
47 |
break
|
48 |
yield data
|
49 |
|
50 |
|
51 |
class SwissArmy(): |
52 |
def __init__(self): |
53 |
self.backend = get_backend()
|
54 |
|
55 |
def cleanup(self): |
56 |
self.backend.close()
|
57 |
|
58 |
def existing_accounts(self): |
59 |
return self.backend.node.node_accounts() |
60 |
|
61 |
def duplicate_accounts(self): |
62 |
accounts = self.existing_accounts()
|
63 |
duplicates = [] |
64 |
for i in range(len(accounts)): |
65 |
account = accounts[i] |
66 |
matcher = re.compile(account, re.IGNORECASE) |
67 |
duplicate = filter(matcher.match, (i for i in accounts[i + 1:] \ |
68 |
if len(i) == len(account))) |
69 |
if duplicate:
|
70 |
duplicate.insert(0, account)
|
71 |
duplicates.append(duplicate) |
72 |
return duplicates
|
73 |
|
74 |
def list_all_containers(self, account, step=10): |
75 |
containers = [] |
76 |
marker = None
|
77 |
while 1: |
78 |
more = self.backend.list_containers(account, account, limit=10, |
79 |
marker=marker) |
80 |
if not more: |
81 |
break
|
82 |
containers.extend(more) |
83 |
marker = more[-1]
|
84 |
return containers
|
85 |
|
86 |
def list_all_container_objects(self, account, container, virtual=False): |
87 |
objects = [] |
88 |
marker = None
|
89 |
while 1: |
90 |
more = self.backend.list_objects(account, account, container,
|
91 |
marker=marker, virtual=virtual) |
92 |
if not more: |
93 |
break
|
94 |
objects.extend((i[0] for i in more)) |
95 |
marker = more[-1][0] |
96 |
return objects
|
97 |
|
98 |
def list_all_objects(self, account, virtual=False): |
99 |
containers = self.list_all_containers(account)
|
100 |
objects = [] |
101 |
extend = objects.extend |
102 |
for c in containers: |
103 |
more = self.list_all_container_objects(account, c, virtual=virtual)
|
104 |
extend([os.path.join(c, i) for i in more]) |
105 |
return objects
|
106 |
|
107 |
def list_past_versions(self, account, container, name): |
108 |
versions = self.backend.list_versions(account, account, container,
|
109 |
name) |
110 |
# do not return the current version
|
111 |
return list(x[0] for x in versions[:-1]) |
112 |
|
113 |
def move_object(self, src_account, src_container, src_name, |
114 |
dest_account, dry=True, silent=False): |
115 |
if src_account not in self.existing_accounts(): |
116 |
raise NameError('%s does not exist' % src_account) |
117 |
if dest_account not in self.existing_accounts(): |
118 |
raise NameError('%s does not exist' % dest_account) |
119 |
|
120 |
trans = self.backend.wrapper.conn.begin()
|
121 |
try:
|
122 |
self._copy_object(src_account, src_container, src_name,
|
123 |
dest_account, move=True)
|
124 |
|
125 |
if dry:
|
126 |
if not silent: |
127 |
print "Skipping database commit." |
128 |
trans.rollback() |
129 |
else:
|
130 |
trans.commit() |
131 |
if not silent: |
132 |
print "%s is deleted." % src_account |
133 |
except:
|
134 |
trans.rollback() |
135 |
raise
|
136 |
|
137 |
def _copy_object(self, src_account, src_container, src_name, |
138 |
dest_account, move=False):
|
139 |
path = os.path.join(src_container, src_name) |
140 |
fullpath = os.path.join(src_account, path) |
141 |
dest_container = src_container |
142 |
dest_name = src_name |
143 |
|
144 |
meta = self.backend.get_object_meta(src_account, src_account,
|
145 |
src_container, src_name, 'pithos',
|
146 |
version=None)
|
147 |
content_type = meta.get('type')
|
148 |
|
149 |
# get source object history
|
150 |
versions = self.list_past_versions(src_account, src_container,
|
151 |
src_name) |
152 |
|
153 |
# get source object permissions
|
154 |
permissions = self.backend.permissions.access_get(fullpath)
|
155 |
|
156 |
# get source object public
|
157 |
public = self.backend.get_object_public(src_account, src_account,
|
158 |
src_container, src_name) |
159 |
|
160 |
if dest_container in self.backend.list_containers(dest_account, |
161 |
dest_account): |
162 |
# Note: if dest_container contains an object with the same name
|
163 |
# a new version with the contents of the source object will be
|
164 |
# created and the one in the destination container will pass to
|
165 |
# history
|
166 |
self.backend.copy_object(dest_account, src_account, src_container,
|
167 |
src_name, dest_account, dest_container, |
168 |
dest_name, content_type, 'pithos',
|
169 |
meta={}, replace_meta=False,
|
170 |
permissions=permissions) |
171 |
else:
|
172 |
# create destination container and retry
|
173 |
self.backend.put_container(dest_account, dest_account,
|
174 |
dest_container) |
175 |
self.backend.copy_object(dest_account, src_account, src_container,
|
176 |
src_name, dest_account, dest_container, |
177 |
dest_name, content_type, 'pithos',
|
178 |
meta={}, replace_meta=False,
|
179 |
permissions=permissions) |
180 |
|
181 |
if move:
|
182 |
self.backend.delete_object(src_account, src_account,
|
183 |
src_container, src_name) |
184 |
|
185 |
dest_path, dest_node = self.backend._lookup_object(dest_account,
|
186 |
dest_container, |
187 |
dest_name) |
188 |
assert dest_path == '/'.join([dest_account, path]) |
189 |
|
190 |
# turn history versions to point to the newly created node
|
191 |
for serial in versions: |
192 |
self.backend.node.version_put_property(serial, 'node', dest_node) |
193 |
|
194 |
if public:
|
195 |
# set destination object public
|
196 |
fullpath = '/'.join([dest_account, dest_container, dest_name])
|
197 |
self.backend.permissions.public_set(fullpath)
|
198 |
|
199 |
def _merge_account(self, src_account, dest_account, delete_src=False): |
200 |
# TODO: handle exceptions
|
201 |
# copy all source objects
|
202 |
for path in self.list_all_objects(src_account): |
203 |
src_container, src_name = split_container_object_string( |
204 |
'/%s' % path)
|
205 |
|
206 |
# give read permissions to the dest_account
|
207 |
permissions = self.backend.get_object_permissions(
|
208 |
src_account, src_account, src_container, src_name) |
209 |
if permissions:
|
210 |
permissions = permissions[2]
|
211 |
permissions['read'] = permissions.get('read', []) |
212 |
permissions['read'].append(dest_account)
|
213 |
self.backend.update_object_permissions(src_account,
|
214 |
src_account, |
215 |
src_container, |
216 |
src_name, |
217 |
permissions) |
218 |
|
219 |
self._copy_object(src_account, src_container, src_name,
|
220 |
dest_account, move=delete_src) |
221 |
|
222 |
# move groups also
|
223 |
groups = self.backend.get_account_groups(src_account, src_account)
|
224 |
(v.replace(src_account, dest_account) for v in groups.values()) |
225 |
self.backend.update_account_groups(dest_account, dest_account,
|
226 |
groups) |
227 |
if delete_src:
|
228 |
self._delete_account(src_account)
|
229 |
|
230 |
def merge_account(self, src_account, dest_account, only_stats=True, |
231 |
dry=True, silent=False, delete_src=False): |
232 |
if src_account not in self.existing_accounts(): |
233 |
raise NameError('%s does not exist' % src_account) |
234 |
if dest_account not in self.existing_accounts(): |
235 |
raise NameError('%s does not exist' % dest_account) |
236 |
|
237 |
if only_stats:
|
238 |
print "The following %s's entries will be moved to %s:" \ |
239 |
% (src_account, dest_account) |
240 |
print "Objects: %r" % self.list_all_objects(src_account) |
241 |
print "Groups: %r" \ |
242 |
% self.backend.get_account_groups(src_account,
|
243 |
src_account).keys() |
244 |
return
|
245 |
|
246 |
trans = self.backend.wrapper.conn.begin()
|
247 |
try:
|
248 |
self._merge_account(src_account, dest_account, delete_src)
|
249 |
|
250 |
if dry:
|
251 |
if not silent: |
252 |
print "Skipping database commit." |
253 |
trans.rollback() |
254 |
else:
|
255 |
trans.commit() |
256 |
if not silent: |
257 |
msg = "%s merged into %s."
|
258 |
print msg % (src_account, dest_account)
|
259 |
except:
|
260 |
trans.rollback() |
261 |
raise
|
262 |
|
263 |
def delete_container_contents(self, account, container): |
264 |
self.backend.delete_container(account, account, container,
|
265 |
delimiter='/')
|
266 |
|
267 |
def delete_container(self, account, container): |
268 |
self.backend.delete_container(account, account, container)
|
269 |
|
270 |
def _delete_account(self, account): |
271 |
for c in self.list_all_containers(account): |
272 |
self.delete_container_contents(account, c)
|
273 |
self.delete_container(account, c)
|
274 |
self.backend.delete_account(account, account)
|
275 |
|
276 |
def delete_account(self, account, only_stats=True, dry=True, silent=False): |
277 |
if account not in self.existing_accounts(): |
278 |
raise NameError('%s does not exist' % account) |
279 |
if only_stats:
|
280 |
print "The following %s's entries will be removed:" % account |
281 |
print "Objects: %r" % self.list_all_objects(account) |
282 |
print "Groups: %r" \ |
283 |
% self.backend.get_account_groups(account, account).keys()
|
284 |
return
|
285 |
|
286 |
trans = self.backend.wrapper.conn.begin()
|
287 |
try:
|
288 |
self._delete_account(account)
|
289 |
|
290 |
if dry:
|
291 |
if not silent: |
292 |
print "Skipping database commit." |
293 |
trans.rollback() |
294 |
else:
|
295 |
trans.commit() |
296 |
if not silent: |
297 |
print "%s is deleted." % account |
298 |
except:
|
299 |
trans.rollback() |
300 |
raise
|
301 |
|
302 |
def create_account(self, account): |
303 |
return self.backend._lookup_account(account, create=True) |
304 |
|
305 |
def create_update_object(self, account, container, name, content_type, |
306 |
data, meta={}, permissions={}, request_user=None):
|
307 |
md5 = hashlib.md5() |
308 |
size = 0
|
309 |
hashmap = [] |
310 |
for block_data in data_read_iterator(data, self.backend.block_size): |
311 |
size += len(block_data)
|
312 |
hashmap.append(self.backend.put_block(block_data))
|
313 |
md5.update(block_data) |
314 |
|
315 |
checksum = md5.hexdigest().lower() |
316 |
|
317 |
request_user = request_user or account
|
318 |
return self.backend.update_object_hashmap(request_user, account, |
319 |
container, name, size, |
320 |
content_type, hashmap, |
321 |
checksum, 'pithos', meta,
|
322 |
True, permissions)
|