root / snf-pithos-app / pithos / api / swiss_army / __init__.py @ 94bff756
History | View | Annotate | Download (12.3 kB)
1 |
# Copyright 2012 GRNET S.A. All rights reserved.
|
---|---|
2 |
#
|
3 |
# Redistribution and use in source and binary forms, with or
|
4 |
# without modification, are permitted provided that the following
|
5 |
# conditions are met:
|
6 |
#
|
7 |
# 1. Redistributions of source code must retain the above
|
8 |
# copyright notice, this list of conditions and the following
|
9 |
# disclaimer.
|
10 |
#
|
11 |
# 2. Redistributions in binary form must reproduce the above
|
12 |
# copyright notice, this list of conditions and the following
|
13 |
# disclaimer in the documentation and/or other materials
|
14 |
# provided with the distribution.
|
15 |
#
|
16 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
|
17 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
18 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
19 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
|
20 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
21 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
22 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
23 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
24 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
25 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
26 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
27 |
# POSSIBILITY OF SUCH DAMAGE.
|
28 |
#
|
29 |
# The views and conclusions contained in the software and
|
30 |
# documentation are those of the authors and should not be
|
31 |
# interpreted as representing official policies, either expressed
|
32 |
# or implied, of GRNET S.A.
|
33 |
|
34 |
from pithos.api.util import get_backend, split_container_object_string |
35 |
|
36 |
import re |
37 |
import hashlib |
38 |
import os |
39 |
|
40 |
|
41 |
def data_read_iterator(str, size=1024): |
42 |
offset = 0
|
43 |
while True: |
44 |
data = str[offset:offset + size]
|
45 |
offset = offset + size |
46 |
if not data: |
47 |
break
|
48 |
yield data
|
49 |
|
50 |
|
51 |
class SwissArmy(): |
52 |
def __init__(self): |
53 |
self.backend = get_backend()
|
54 |
|
55 |
def cleanup(self): |
56 |
self.backend.close()
|
57 |
|
58 |
def existing_accounts(self): |
59 |
return self.backend.node.node_accounts() |
60 |
|
61 |
def duplicate_accounts(self): |
62 |
accounts = self.existing_accounts()
|
63 |
duplicates = [] |
64 |
for i in range(len(accounts)): |
65 |
account = accounts[i] |
66 |
matcher = re.compile(account, re.IGNORECASE) |
67 |
duplicate = filter(matcher.match, accounts[i + 1:]) |
68 |
if duplicate:
|
69 |
duplicate.insert(0, account)
|
70 |
duplicates.append(duplicate) |
71 |
return duplicates
|
72 |
|
73 |
def list_all_containers(self, account, step=10): |
74 |
containers = [] |
75 |
marker = None
|
76 |
while 1: |
77 |
more = self.backend.list_containers(account, account, limit=10, |
78 |
marker=marker) |
79 |
if not more: |
80 |
break
|
81 |
containers.extend(more) |
82 |
marker = more[-1]
|
83 |
return containers
|
84 |
|
85 |
def list_all_container_objects(self, account, container, virtual=False): |
86 |
objects = [] |
87 |
marker = None
|
88 |
while 1: |
89 |
more = self.backend.list_objects(account, account, container,
|
90 |
marker=marker, virtual=virtual) |
91 |
if not more: |
92 |
break
|
93 |
objects.extend((i[0] for i in more)) |
94 |
marker = more[-1][0] |
95 |
return objects
|
96 |
|
97 |
def list_all_objects(self, account, virtual=False): |
98 |
containers = self.list_all_containers(account)
|
99 |
objects = [] |
100 |
extend = objects.extend |
101 |
for c in containers: |
102 |
more = self.list_all_container_objects(account, c, virtual=virtual)
|
103 |
extend([os.path.join(c, i) for i in more]) |
104 |
return objects
|
105 |
|
106 |
def list_past_versions(self, account, container, name): |
107 |
versions = self.backend.list_versions(account, account, container,
|
108 |
name) |
109 |
# do not return the current version
|
110 |
return list(x[0] for x in versions[:-1]) |
111 |
|
112 |
def move_object(self, src_account, src_container, src_name, |
113 |
dest_account, dry=True, silent=False): |
114 |
|
115 |
trans = self.backend.wrapper.conn.begin()
|
116 |
try:
|
117 |
self._move_object(src_account, src_container, src_name,
|
118 |
dest_account) |
119 |
|
120 |
if dry:
|
121 |
if not silent: |
122 |
print "Skipping database commit." |
123 |
trans.rollback() |
124 |
else:
|
125 |
trans.commit() |
126 |
if not silent: |
127 |
print "%s is deleted." % src_account |
128 |
except:
|
129 |
trans.rollback() |
130 |
raise
|
131 |
|
132 |
def _move_object(self, src_account, src_container, src_name, |
133 |
dest_account): |
134 |
path = os.path.join(src_container, src_name) |
135 |
fullpath = os.path.join(src_account, path) |
136 |
dest_container = src_container |
137 |
dest_name = src_name |
138 |
|
139 |
meta = self.backend.get_object_meta(src_account, src_account,
|
140 |
src_container, src_name, 'pithos',
|
141 |
version=None)
|
142 |
content_type = meta.get('type')
|
143 |
|
144 |
# get source object history
|
145 |
versions = self.list_past_versions(src_account, src_container,
|
146 |
src_name) |
147 |
|
148 |
# get source object permissions
|
149 |
permissions = self.backend.permissions.access_get(fullpath)
|
150 |
|
151 |
# get source object public
|
152 |
public = self.backend.get_object_public(src_account, src_account,
|
153 |
src_container, src_name) |
154 |
|
155 |
if dest_container in self.backend.list_containers(dest_account, |
156 |
dest_account): |
157 |
# Note: if dest_container contains an object with the same name
|
158 |
# a new version with the contents of the source object will be
|
159 |
# created and the one in the destination container will pass to
|
160 |
# history
|
161 |
self.backend.copy_object(dest_account, src_account, src_container,
|
162 |
src_name, dest_account, dest_container, |
163 |
dest_name, content_type, 'pithos',
|
164 |
meta={}, replace_meta=False,
|
165 |
permissions=permissions) |
166 |
else:
|
167 |
# create destination container and retry
|
168 |
self.backend.put_container(dest_account, dest_account,
|
169 |
dest_container) |
170 |
self.backend.copy_object(dest_account, src_account, src_container,
|
171 |
src_name, dest_account, dest_container, |
172 |
dest_name, content_type, 'pithos',
|
173 |
meta={}, replace_meta=False,
|
174 |
permissions=permissions) |
175 |
|
176 |
self.backend.delete_object(src_account, src_account, src_container,
|
177 |
src_name) |
178 |
|
179 |
dest_path, dest_node = self.backend._lookup_object(dest_account,
|
180 |
dest_container, |
181 |
dest_name) |
182 |
assert dest_path == '/'.join([dest_account, path]) |
183 |
|
184 |
# turn history versions to point to the newly created node
|
185 |
for serial in versions: |
186 |
self.backend.node.version_put_property(serial, 'node', dest_node) |
187 |
|
188 |
if public:
|
189 |
# set destination object public
|
190 |
fullpath = '/'.join([dest_account, dest_container, dest_name])
|
191 |
self.backend.permissions.public_set(fullpath)
|
192 |
|
193 |
def _merge_account(self, src_account, dest_account): |
194 |
# TODO: handle exceptions
|
195 |
# copy all source objects
|
196 |
for path in self.list_all_objects(src_account): |
197 |
src_container, src_name = split_container_object_string( |
198 |
'/%s' % path)
|
199 |
|
200 |
# give read permissions to the dest_account
|
201 |
permissions = self.backend.get_object_permissions(
|
202 |
src_account, src_account, src_container, src_name) |
203 |
if permissions:
|
204 |
permissions = permissions[2]
|
205 |
permissions['read'] = permissions.get('read', []) |
206 |
permissions['read'].append(dest_account)
|
207 |
self.backend.update_object_permissions(src_account,
|
208 |
src_account, |
209 |
src_container, |
210 |
src_name, |
211 |
permissions) |
212 |
|
213 |
self._move_object(src_account, src_container, src_name,
|
214 |
dest_account) |
215 |
|
216 |
# move groups also
|
217 |
groups = self.backend.get_account_groups(src_account, src_account)
|
218 |
(v.replace(src_account, dest_account) for v in groups.values()) |
219 |
self.backend.update_account_groups(dest_account, dest_account,
|
220 |
groups) |
221 |
self._delete_account(src_account)
|
222 |
|
223 |
def merge_account(self, src_account, dest_account, only_stats=True, |
224 |
dry=True, silent=False): |
225 |
if only_stats:
|
226 |
print "The following %s's entries will be moved to %s:" \ |
227 |
% (src_account, dest_account) |
228 |
print "Objects: %r" % self.list_all_objects(src_account) |
229 |
print "Groups: %r" \ |
230 |
% self.backend.get_account_groups(src_account,
|
231 |
src_account).keys() |
232 |
return
|
233 |
|
234 |
trans = self.backend.wrapper.conn.begin()
|
235 |
try:
|
236 |
self._merge_account(src_account, dest_account)
|
237 |
|
238 |
if dry:
|
239 |
if not silent: |
240 |
print "Skipping database commit." |
241 |
trans.rollback() |
242 |
else:
|
243 |
trans.commit() |
244 |
if not silent: |
245 |
msg = "%s merged into %s and deleted."
|
246 |
print msg % (src_account, dest_account)
|
247 |
except:
|
248 |
trans.rollback() |
249 |
raise
|
250 |
|
251 |
def delete_container_contents(self, account, container): |
252 |
self.backend.delete_container(account, account, container,
|
253 |
delimiter='/')
|
254 |
|
255 |
def delete_container(self, account, container): |
256 |
self.backend.delete_container(account, account, container)
|
257 |
|
258 |
def _delete_account(self, account): |
259 |
for c in self.list_all_containers(account): |
260 |
self.delete_container_contents(account, c)
|
261 |
self.delete_container(account, c)
|
262 |
self.backend.delete_account(account, account)
|
263 |
|
264 |
def delete_account(self, account, only_stats=True, dry=True, silent=False): |
265 |
if only_stats:
|
266 |
print "The following %s's entries will be removed:" % account |
267 |
print "Objects: %r" % self.list_all_objects(account) |
268 |
print "Groups: %r" \ |
269 |
% self.backend.get_account_groups(account, account).keys()
|
270 |
return
|
271 |
|
272 |
trans = self.backend.wrapper.conn.begin()
|
273 |
try:
|
274 |
self._delete_account(account)
|
275 |
|
276 |
if dry:
|
277 |
if not silent: |
278 |
print "Skipping database commit." |
279 |
trans.rollback() |
280 |
else:
|
281 |
trans.commit() |
282 |
if not silent: |
283 |
print "%s is deleted." % account |
284 |
except:
|
285 |
trans.rollback() |
286 |
raise
|
287 |
|
288 |
def create_account(self, account): |
289 |
return self.backend._lookup_account(account, create=True) |
290 |
|
291 |
def create_update_object(self, account, container, name, content_type, |
292 |
data, meta={}, permissions={}, request_user=None):
|
293 |
md5 = hashlib.md5() |
294 |
size = 0
|
295 |
hashmap = [] |
296 |
for block_data in data_read_iterator(data, self.backend.block_size): |
297 |
size += len(block_data)
|
298 |
hashmap.append(self.backend.put_block(block_data))
|
299 |
md5.update(block_data) |
300 |
|
301 |
checksum = md5.hexdigest().lower() |
302 |
|
303 |
request_user = request_user or account
|
304 |
return self.backend.update_object_hashmap(request_user, account, |
305 |
container, name, size, |
306 |
content_type, hashmap, |
307 |
checksum, 'pithos', meta,
|
308 |
True, permissions)
|