Statistics
| Branch: | Tag: | Revision:

root / snf-pithos-app / pithos / api / swiss_army / __init__.py @ d1e7d2b4

History | View | Annotate | Download (13.3 kB)

1
# Copyright 2012 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
from pithos.api.util import get_backend, split_container_object_string
35

    
36
import re
37
import hashlib
38
import os
39

    
40

    
41
def data_read_iterator(str, size=1024):
42
    offset = 0
43
    while True:
44
        data = str[offset:offset + size]
45
        offset = offset + size
46
        if not data:
47
            break
48
        yield data
49

    
50

    
51
class SwissArmy():
52
    def __init__(self):
53
        self.backend = get_backend()
54

    
55
    def cleanup(self):
56
        self.backend.close()
57

    
58
    def existing_accounts(self):
59
        return sorted([path for path, _ in self.backend.node.node_accounts()])
60

    
61
    def duplicate_accounts(self):
62
        accounts = self.existing_accounts()
63
        duplicates = []
64
        for i in range(len(accounts)):
65
            account = accounts[i]
66
            matcher = re.compile(account, re.IGNORECASE)
67
            duplicate = filter(matcher.match, (i for i in accounts[i + 1:] \
68
                if len(i) == len(account)))
69
            if duplicate:
70
                duplicate.insert(0, account)
71
                duplicates.append(duplicate)
72
        return duplicates
73

    
74
    def list_all_containers(self, account, step=10):
75
        containers = []
76
        marker = None
77
        while 1:
78
            more = self.backend.list_containers(account, account, limit=10,
79
                                                marker=marker)
80
            if not more:
81
                break
82
            containers.extend(more)
83
            marker = more[-1]
84
        return containers
85

    
86
    def list_all_container_objects(self, account, container, virtual=False):
87
        objects = []
88
        marker = None
89
        while 1:
90
            more = self.backend.list_objects(account, account, container,
91
                                             marker=marker, virtual=virtual)
92
            if not more:
93
                break
94
            objects.extend((i[0] for i in more))
95
            marker = more[-1][0]
96
        return objects
97

    
98
    def list_all_objects(self, account, virtual=False):
99
        containers = self.list_all_containers(account)
100
        objects = []
101
        extend = objects.extend
102
        for c in containers:
103
            more = self.list_all_container_objects(account, c, virtual=virtual)
104
            extend([os.path.join(c, i) for i in more])
105
        return objects
106

    
107
    def list_past_versions(self, account, container, name):
108
        versions = self.backend.list_versions(account, account, container,
109
                                              name)
110
        # do not return the current version
111
        return list(x[0] for x in versions[:-1])
112

    
113
    def move_object(self, src_account, src_container, src_name,
114
                    dest_account, dry=True, silent=False):
115
        if src_account not in self.existing_accounts():
116
            raise NameError('%s does not exist' % src_account)
117
        if dest_account not in self.existing_accounts():
118
            raise NameError('%s does not exist' % dest_account)
119

    
120
        trans = self.backend.wrapper.conn.begin()
121
        try:
122
            self._copy_object(src_account, src_container, src_name,
123
                              dest_account, move=True)
124

    
125
            if dry:
126
                if not silent:
127
                    print "Skipping database commit."
128
                trans.rollback()
129
            else:
130
                trans.commit()
131
                if not silent:
132
                    print "%s is deleted." % src_account
133
        except:
134
            trans.rollback()
135
            raise
136

    
137
    def _copy_object(self, src_account, src_container, src_name,
138
                    dest_account, move=False):
139
        path = os.path.join(src_container, src_name)
140
        fullpath = os.path.join(src_account, path)
141
        dest_container = src_container
142
        dest_name = src_name
143

    
144
        meta = self.backend.get_object_meta(src_account, src_account,
145
                                            src_container, src_name, 'pithos',
146
                                            version=None)
147
        content_type = meta.get('type')
148

    
149
        # get source object history
150
        versions = self.list_past_versions(src_account, src_container,
151
                                           src_name)
152

    
153
        # get source object permissions
154
        permissions = self.backend.permissions.access_get(fullpath)
155

    
156
        # get source object public
157
        public = self.backend.get_object_public(src_account, src_account,
158
                                                src_container, src_name)
159

    
160
        if dest_container in self.backend.list_containers(dest_account,
161
                                                          dest_account):
162
            # Note: if dest_container contains an object with the same name
163
            # a new version with the contents of the source object will be
164
            # created and the one in the destination container will pass to
165
            # history
166
            self.backend.copy_object(dest_account, src_account, src_container,
167
                                     src_name, dest_account, dest_container,
168
                                     dest_name, content_type, 'pithos',
169
                                     meta={}, replace_meta=False,
170
                                     permissions=permissions)
171
        else:
172
            # create destination container and retry
173
            self.backend.put_container(dest_account, dest_account,
174
                                       dest_container)
175
            self.backend.copy_object(dest_account, src_account, src_container,
176
                                     src_name, dest_account, dest_container,
177
                                     dest_name, content_type, 'pithos',
178
                                     meta={}, replace_meta=False,
179
                                     permissions=permissions)
180

    
181
        if move:
182
            self.backend.delete_object(src_account, src_account,
183
                                       src_container, src_name)
184

    
185
        dest_path, dest_node = self.backend._lookup_object(dest_account,
186
                                                           dest_container,
187
                                                           dest_name)
188
        assert dest_path == '/'.join([dest_account, path])
189

    
190
        # turn history versions to point to the newly created node
191
        for serial in versions:
192
            self.backend.node.version_put_property(serial, 'node', dest_node)
193

    
194
        if public:
195
            # set destination object public
196
            fullpath = '/'.join([dest_account, dest_container, dest_name])
197
            self.backend.permissions.public_set(
198
                fullpath,
199
                self.backend.public_url_security,
200
                self.backend.public_url_alphabet
201
            )
202

    
203
    def _merge_account(self, src_account, dest_account, delete_src=False):
204
            # TODO: handle exceptions
205
            # copy all source objects
206
            for path in self.list_all_objects(src_account):
207
                src_container, src_name = split_container_object_string(
208
                    '/%s' % path)
209

    
210
                # give read permissions to the dest_account
211
                permissions = self.backend.get_object_permissions(
212
                    src_account, src_account, src_container, src_name)
213
                if permissions:
214
                    permissions = permissions[2]
215
                permissions['read'] = permissions.get('read', [])
216
                permissions['read'].append(dest_account)
217
                self.backend.update_object_permissions(src_account,
218
                                                       src_account,
219
                                                       src_container,
220
                                                       src_name,
221
                                                       permissions)
222

    
223
                self._copy_object(src_account, src_container, src_name,
224
                                 dest_account, move=delete_src)
225

    
226
            # move groups also
227
            groups = self.backend.get_account_groups(src_account, src_account)
228
            (v.replace(src_account, dest_account) for v in groups.values())
229
            self.backend.update_account_groups(dest_account, dest_account,
230
                                               groups)
231
            if delete_src:
232
                self._delete_account(src_account)
233

    
234
    def merge_account(self, src_account, dest_account, only_stats=True,
235
                      dry=True, silent=False, delete_src=False):
236
        if src_account not in self.existing_accounts():
237
            raise NameError('%s does not exist' % src_account)
238
        if dest_account not in self.existing_accounts():
239
            raise NameError('%s does not exist' % dest_account)
240

    
241
        if only_stats:
242
            print "The following %s's entries will be moved to %s:" \
243
                % (src_account, dest_account)
244
            print "Objects: %r" % self.list_all_objects(src_account)
245
            print "Groups: %r" \
246
                % self.backend.get_account_groups(src_account,
247
                                                  src_account).keys()
248
            return
249

    
250
        trans = self.backend.wrapper.conn.begin()
251
        try:
252
            self._merge_account(src_account, dest_account, delete_src)
253

    
254
            if dry:
255
                if not silent:
256
                    print "Skipping database commit."
257
                trans.rollback()
258
            else:
259
                trans.commit()
260
                if not silent:
261
                    msg = "%s merged into %s."
262
                    print msg % (src_account, dest_account)
263
        except:
264
            trans.rollback()
265
            raise
266

    
267
    def delete_container_contents(self, account, container):
268
        self.backend.delete_container(account, account, container,
269
                                      delimiter='/')
270

    
271
    def delete_container(self, account, container):
272
        self.backend.delete_container(account, account, container)
273

    
274
    def _delete_account(self, account):
275
        for c in self.list_all_containers(account):
276
            self.delete_container_contents(account, c)
277
            self.delete_container(account, c)
278
        self.backend.delete_account(account, account)
279

    
280
    def delete_account(self, account, only_stats=True, dry=True, silent=False):
281
        if account not in self.existing_accounts():
282
            raise NameError('%s does not exist' % account)
283
        if only_stats:
284
            print "The following %s's entries will be removed:" % account
285
            print "Objects: %r" % self.list_all_objects(account)
286
            print "Groups: %r" \
287
                % self.backend.get_account_groups(account, account).keys()
288
            return
289

    
290
        trans = self.backend.wrapper.conn.begin()
291
        try:
292
            self._delete_account(account)
293

    
294
            if dry:
295
                if not silent:
296
                    print "Skipping database commit."
297
                trans.rollback()
298
            else:
299
                trans.commit()
300
                if not silent:
301
                    print "%s is deleted." % account
302
        except:
303
            trans.rollback()
304
            raise
305

    
306
    def create_account(self, account):
307
        return self.backend._lookup_account(account, create=True)
308

    
309
    def create_update_object(self, account, container, name, content_type,
310
                             data, meta=None, permissions=None, request_user=None):
311
        meta = meta or {}
312
        permissions = permissions or {}
313
        md5 = hashlib.md5()
314
        size = 0
315
        hashmap = []
316
        for block_data in data_read_iterator(data, self.backend.block_size):
317
            size += len(block_data)
318
            hashmap.append(self.backend.put_block(block_data))
319
            md5.update(block_data)
320

    
321
        checksum = md5.hexdigest().lower()
322

    
323
        request_user = request_user or account
324
        return self.backend.update_object_hashmap(request_user, account,
325
                                                  container, name, size,
326
                                                  content_type, hashmap,
327
                                                  checksum, 'pithos', meta,
328
                                                  True, permissions)