Revision 94bff756

b/snf-pithos-app/pithos/api/management/commands/pithos-manage-accounts.py
1
# Copyright 2012 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

  
34
from optparse import make_option
35

  
36
from django.core.management.base import NoArgsCommand, CommandError
37

  
38
from pithos.api.swiss_army import SwissArmy
39

  
40

  
41
class Command(NoArgsCommand):
42
    help = "Quotas migration helper"
43

  
44
    option_list = NoArgsCommand.option_list + (
45
        make_option('--duplicate',
46
                    dest='duplicate-accounts',
47
                    action="store_true",
48
                    default=True,
49
                    help="Display case insensitive duplicate accounts."),
50
        make_option('--existing',
51
                    dest='existing-accounts',
52
                    action="store_true",
53
                    default=False,
54
                    help="Display existing accounts."),
55
        make_option('--merge-accounts',
56
                    dest='merge_accounts',
57
                    action='store_true',
58
                    default=False,
59
                    help="Merge SOURCE_ACCOUNT and DEST_ACCOUNT."),
60
        make_option('--delete-account',
61
                    dest='delete_account',
62
                    action='store',
63
                    help="Account to be deleted."),
64
        make_option('--src-account',
65
                    dest='src_account',
66
                    action='store',
67
                    help="Account to be merged and then deleted."),
68
        make_option('--dest-account',
69
                    dest='dest_account',
70
                    action='store',
71
                    help="Account where SOURCE_ACCOUNT contents will move."),
72
        make_option('--dry',
73
                    dest='dry',
74
                    action="store_true",
75
                    default=False,
76
                    help="Do not commit database changes.")
77
    )
78

  
79
    def handle(self, *args, **options):
80
        try:
81
            utils = SwissArmy()
82
            self.strict = options.get('strict')
83
            self.dry = options.get('dry')
84

  
85
            if options.get('duplicate-accounts') and \
86
                    not options.get('existing-accounts') and \
87
                    not options.get('merge_accounts') and \
88
                    not options.get('delete_account'):
89
                duplicates = utils.duplicate_accounts()
90
                if duplicates:
91
                    msg = "The following case insensitive duplicates found: %r"
92
                    raise CommandError(msg % duplicates)
93
                else:
94
                    print "No duplicate accounts are found."
95

  
96
            if options.get('existing-accounts') and \
97
                    not options.get('merge_accounts') and \
98
                    not options.get('delete_account'):
99
                accounts = utils.existing_accounts()
100
                print "The following accounts found:"
101
                print "%s" % '\n'.join(accounts)
102

  
103
            if options.get('merge_accounts'):
104
                src_account = options.get('src_account')
105
                dest_account = options.get('dest_account')
106
                if not src_account:
107
                    raise CommandError('Please specify a source account')
108
                if not dest_account:
109
                    raise CommandError('Please specify a destination account')
110
                utils.merge_account(src_account, dest_account,
111
                                         only_stats=True)
112

  
113
                confirm = raw_input("Type 'yes' if you are sure you want"
114
                                    " to remove those entries: ")
115
                if not confirm == 'yes':
116
                    return
117
                else:
118
                    utils.merge_account(options.get('src_account'),
119
                                        options.get('dest_account'),
120
                                        only_stats=False,
121
                                        dry=self.dry)
122
                return
123

  
124
            if options.get('delete_account'):
125
                utils.delete_account(options.get('delete_account'),
126
                                     only_stats=True)
127

  
128
                confirm = raw_input("Type 'yes' if you are sure you want"
129
                                    " to remove those entries: ")
130
                if not confirm == 'yes':
131
                    return
132
                else:
133
                    utils.delete_account(options.get('delete_account'),
134
                                         only_stats=False,
135
                                         dry=self.dry)
136
                return
137
        except Exception, e:
138
            raise CommandError(e)
139
        finally:
140
            utils.backend.close()
b/snf-pithos-app/pithos/api/swiss_army/__init__.py
1
# Copyright 2012 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

  
34
from pithos.api.util import get_backend, split_container_object_string
35

  
36
import re
37
import hashlib
38
import os
39

  
40

  
41
def data_read_iterator(str, size=1024):
42
    offset = 0
43
    while True:
44
        data = str[offset:offset + size]
45
        offset = offset + size
46
        if not data:
47
            break
48
        yield data
49

  
50

  
51
class SwissArmy():
52
    def __init__(self):
53
        self.backend = get_backend()
54

  
55
    def cleanup(self):
56
        self.backend.close()
57

  
58
    def existing_accounts(self):
59
        return self.backend.node.node_accounts()
60

  
61
    def duplicate_accounts(self):
62
        accounts = self.existing_accounts()
63
        duplicates = []
64
        for i in range(len(accounts)):
65
            account = accounts[i]
66
            matcher = re.compile(account, re.IGNORECASE)
67
            duplicate = filter(matcher.match, accounts[i + 1:])
68
            if duplicate:
69
                duplicate.insert(0, account)
70
                duplicates.append(duplicate)
71
        return duplicates
72

  
73
    def list_all_containers(self, account, step=10):
74
        containers = []
75
        marker = None
76
        while 1:
77
            more = self.backend.list_containers(account, account, limit=10,
78
                                                marker=marker)
79
            if not more:
80
                break
81
            containers.extend(more)
82
            marker = more[-1]
83
        return containers
84

  
85
    def list_all_container_objects(self, account, container, virtual=False):
86
        objects = []
87
        marker = None
88
        while 1:
89
            more = self.backend.list_objects(account, account, container,
90
                                             marker=marker, virtual=virtual)
91
            if not more:
92
                break
93
            objects.extend((i[0] for i in more))
94
            marker = more[-1][0]
95
        return objects
96

  
97
    def list_all_objects(self, account, virtual=False):
98
        containers = self.list_all_containers(account)
99
        objects = []
100
        extend = objects.extend
101
        for c in containers:
102
            more = self.list_all_container_objects(account, c, virtual=virtual)
103
            extend([os.path.join(c, i) for i in more])
104
        return objects
105

  
106
    def list_past_versions(self, account, container, name):
107
        versions = self.backend.list_versions(account, account, container,
108
                                              name)
109
        # do not return the current version
110
        return list(x[0] for x in versions[:-1])
111

  
112
    def move_object(self, src_account, src_container, src_name,
113
                    dest_account, dry=True, silent=False):
114

  
115
        trans = self.backend.wrapper.conn.begin()
116
        try:
117
            self._move_object(src_account, src_container, src_name,
118
                    dest_account)
119

  
120
            if dry:
121
                if not silent:
122
                    print "Skipping database commit."
123
                trans.rollback()
124
            else:
125
                trans.commit()
126
                if not silent:
127
                    print "%s is deleted." % src_account
128
        except:
129
            trans.rollback()
130
            raise
131

  
132
    def _move_object(self, src_account, src_container, src_name,
133
                    dest_account):
134
        path = os.path.join(src_container, src_name)
135
        fullpath = os.path.join(src_account, path)
136
        dest_container = src_container
137
        dest_name = src_name
138

  
139
        meta = self.backend.get_object_meta(src_account, src_account,
140
                                            src_container, src_name, 'pithos',
141
                                            version=None)
142
        content_type = meta.get('type')
143

  
144
        # get source object history
145
        versions = self.list_past_versions(src_account, src_container,
146
                                           src_name)
147

  
148
        # get source object permissions
149
        permissions = self.backend.permissions.access_get(fullpath)
150

  
151
        # get source object public
152
        public = self.backend.get_object_public(src_account, src_account,
153
                                                src_container, src_name)
154

  
155
        if dest_container in self.backend.list_containers(dest_account,
156
                                                          dest_account):
157
            # Note: if dest_container contains an object with the same name
158
            # a new version with the contents of the source object will be
159
            # created and the one in the destination container will pass to
160
            # history
161
            self.backend.copy_object(dest_account, src_account, src_container,
162
                                     src_name, dest_account, dest_container,
163
                                     dest_name, content_type, 'pithos',
164
                                     meta={}, replace_meta=False,
165
                                     permissions=permissions)
166
        else:
167
            # create destination container and retry
168
            self.backend.put_container(dest_account, dest_account,
169
                                       dest_container)
170
            self.backend.copy_object(dest_account, src_account, src_container,
171
                                     src_name, dest_account, dest_container,
172
                                     dest_name, content_type, 'pithos',
173
                                     meta={}, replace_meta=False,
174
                                     permissions=permissions)
175

  
176
        self.backend.delete_object(src_account, src_account, src_container,
177
                                   src_name)
178

  
179
        dest_path, dest_node = self.backend._lookup_object(dest_account,
180
                                                           dest_container,
181
                                                           dest_name)
182
        assert dest_path == '/'.join([dest_account, path])
183

  
184
        # turn history versions to point to the newly created node
185
        for serial in versions:
186
            self.backend.node.version_put_property(serial, 'node', dest_node)
187

  
188
        if public:
189
            # set destination object public
190
            fullpath = '/'.join([dest_account, dest_container, dest_name])
191
            self.backend.permissions.public_set(fullpath)
192

  
193
    def _merge_account(self, src_account, dest_account):
194
            # TODO: handle exceptions
195
            # copy all source objects
196
            for path in self.list_all_objects(src_account):
197
                src_container, src_name = split_container_object_string(
198
                    '/%s' % path)
199

  
200
                # give read permissions to the dest_account
201
                permissions  = self.backend.get_object_permissions(
202
                    src_account, src_account, src_container, src_name)
203
                if permissions:
204
                    permissions = permissions[2]
205
                permissions['read'] = permissions.get('read', [])
206
                permissions['read'].append(dest_account)
207
                self.backend.update_object_permissions(src_account,
208
                                                       src_account,
209
                                                       src_container,
210
                                                       src_name,
211
                                                       permissions)
212

  
213
                self._move_object(src_account, src_container, src_name,
214
                                 dest_account)
215

  
216
            # move groups also
217
            groups = self.backend.get_account_groups(src_account, src_account)
218
            (v.replace(src_account, dest_account) for v in groups.values())
219
            self.backend.update_account_groups(dest_account, dest_account,
220
                                               groups)
221
            self._delete_account(src_account)
222

  
223
    def merge_account(self, src_account, dest_account, only_stats=True,
224
                      dry=True, silent=False):
225
        if only_stats:
226
            print "The following %s's entries will be moved to %s:" \
227
                % (src_account, dest_account)
228
            print "Objects: %r" % self.list_all_objects(src_account)
229
            print "Groups: %r" \
230
                % self.backend.get_account_groups(src_account,
231
                                                  src_account).keys()
232
            return
233

  
234
        trans = self.backend.wrapper.conn.begin()
235
        try:
236
            self._merge_account(src_account, dest_account)
237

  
238
            if dry:
239
                if not silent:
240
                    print "Skipping database commit."
241
                trans.rollback()
242
            else:
243
                trans.commit()
244
                if not silent:
245
                    msg = "%s merged into %s and deleted."
246
                    print msg % (src_account, dest_account)
247
        except:
248
            trans.rollback()
249
            raise
250

  
251
    def delete_container_contents(self, account, container):
252
        self.backend.delete_container(account, account, container,
253
                                      delimiter='/')
254

  
255
    def delete_container(self, account, container):
256
        self.backend.delete_container(account, account, container)
257

  
258
    def _delete_account(self, account):
259
        for c in self.list_all_containers(account):
260
            self.delete_container_contents(account, c)
261
            self.delete_container(account, c)
262
        self.backend.delete_account(account, account)
263

  
264
    def delete_account(self, account, only_stats=True, dry=True, silent=False):
265
        if only_stats:
266
            print "The following %s's entries will be removed:" % account
267
            print "Objects: %r" % self.list_all_objects(account)
268
            print "Groups: %r" \
269
                % self.backend.get_account_groups(account, account).keys()
270
            return
271

  
272
        trans = self.backend.wrapper.conn.begin()
273
        try:
274
            self._delete_account(account)
275

  
276
            if dry:
277
                if not silent:
278
                    print "Skipping database commit."
279
                trans.rollback()
280
            else:
281
                trans.commit()
282
                if not silent:
283
                    print "%s is deleted." % account
284
        except:
285
            trans.rollback()
286
            raise
287

  
288
    def create_account(self, account):
289
        return self.backend._lookup_account(account, create=True)
290

  
291
    def create_update_object(self, account, container, name, content_type,
292
                             data, meta={}, permissions={}, request_user=None):
293
        md5 = hashlib.md5()
294
        size = 0
295
        hashmap = []
296
        for block_data in data_read_iterator(data, self.backend.block_size):
297
            size += len(block_data)
298
            hashmap.append(self.backend.put_block(block_data))
299
            md5.update(block_data)
300

  
301
        checksum = md5.hexdigest().lower()
302

  
303
        request_user = request_user or account
304
        return self.backend.update_object_hashmap(request_user, account,
305
                                                  container, name, size,
306
                                                  content_type, hashmap,
307
                                                  checksum, 'pithos', meta,
308
                                                  True, permissions)
b/snf-pithos-app/pithos/api/swiss_army/tests.py
1
# Copyright 2011 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

  
34
import unittest
35
import uuid
36
import random
37
import string
38
import os
39

  
40
from collections import defaultdict
41

  
42
from pithos.api.swiss_army import SwissArmy
43

  
44

  
45
def get_random_data(length=500):
46
    char_set = string.ascii_uppercase + string.digits
47
    return ''.join(random.choice(char_set) for x in xrange(length))
48

  
49

  
50
class SwissArmyTests(unittest.TestCase):
51
    def setUp(self):
52
        self.utils = SwissArmy()
53
        self.accounts = ('account1', 'Account1', 'account2', 'account3')
54
        for i in self.accounts:
55
            self.utils.create_account(i)
56

  
57
    def tearDown(self):
58
        for i in self.accounts:
59
            self.utils._delete_account(i)
60
        self.utils.cleanup()
61

  
62
    def _verify_object(self, account, container, object, expected={},
63
                       strict=True):
64
        self._verify_object_metadata(account, container, object,
65
                                     expected.get('meta'))
66
        self._verify_object_history(account, container, object,
67
                                    expected.get('versions'),
68
                                    strict=strict)
69
        self._verify_object_permissions(account, container, object,
70
                                        expected.get('permissions'))
71

  
72
    def _verify_object_metadata(self, account, container, object, expected):
73
        object_meta = self.utils.backend.get_object_meta(
74
            account, account, container, object, 'pithos')
75
        for k in expected:
76
            self.assertTrue(k in object_meta)
77
            self.assertEquals(object_meta[k], expected[k])
78

  
79
    def _verify_object_history(self, account, container, object, expected,
80
                               strict=True):
81
        history = self.utils.list_past_versions(account, container, object)
82
        if strict:
83
            self.assertEquals(sorted(expected), history)
84
        else:
85
            self.assertTrue(set(expected) <= set(history))
86

  
87
    def _verify_object_permissions(self, account, container, object, expected):
88
        expected = expected or {}
89
        perms_tuple = self.utils.backend.get_object_permissions(
90
            account, account, container, object)
91

  
92
        self.assertEqual(len(perms_tuple), 3)
93

  
94
        object_perms = perms_tuple[2]
95

  
96
        for k in expected:
97
            self.assertTrue(set(expected.get(k)) <= set(object_perms.get(k)))
98

  
99
        for holder in expected.get('read', []):
100
            if holder == '*':
101
                continue
102
            try:
103
                # check first for a group permission
104
                owner, group = holder.split(':')
105
            except ValueError:
106
                holders = [holder]
107
            else:
108
                holders = self.utils.backend.permissions.group_members(owner,
109
                                                                       group)
110

  
111
            for h in holders:
112
                try:
113
                    self.utils.backend.get_object_meta(
114
                        holder, account, container, object, 'pithos')
115
                except Exception, e:
116
                    self.fail(e)
117

  
118
    def test_existing_accounts(self):
119
        accounts = self.utils.existing_accounts()
120
        self.assertEquals(sorted(accounts), accounts)
121
        self.assertTrue(set(['account1', 'account2']) <= set(accounts))
122

  
123
    def test_duplicate_accounts(self):
124
        duplicates = self.utils.duplicate_accounts()
125
        self.assertTrue(['Account1', 'account1'] in duplicates)
126

  
127
    def test_list_all_containers(self):
128
        step = 10
129
        containers = []
130
        append = containers.append
131
        for i in range(3 * step + 1):
132
            while 1:
133
                cname = unicode(uuid.uuid4())
134
                if cname not in containers:
135
                    append(cname)
136
                    break
137
            self.utils.backend.put_container('account1', 'account1', cname)
138
        self.assertEquals(sorted(containers),
139
                          self.utils.list_all_containers('account1',
140
                                                         step=step))
141

  
142
    def test_list_all_container_objects(self):
143
        containers = ('container1', 'container2')
144
        objects = defaultdict(list)
145
        for c in containers:
146
            self.utils.backend.put_container('account1', 'account1', c)
147
            step = 10
148
            append = objects[c].append
149
            content_type = 'application/octet-stream'
150
            for i in range(3 * step + 1):
151
                while 1:
152
                    oname = unicode(uuid.uuid4())
153
                    if oname not in objects:
154
                        append(oname)
155
                        break
156
                data = get_random_data(int(random.random()))
157
                self.utils.create_update_object('account1', c, oname,
158
                                                content_type, data)
159

  
160
        (self.assertEquals(sorted(objects.get(c)),
161
                           self.utils.list_all_container_objects('account1', c)
162
                           ) for c in containers)
163

  
164
    def test_list_all_objects(self):
165
        containers = ('container1', 'container2')
166
        objects = []
167
        append = objects.append
168
        for c in containers:
169
            self.utils.backend.put_container('account1', 'account1', c)
170
            step = 10
171
            content_type = 'application/octet-stream'
172
            for i in range(3 * step + 1):
173
                while 1:
174
                    oname = unicode(uuid.uuid4())
175
                    if oname not in objects:
176
                        append(os.path.join(c, oname))
177
                        break
178
                data = get_random_data(int(random.random()))
179
                self.utils.create_update_object('account1', c, oname,
180
                                                content_type, data)
181

  
182
        self.assertEquals(len(objects),
183
                          len(self.utils.list_all_objects('account1')))
184
        self.assertEquals(sorted(objects),
185
                          self.utils.list_all_objects('account1'))
186

  
187
    def test_list_past_versions(self):
188
        self.utils.backend.put_container('account1', 'account1', 'container1')
189
        versions = []
190
        append = versions.append
191
        for i in range(5):
192
            data = get_random_data(int(random.random()))
193
            append(self.utils.create_update_object('account1', 'container1',
194
                                                   'object1',
195
                                                   'application/octet-stream',
196
                                                   data))
197
        self.assertEquals(sorted(versions[:-1]),
198
                          self.utils.list_past_versions('account1',
199
                                                        'container1',
200
                                                        'object1'))
201

  
202
    def test_move(self):
203
        # create containers
204
        self.utils.backend.put_container('account1', 'account1', 'container1')
205
        self.utils.backend.put_container('Account1', 'Account1', 'container1')
206

  
207
        # add group
208
        self.utils.backend.update_account_groups('Account1', 'Account1',
209
                                                 {'test': ['account3']})
210

  
211
        # upload object and update it several times
212
        versions = []
213
        append = versions.append
214
        meta = {'foo': 'bar'}
215
        permissions = {'read': ['account1', 'account2', 'Account1:test'],
216
                       'write': ['account2', 'Account1:test']}
217
        for i in range(5):
218
            data = get_random_data(int(random.random()))
219
            append(self.utils.create_update_object('Account1', 'container1',
220
                                                   'object1',
221
                                                   'application/octet-stream',
222
                                                   data, meta, permissions))
223

  
224
        self.utils.move_object('Account1', 'container1', 'object1', 'account1',
225
                               dry=False, silent=True)
226

  
227
        expected = {'meta': meta,
228
                    'versions': versions[:-1],
229
                    'permissions': permissions}
230
        self._verify_object('account1', 'container1', 'object1', expected)
231

  
232
    def test_merge(self):
233
        # create container
234
        self.utils.backend.put_container('Account1', 'Account1', 'container0')
235
        self.utils.backend.put_container('Account1', 'Account1', 'container1')
236

  
237
        # add group
238
        self.utils.backend.update_account_groups('Account1', 'Account1',
239
                                                 {'test': ['account3']})
240

  
241
        # upload objects and update them several times
242
        versions = defaultdict(list)
243
        meta = {'foo': 'bar'}
244
        permissions = {'read': ['account2', 'Account1:test'],
245
                       'write': ['account2', 'Account1:test']}
246

  
247
        for i in range(2):
248
            container = 'container%s' % i
249
            versions[container] = {}
250
            for j in range(3):
251
                object = 'object%s' % j
252
                versions[container][object] = []
253
                append = versions[container][object].append
254
                for k in range(5):
255
                    data = get_random_data(int(random.random()))
256
                    append(self.utils.create_update_object(
257
                        'Account1', container, object,
258
                        'application/octet-stream', data, meta, permissions))
259

  
260
        self.utils.merge_account('Account1', 'account1', only_stats=False,
261
                                 dry=False, silent=True)
262

  
263
        self.assertTrue('Account1' not in self.utils.existing_accounts())
264
        self.assertTrue('account1' in self.utils.existing_accounts())
265

  
266
        # assert container has been created
267
        try:
268
            self.utils.backend.get_container_meta('account1', 'account1',
269
                                                  'container1', 'pithos')
270
        except NameError, e:
271
            self.fail(e)
272

  
273
        expected = {'meta': meta,
274
                    'permissions': permissions}
275
        for c, o_dict in versions.iteritems():
276
            for o, versions in o_dict.iteritems():
277
                expected['versions'] = versions[:-1]
278
                self._verify_object('account1', c, o, expected)
279

  
280
    def test_merge_existing_dest_container(self):
281
        # create container
282
        self.utils.backend.put_container('Account1', 'Account1', 'container1')
283
        self.utils.backend.put_container('account1', 'account1', 'container1')
284

  
285
        # add group
286
        self.utils.backend.update_account_groups('Account1', 'Account1',
287
                                                 {'test': ['account3']})
288

  
289
        # upload objects and update them several times
290
        versions = defaultdict(list)
291
        meta = {'foo': 'bar'}
292
        permissions = {'read': ['account2', 'Account1:test'],
293
                       'write': ['account2', 'Account1:test']}
294

  
295
        versions = []
296
        append = versions.append
297
        for k in range(5):
298
            data = get_random_data(int(random.random()))
299
            append(self.utils.create_update_object(
300
                'Account1', 'container1', 'object1',
301
                'application/octet-stream', data, meta, permissions))
302

  
303
        self.utils.merge_account('Account1', 'account1', only_stats=False,
304
                                 dry=False, silent=True)
305

  
306
        self.utils.merge_account('Account1', 'account1', only_stats=False,
307
                                 dry=False, silent=True)
308

  
309
        try:
310
            self.utils.backend.get_container_meta('Account1', 'Account1',
311
                                                  'container1', 'pithos')
312
        except NameError:
313
            pass
314
        else:
315
            self.fail('Container not deleted')
316

  
317
        try:
318
            self.utils.backend.get_container_meta('account1', 'account1',
319
                                                  'container1', 'pithos')
320
        except NameError, e:
321
            self.fail(e)
322

  
323
        expected = {'meta': meta,
324
                    'versions': versions[:-1],
325
                    'permissions': permissions}
326
        self._verify_object('account1', 'container1', 'object1', expected)
327

  
328
    def test_merge_existing_dest_object(self):
329
        # create container
330
        self.utils.backend.put_container('Account1', 'Account1', 'container1')
331
        self.utils.backend.put_container('account1', 'account1', 'container1')
332

  
333
        # add group
334
        self.utils.backend.update_account_groups('Account1', 'Account1',
335
                                                 {'test': ['account3']})
336

  
337
        # upload objects and update them several times
338
        versions = defaultdict(list)
339
        meta = {'foo': 'bar'}
340
        permissions = {'read': ['account2', 'Account1:test'],
341
                       'write': ['account2', 'Account1:test']}
342

  
343
        container = 'container1'
344
        object = 'object1'
345
        versions = []
346
        append = versions.append
347
        for k in range(5):
348
            data = get_random_data(int(random.random()))
349
            append(self.utils.create_update_object(
350
                   'Account1', container, object,
351
                   'application/octet-stream', data, meta, permissions))
352
            data = get_random_data(int(random.random()))
353
            self.utils.create_update_object(
354
                   'account1', container, object,
355
                   'application/octet-stream', data, meta, permissions)
356

  
357
        self.utils.merge_account('Account1', 'account1', only_stats=False,
358
                                 dry=False, silent=True)
359

  
360
        try:
361
            self.utils.backend.get_container_meta('Account1', 'Account1',
362
                                                  'container1', 'pithos')
363
        except NameError:
364
            pass
365
        else:
366
            self.fail('Container not deleted')
367

  
368
        try:
369
            self.utils.backend.get_container_meta('account1', 'account1',
370
                                                  'container1', 'pithos')
371
        except NameError, e:
372
            self.fail(e)
373

  
374
        expected = {'meta': meta,
375
                    'permissions': permissions,
376
                    'versions': versions[:-1]}
377
        self._verify_object('account1', container, object, expected,
378
                            strict=False)
379

  
380

  
381
if __name__ == '__main__':
382
    unittest.main()
b/snf-pithos-backend/pithos/backends/lib/sqlalchemy/node.py
438 438
        self.conn.execute(s).close()
439 439
        return True
440 440

  
441
    def node_accounts(self):
442
        s = select([self.nodes.c.path])
443
        s = s.where(and_(self.nodes.c.node != 0, self.nodes.c.parent == 0))
444
        account_nodes = self.conn.execute(s).fetchall()
445
        return sorted(i[0] for i in account_nodes)
446

  
441 447
    def policy_get(self, node):
442 448
        s = select([self.policy.c.key, self.policy.c.value],
443 449
                   self.policy.c.node == node)

Also available in: Unified diff