Revision ec0c9d21

b/snf-cyclades-app/synnefo/api/management/commands/cyclades-astakos-migrate-013.py
1
# Copyright 2012 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

  
34
import itertools
35
import warnings
36
import functools
37

  
38
from optparse import make_option
39

  
40
from django.core.management.base import NoArgsCommand, CommandError, BaseCommand
41
from django.db import transaction
42
from django.conf import settings
43

  
44
from synnefo.quotas import get_quota_holder
45
from synnefo.api.util import get_existing_users
46
from synnefo.lib.utils import case_unique
47
from synnefo.db.models import Network, VirtualMachine
48
from synnefo.ui.userdata.models import PublicKeyPair
49

  
50
from synnefo.lib import astakos
51

  
52
def warn(*msgs):
53
    print "WARNING: %s" % ' '.join(msgs)
54

  
55
get_displayname = functools.partial(astakos.get_displayname,
56
                                 settings.CYCLADES_ASTAKOS_SERVICE_TOKEN,
57
                                 url=settings.ASTAKOS_URL.replace('im/authenticate',
58
                                                                 'service/api/user_catalogs'))
59
get_user_uuid = functools.partial(astakos.get_user_uuid,
60
                                 settings.CYCLADES_ASTAKOS_SERVICE_TOKEN,
61
                                 url=settings.ASTAKOS_URL.replace('im/authenticate',
62
                                                                 'service/api/user_catalogs'))
63

  
64
def _validate_db_state(usernames):
65

  
66
    usernames = filter(bool, usernames)
67
    invalid_case_users = case_unique(usernames)
68
    if invalid_case_users:
69
        invalid_case_users.append(invalid_case_users[0].lower())
70
        raise CommandError("Duplicate case insensitive user identifiers exist %r" % invalid_case_users)
71

  
72
    uuidusers = filter(lambda uid:'@' in uid or uid == None, usernames)
73
    if len(uuidusers) != len(usernames):
74
        warn('It seems that mixed uuid/email user identifiers exist in database.')
75
        return False
76

  
77
    return True
78

  
79

  
80
@transaction.commit_manually
81
def delete_user(username, only_stats=True, dry=True):
82
    vms = VirtualMachine.objects.filter(userid__exact=username)
83
    networks = Network.objects.filter(userid__exact=username)
84
    keys = PublicKeyPair.objects.filter(user__exact=username)
85

  
86
    if not len(list(itertools.ifilter(bool, map(lambda q: q.count(), [vms,
87
                                                                      networks,
88
                                                                      keys])))):
89
        print "No entries exist for '%s'" % username
90
        return -1
91

  
92
    if only_stats:
93
        print "The following entries will be deleted if you decide to remove this user"
94
        print "%d Virtual Machines" % vms.exclude(operstate='DESTROYED').count()
95
        print "%d Destroyed Virtual Machines" % vms.filter(operstate='DESTROYED').count()
96
        print "%d Networks" % networks.count()
97
        print "%d PublicKeyPairs" % keys.count()
98
        return
99

  
100
    for o in itertools.chain(vms, networks):
101
        o.delete()
102

  
103
    for key in keys:
104
        key.delete()
105

  
106
    if dry:
107
        print "Skipping database commit."
108
        transaction.rollback()
109
    else:
110
        transaction.commit()
111
        print "User entries removed."
112

  
113

  
114
@transaction.commit_on_success
115
def merge_user(username):
116
    vms = VirtualMachine.objects.filter(userid__iexact=username)
117
    networks = Network.objects.filter(userid__iexact=username)
118
    keys = PublicKeyPair.objects.filter(user__iexact=username)
119

  
120
    for o in itertools.chain(vms, networks):
121
        o.userid = username.lower()
122
        o.save()
123

  
124
    for key in keys:
125
        key.user = username.lower()
126
        key.save()
127

  
128

  
129
def migrate_user(username, uuid):
130
    """
131
    Warn: no transaction handling. Consider wrapping within another function.
132
    """
133
    vms = VirtualMachine.objects.filter(userid__exact=username)
134
    networks = Network.objects.filter(userid__exact=username)
135
    keys = PublicKeyPair.objects.filter(user__exact=username)
136

  
137
    for o in itertools.chain(vms, networks):
138
        o.userid = uuid or o.userid
139
        o.save()
140

  
141
    for key in keys:
142
        key.user = uuid
143
        key.save()
144

  
145

  
146
@transaction.commit_manually
147
def migrate_users(usernames, dry=True):
148
    usernames = filter(bool, usernames)
149
    count = 0
150
    for u in usernames:
151
        if not '@' in u:
152
            warn('Skipping %s. It doesn\'t seem to be an email' % u)
153
            continue
154

  
155
        try:
156
            uuid = get_user_uuid(u)
157
            print "%s -> %s" % (u, uuid)
158
            if not uuid:
159
                raise Exception("No uuid for %s" % u)
160
            migrate_user(u, uuid)
161
            count += 1
162
        except Exception, e:
163
            print "ERROR: User id migration failed (%s)" % e
164

  
165
    if dry:
166
        print "Skipping database commit."
167
        transaction.rollback()
168
    else:
169
        transaction.commit()
170
        print "Migrated %d users" % count
171

  
172

  
173
class Command(NoArgsCommand):
174
    help = "Quotas migration helper"
175

  
176
    option_list = BaseCommand.option_list + (
177
        make_option('--strict',
178
                    dest='strict',
179
                    action="store_false",
180
                    default=True,
181
                    help="Exit on warnings."),
182
        make_option('--validate-db',
183
                    dest='validate',
184
                    action="store_true",
185
                    default=True,
186
                    help=("Check if cyclades database contents are valid for "
187
                          "migration.")),
188
        make_option('--migrate-users',
189
                    dest='migrate_users',
190
                    action="store_true",
191
                    default=False,
192
                    help=("Convert emails to uuids for all users stored in "
193
                          "database.")),
194
        make_option('--merge-user',
195
                    dest='merge_user',
196
                    default=False,
197
                    help="Merge case insensitive duplicates of a user."),
198
        make_option('--delete-user',
199
                    dest='delete_user',
200
                    action='store',
201
                    default=False,
202
                    help="Delete user entries."),
203
        make_option('--user-entries',
204
                    dest='user_entries',
205
                    action='store',
206
                    default=False,
207
                    help="Display user summary."),
208
        make_option('--dry',
209
                    dest='dry',
210
                    action="store_true",
211
                    default=False,
212
                    help="Do not commit database changes. Do not communicate "
213
                         "with quotaholder"),
214
        make_option('--user',
215
                    dest='user',
216
                    action="store",
217
                    default=False,)
218
    )
219

  
220
    def resolve_conflicts(self, options):
221
        conflicting = map(options.get, ['migrate_users',
222
                                        'merge_user'])
223
        if len(filter(bool, conflicting)) > 1:
224
            raise CommandError('You can use only one of --validate,'
225
                               '--migrate-users')
226

  
227
    def handle(self, *args, **options):
228
        self.resolve_conflicts(options)
229
        self.strict = options.get('strict')
230
        self.dry = options.get('dry')
231

  
232
        if options.get('validate') and not options.get('merge_user') and not \
233
                options.get('delete_user') and not options.get('user_entries'):
234
            usernames = get_existing_users()
235
            _validate_db_state(usernames)
236

  
237
        if options.get('migrate_users'):
238
            migrate_users(usernames, dry=self.dry)
239

  
240
        if options.get('merge_user'):
241
            merge_user(options.get('merge_user'))
242
            print "Merge finished."
243

  
244
        if options.get('delete_user'):
245
            entries = delete_user(options.get('delete_user'), only_stats=True)
246
            if entries == -1:
247
                return
248

  
249
            confirm = raw_input("Type 'yes of course' if you are sure you want"
250
                                " to remove those entries: ")
251
            if not confirm == 'yes of course':
252
                return
253
            else:
254
                delete_user(options.get('delete_user'), only_stats=False,
255
                            dry=self.dry)
256

  
257
        if options.get('user_entries'):
258
            delete_user(options.get('user_entries'))
/dev/null
1
# Copyright 2012 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

  
34
import itertools
35
import warnings
36
import functools
37

  
38
from optparse import make_option
39

  
40
from django.core.management.base import NoArgsCommand, CommandError, BaseCommand
41
from django.db import transaction
42
from django.conf import settings
43

  
44
from synnefo.quotas import get_quota_holder
45
from synnefo.api.util import get_existing_users
46
from synnefo.lib.utils import case_unique
47
from synnefo.db.models import Network, VirtualMachine
48
from synnefo.ui.userdata.models import PublicKeyPair
49

  
50
from synnefo.lib import astakos
51

  
52
def warn(*msgs):
53
    print "WARNING: %s" % ' '.join(msgs)
54

  
55
get_displayname = functools.partial(astakos.get_displayname,
56
                                 settings.CYCLADES_ASTAKOS_SERVICE_TOKEN,
57
                                 url=settings.ASTAKOS_URL.replace('im/authenticate',
58
                                                                 'service/api/user_catalogs'))
59
get_user_uuid = functools.partial(astakos.get_user_uuid,
60
                                 settings.CYCLADES_ASTAKOS_SERVICE_TOKEN,
61
                                 url=settings.ASTAKOS_URL.replace('im/authenticate',
62
                                                                 'service/api/user_catalogs'))
63

  
64
def _validate_db_state(usernames):
65

  
66
    usernames = filter(bool, usernames)
67
    invalid_case_users = case_unique(usernames)
68
    if invalid_case_users:
69
        invalid_case_users.append(invalid_case_users[0].lower())
70
        raise CommandError("Duplicate case insensitive user identifiers exist %r" % invalid_case_users)
71

  
72
    uuidusers = filter(lambda uid:'@' in uid or uid == None, usernames)
73
    if len(uuidusers) != len(usernames):
74
        warn('It seems that mixed uuid/email user identifiers exist in database.')
75
        return False
76

  
77
    return True
78

  
79

  
80
@transaction.commit_manually
81
def delete_user(username, only_stats=True, dry=True):
82
    vms = VirtualMachine.objects.filter(userid__exact=username)
83
    networks = Network.objects.filter(userid__exact=username)
84
    keys = PublicKeyPair.objects.filter(user__exact=username)
85

  
86
    if not len(list(itertools.ifilter(bool, map(lambda q: q.count(), [vms,
87
                                                                      networks,
88
                                                                      keys])))):
89
        print "No entries exist for '%s'" % username
90
        return -1
91

  
92
    if only_stats:
93
        print "The following entries will be deleted if you decide to remove this user"
94
        print "%d Virtual Machines" % vms.exclude(operstate='DESTROYED').count()
95
        print "%d Destroyed Virtual Machines" % vms.filter(operstate='DESTROYED').count()
96
        print "%d Networks" % networks.count()
97
        print "%d PublicKeyPairs" % keys.count()
98
        return
99

  
100
    for o in itertools.chain(vms, networks):
101
        o.delete()
102

  
103
    for key in keys:
104
        key.delete()
105

  
106
    if dry:
107
        print "Skipping database commit."
108
        transaction.rollback()
109
    else:
110
        transaction.commit()
111
        print "User entries removed."
112

  
113

  
114
@transaction.commit_on_success
115
def merge_user(username):
116
    vms = VirtualMachine.objects.filter(userid__iexact=username)
117
    networks = Network.objects.filter(userid__iexact=username)
118
    keys = PublicKeyPair.objects.filter(user__iexact=username)
119

  
120
    for o in itertools.chain(vms, networks):
121
        o.userid = username.lower()
122
        o.save()
123

  
124
    for key in keys:
125
        key.user = username.lower()
126
        key.save()
127

  
128
def migrate_user(username, uuid):
129
    """
130
    Warn: no transaction handling. Consider wrapping within another function.
131
    """
132
    vms = VirtualMachine.objects.filter(userid__exact=username)
133
    networks = Network.objects.filter(userid__exact=username)
134
    keys = PublicKeyPair.objects.filter(user__exact=username)
135

  
136
    for o in itertools.chain(vms, networks):
137
        o.userid = uuid
138
        o.save()
139

  
140
    for key in keys:
141
        key.user = uuid
142
        key.save()
143

  
144

  
145
@transaction.commit_manually
146
def migrate_users(usernames, dry=True):
147
    usernames = filter(bool, usernames)
148
    count = 0
149
    for u in usernames:
150
        if not '@' in u:
151
            warn('Skipping %s. It doesn\'t seem to be an email' % u)
152
            continue
153

  
154
        try:
155
            uuid = get_user_uuid(u)
156
            print "%s -> %s" % (u, uuid)
157
            migrate_user(u, uuid)
158
            count += 1
159
        except Exception, e:
160
            print "ERROR: User id migration failed (%s)" % e
161

  
162

  
163
    if dry:
164
        print "Skipping database commit."
165
        transaction.rollback()
166
    else:
167
        transaction.commit()
168
        print "Migrated %d users" % count
169

  
170

  
171
def migrate_quotas(usernames):
172
    pass
173

  
174

  
175
class Command(NoArgsCommand):
176
    help = "Quotas migration helper"
177

  
178
    option_list = BaseCommand.option_list + (
179
        make_option('--strict',
180
                    dest='strict',
181
                    action="store_false",
182
                    default=True,
183
                    help="Exit on warnings."),
184
        make_option('--validate-db',
185
                    dest='validate',
186
                    action="store_true",
187
                    default=True,
188
                    help="Check if cyclades database contents are valid for migration."),
189
        make_option('--migrate-users',
190
                    dest='migrate_users',
191
                    action="store_true",
192
                    default=False,
193
                    help="Convert emails to uuids for all users stored in database."),
194
        make_option('--migrate-quotas',
195
                    dest='migrate_quota',
196
                    action="store_true",
197
                    default=False,
198
                    help="Store existing vms/networks usage/limits to quotaholder."),
199
        make_option('--merge-user',
200
                    dest='merge_user',
201
                    default=False,
202
                    help="Merge case insensitive duplicates of a user."),
203
        make_option('--delete-user',
204
                    dest='delete_user',
205
                    action='store',
206
                    default=False,
207
                    help="Delete user entries."),
208
        make_option('--user-entries',
209
                    dest='user_entries',
210
                    action='store',
211
                    default=False,
212
                    help="Display user summary."),
213
        make_option('--dry',
214
                    dest='dry',
215
                    action="store_true",
216
                    default=False,
217
                    help="Do not commit database changes. Do not communicate "
218
                         "with quotaholder"),
219
        make_option('--user',
220
                    dest='user',
221
                    action="store",
222
                    default=False,)
223
    )
224

  
225
    def resolve_conflicts(self, options):
226
        conflicting = map(options.get, ['migrate_users',
227
                                        'merge_user'])
228
        if len(filter(bool, conflicting)) > 1:
229
            raise CommandError('You can use only one of --validate,'
230
                               '--migrate-userids, migrate-quotas')
231

  
232
    def handle(self, *args, **options):
233
        self.resolve_conflicts(options)
234
        self.strict = options.get('strict')
235
        self.dry = options.get('dry')
236

  
237
        if options.get('validate') and not options.get('merge_user') and not \
238
                options.get('delete_user') and not options.get('user_entries'):
239
            usernames = get_existing_users()
240
            _validate_db_state(usernames)
241

  
242
        if options.get('migrate_users'):
243
            migrate_users(usernames, dry=self.dry)
244

  
245
        if options.get('merge_user'):
246
            merge_user(options.get('merge_user'))
247
            print "Merge finished."
248

  
249
        if options.get('delete_user'):
250
            entries = delete_user(options.get('delete_user'), only_stats=True)
251
            if entries == -1:
252
                return
253

  
254
            confirm = raw_input("Type 'yes of course' if you are sure you want"
255
                                " to remove those entries: ")
256
            if not confirm == 'yes of course':
257
                return
258
            else:
259
                delete_user(options.get('delete_user'), only_stats=False,
260
                            dry=self.dry)
261

  
262
        if options.get('user_entries'):
263
            delete_user(options.get('user_entries'))
264

  
265

  

Also available in: Unified diff