Statistics
| Branch: | Tag: | Revision:

root / snf-cyclades-app / synnefo / logic / management / commands / reconcile-servers.py @ 8e440b3a

History | View | Annotate | Download (15.5 kB)

1
# Copyright 2011-2012 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or without
4
# modification, are permitted provided that the following conditions
5
# are met:
6
#
7
#   1. Redistributions of source code must retain the above copyright
8
#      notice, this list of conditions and the following disclaimer.
9
#
10
#  2. Redistributions in binary form must reproduce the above copyright
11
#     notice, this list of conditions and the following disclaimer in the
12
#     documentation and/or other materials provided with the distribution.
13
#
14
# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
15
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17
# ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
18
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24
# SUCH DAMAGE.
25
#
26
# The views and conclusions contained in the software and documentation are
27
# those of the authors and should not be interpreted as representing official
28
# policies, either expressed or implied, of GRNET S.A.
29
#
30
"""Reconciliation management command
31

32
Management command to reconcile the contents of the Synnefo DB with
33
the state of the Ganeti backend. See docstring on top of
34
logic/reconciliation.py for a description of reconciliation rules.
35

36
"""
37
import sys
38
import datetime
39

    
40
from optparse import make_option
41

    
42
from django.core.management.base import BaseCommand, CommandError
43

    
44
from synnefo.db.models import (Backend, VirtualMachine, Network,
45
                               pooled_rapi_client)
46
from synnefo.logic import reconciliation, utils
47
from synnefo.logic import backend as backend_mod
48
from synnefo.util.mac2eui64 import mac2eui64
49
from synnefo.management.common import get_backend
50

    
51

    
52
class Command(BaseCommand):
53
    can_import_settings = True
54

    
55
    help = 'Reconcile contents of Synnefo DB with state of Ganeti backend'
56
    output_transaction = True  # The management command runs inside
57
                               # an SQL transaction
58
    option_list = BaseCommand.option_list + (
59
        make_option('--detect-stale', action='store_true', dest='detect_stale',
60
                    default=False, help='Detect stale VM entries in DB'),
61
        make_option('--detect-orphans', action='store_true',
62
                    dest='detect_orphans',
63
                    default=False, help='Detect orphan instances in Ganeti'),
64
        make_option('--detect-unsynced', action='store_true',
65
                    dest='detect_unsynced',
66
                    default=False, help='Detect unsynced operstate between ' +
67
                                        'DB and Ganeti'),
68
        make_option('--detect-build-errors', action='store_true',
69
                    dest='detect_build_errors', default=False,
70
                    help='Detect instances with build error'),
71
        make_option('--detect-unsynced-nics', action='store_true',
72
                    dest='detect_unsynced_nics', default=False,
73
                    help='Detect unsynced nics between DB and Ganeti'),
74
        make_option('--detect-unsynced-flavors', action='store_true',
75
                    dest='detect_unsynced_flavors', default=False,
76
                    help='Detect unsynced flavors between DB and Ganeti'),
77
        make_option('--detect-all', action='store_true',
78
                    dest='detect_all',
79
                    default=False, help='Enable all --detect-* arguments'),
80
        make_option('--fix-stale', action='store_true', dest='fix_stale',
81
                    default=False, help='Fix (remove) stale DB entries in DB'),
82
        make_option('--fix-orphans', action='store_true', dest='fix_orphans',
83
                    default=False, help='Fix (remove) orphan Ganeti VMs'),
84
        make_option('--fix-unsynced', action='store_true', dest='fix_unsynced',
85
                    default=False, help='Fix server operstate in DB, set ' +
86
                                        'from Ganeti'),
87
        make_option('--fix-build-errors', action='store_true',
88
                    dest='fix_build_errors', default=False,
89
                    help='Fix (remove) instances with build errors'),
90
        make_option('--fix-unsynced-nics', action='store_true',
91
                    dest='fix_unsynced_nics', default=False,
92
                    help='Fix unsynced nics between DB and Ganeti'),
93
        make_option('--fix-unsynced-flavors', action='store_true',
94
                    dest='fix_unsynced_flavors', default=False,
95
                    help='Fix unsynced flavors between DB and Ganeti'),
96
        make_option('--fix-all', action='store_true', dest='fix_all',
97
                    default=False, help='Enable all --fix-* arguments'),
98
        make_option('--backend-id', default=None, dest='backend-id',
99
                    help='Reconcilie VMs only for this backend'),
100
    )
101

    
102
    def _process_args(self, options):
103
        keys_detect = [k for k in options.keys() if k.startswith('detect_')]
104
        keys_fix = [k for k in options.keys() if k.startswith('fix_')]
105

    
106
        if not reduce(lambda x, y: x or y,
107
                      map(lambda x: options[x], keys_detect)):
108
            options['detect_all'] = True
109

    
110
        if options['detect_all']:
111
            for kd in keys_detect:
112
                options[kd] = True
113
        if options['fix_all']:
114
            for kf in keys_fix:
115
                options[kf] = True
116

    
117
        for kf in keys_fix:
118
            kd = kf.replace('fix_', 'detect_', 1)
119
            if (options[kf] and not options[kd]):
120
                raise CommandError("Cannot use --%s without corresponding "
121
                                   "--%s argument" % (kf, kd))
122

    
123
    def handle(self, **options):
124
        verbosity = int(options['verbosity'])
125
        self._process_args(options)
126
        backend_id = options['backend-id']
127
        if backend_id:
128
            backends = [get_backend(backend_id)]
129
        else:
130
            backends = Backend.objects.filter(offline=False)
131

    
132
        with_nics = options["detect_unsynced_nics"]
133

    
134
        DBVMs = reconciliation.get_servers_from_db(backends, with_nics)
135
        GanetiVMs = reconciliation.get_instances_from_ganeti(backends)
136

    
137
        #
138
        # Detect problems
139
        #
140
        if options['detect_stale']:
141
            stale = reconciliation.stale_servers_in_db(DBVMs, GanetiVMs)
142
            if len(stale) > 0:
143
                print >> sys.stderr, "Found the following stale server IDs: "
144
                print "    " + "\n    ".join(
145
                    [str(x) for x in stale])
146
            elif verbosity == 2:
147
                print >> sys.stderr, "Found no stale server IDs in DB."
148

    
149
        if options['detect_orphans']:
150
            orphans = reconciliation.orphan_instances_in_ganeti(DBVMs,
151
                                                                GanetiVMs)
152
            if len(orphans) > 0:
153
                print >> sys.stderr, "Found orphan Ganeti instances with IDs: "
154
                print "    " + "\n    ".join(
155
                    [str(x) for x in orphans])
156
            elif verbosity == 2:
157
                print >> sys.stderr, "Found no orphan Ganeti instances."
158

    
159
        if options['detect_unsynced']:
160
            unsynced = reconciliation.unsynced_operstate(DBVMs, GanetiVMs)
161
            if len(unsynced) > 0:
162
                print >> sys.stderr, "The operstate of the following server" \
163
                                     " IDs is out-of-sync:"
164
                print "    " + "\n    ".join(
165
                    ["%d is %s in DB, %s in Ganeti" %
166
                     (x[0], x[1], ('UP' if x[2] else 'DOWN'))
167
                     for x in unsynced])
168
            elif verbosity == 2:
169
                print >> sys.stderr, "The operstate of all servers is in sync."
170

    
171
        if options['detect_build_errors']:
172
            build_errors = reconciliation.\
173
                instances_with_build_errors(DBVMs, GanetiVMs)
174
            if len(build_errors) > 0:
175
                msg = "The os for the following server IDs was not build"\
176
                      " successfully:"
177
                print >> sys.stderr, msg
178
                print "    " + "\n    ".join(
179
                    ["%d" % x for x in build_errors])
180
            elif verbosity == 2:
181
                print >> sys.stderr, "Found no instances with build errors."
182

    
183
        if options['detect_unsynced_nics']:
184
            def pretty_print_nics(nics):
185
                if not nics:
186
                    print ''.ljust(18) + 'None'
187
                for index, info in nics.items():
188
                    print ''.ljust(18) + 'nic/' + str(index) +\
189
                          ': MAC: %s, IP: %s, Network: %s' % \
190
                          (info['mac'], info['ipv4'], info['network'])
191

    
192
            unsynced_nics = reconciliation.unsynced_nics(DBVMs, GanetiVMs)
193
            if len(unsynced_nics) > 0:
194
                msg = "The NICs of the servers with the following IDs are"\
195
                      " unsynced:"
196
                print >> sys.stderr, msg
197
                for id, nics in unsynced_nics.items():
198
                    print ''.ljust(2) + '%6d:' % id
199
                    print ''.ljust(8) + '%8s:' % 'DB'
200
                    pretty_print_nics(nics[0])
201
                    print ''.ljust(8) + '%8s:' % 'Ganeti'
202
                    pretty_print_nics(nics[1])
203
            elif verbosity == 2:
204
                print >> sys.stderr, "All instance nics are synced."
205

    
206
        if options["detect_unsynced_flavors"]:
207
            unsynced_flavors = reconciliation.unsynced_flavors(DBVMs,
208
                                                               GanetiVMs)
209
            if len(unsynced_flavors) > 0:
210
                print >> sys.stderr, "The flavor of the following server" \
211
                                     " IDs is out-of-sync:"
212
                print "    " + "\n    ".join(
213
                    ["%d is %s in DB, %s in Ganeti" %
214
                     (x[0], x[1], x[2])
215
                     for x in unsynced_flavors])
216
            elif verbosity == 2:
217
                print >> sys.stderr, "All instance flavors are synced."
218

    
219
        #
220
        # Then fix them
221
        #
222
        if options['fix_stale'] and len(stale) > 0:
223
            print >> sys.stderr, \
224
                "Simulating successful Ganeti removal for %d " \
225
                "servers in the DB:" % len(stale)
226
            for vm in VirtualMachine.objects.filter(pk__in=stale):
227
                event_time = datetime.datetime.now()
228
                backend_mod.process_op_status(
229
                    vm=vm,
230
                    etime=event_time,
231
                    jobid=-0,
232
                    opcode='OP_INSTANCE_REMOVE', status='success',
233
                    logmsg='Reconciliation: simulated Ganeti event')
234
            print >> sys.stderr, "    ...done"
235

    
236
        if options['fix_orphans'] and len(orphans) > 0:
237
            print >> sys.stderr, \
238
                "Issuing OP_INSTANCE_REMOVE for %d Ganeti instances:" % \
239
                len(orphans)
240
            for id in orphans:
241
                try:
242
                    vm = VirtualMachine.objects.get(pk=id)
243
                    with pooled_rapi_client(vm) as client:
244
                        client.DeleteInstance(utils.id_to_instance_name(id))
245
                except VirtualMachine.DoesNotExist:
246
                    print >> sys.stderr, "No entry for VM %d in DB !!" % id
247
            print >> sys.stderr, "    ...done"
248

    
249
        if options['fix_unsynced'] and len(unsynced) > 0:
250
            print >> sys.stderr, "Setting the state of %d out-of-sync VMs:" % \
251
                len(unsynced)
252
            for id, db_state, ganeti_up in unsynced:
253
                vm = VirtualMachine.objects.get(pk=id)
254
                opcode = "OP_INSTANCE_REBOOT" if ganeti_up \
255
                         else "OP_INSTANCE_SHUTDOWN"
256
                event_time = datetime.datetime.now()
257
                backend_mod.process_op_status(
258
                    vm=vm, etime=event_time, jobid=-0,
259
                    opcode=opcode, status='success',
260
                    logmsg='Reconciliation: simulated Ganeti event')
261
            print >> sys.stderr, "    ...done"
262

    
263
        if options['fix_build_errors'] and len(build_errors) > 0:
264
            print >> sys.stderr, "Setting the state of %d build-errors VMs:" %\
265
                                 len(build_errors)
266
            for id in build_errors:
267
                vm = VirtualMachine.objects.get(pk=id)
268
                event_time = datetime.datetime.now()
269
                backend_mod.process_op_status(
270
                    vm=vm, etime=event_time, jobid=-0,
271
                    opcode="OP_INSTANCE_CREATE", status='error',
272
                    logmsg='Reconciliation: simulated Ganeti event')
273
            print >> sys.stderr, "    ...done"
274

    
275
        if options['fix_unsynced_nics'] and len(unsynced_nics) > 0:
276
            print >> sys.stderr, "Setting the nics of %d out-of-sync VMs:" % \
277
                                 len(unsynced_nics)
278
            for id, nics in unsynced_nics.items():
279
                vm = VirtualMachine.objects.get(pk=id)
280
                nics = nics[1]  # Ganeti nics
281
                if nics == {}:  # No nics
282
                    vm.nics.all.delete()
283
                    continue
284
                for index, nic in nics.items():
285
                    net_id = utils.id_from_network_name(nic['network'])
286
                    subnet6 = Network.objects.get(id=net_id).subnet6
287
                    # Produce ipv6
288
                    ipv6 = subnet6 and mac2eui64(nic['mac'], subnet6) or None
289
                    nic['ipv6'] = ipv6
290
                    # Rename ipv4 to ip
291
                    nic['ip'] = nic['ipv4']
292
                # Dict to sorted list
293
                final_nics = []
294
                nics_keys = nics.keys()
295
                nics_keys.sort()
296
                for i in nics_keys:
297
                    if nics[i]['network']:
298
                        final_nics.append(nics[i])
299
                    else:
300
                        print 'Network of nic %d of vm %s is None. ' \
301
                              'Can not reconcile' % (i, vm.backend_vm_id)
302
                event_time = datetime.datetime.now()
303
                backend_mod.process_net_status(vm=vm, etime=event_time,
304
                                               nics=final_nics)
305
            print >> sys.stderr, "    ...done"
306
        if options["fix_unsynced_flavors"] and len(unsynced_flavors) > 0:
307
            print >> sys.stderr, "Setting the flavor of %d unsynced VMs:" % \
308
                len(unsynced_flavors)
309
            for id, db_flavor, gnt_flavor in unsynced_flavors:
310
                vm = VirtualMachine.objects.get(pk=id)
311
                old_state = vm.operstate
312
                opcode = "OP_INSTANCE_SET_PARAMS"
313
                beparams = {"vcpus": gnt_flavor.cpu,
314
                            "minmem": gnt_flavor.ram,
315
                            "maxmem": gnt_flavor.ram}
316
                event_time = datetime.datetime.now()
317
                backend_mod.process_op_status(
318
                    vm=vm, etime=event_time, jobid=-0,
319
                    opcode=opcode, status='success',
320
                    beparams=beparams,
321
                    logmsg='Reconciliation: simulated Ganeti event')
322
                # process_op_status with beparams will set the vmstate to
323
                # shutdown. Fix this be returning it to old state
324
                vm = VirtualMachine.objects.get(pk=id)
325
                vm.operstate = old_state
326
                vm.save()
327
            print >> sys.stderr, "    ...done"