root / snf-cyclades-app / synnefo / logic / management / commands / reconcile-servers.py @ 8e440b3a
History | View | Annotate | Download (15.5 kB)
1 |
# Copyright 2011-2012 GRNET S.A. All rights reserved.
|
---|---|
2 |
#
|
3 |
# Redistribution and use in source and binary forms, with or without
|
4 |
# modification, are permitted provided that the following conditions
|
5 |
# are met:
|
6 |
#
|
7 |
# 1. Redistributions of source code must retain the above copyright
|
8 |
# notice, this list of conditions and the following disclaimer.
|
9 |
#
|
10 |
# 2. Redistributions in binary form must reproduce the above copyright
|
11 |
# notice, this list of conditions and the following disclaimer in the
|
12 |
# documentation and/or other materials provided with the distribution.
|
13 |
#
|
14 |
# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
|
15 |
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
16 |
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
17 |
# ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
|
18 |
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
19 |
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
20 |
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
21 |
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
22 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
23 |
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
24 |
# SUCH DAMAGE.
|
25 |
#
|
26 |
# The views and conclusions contained in the software and documentation are
|
27 |
# those of the authors and should not be interpreted as representing official
|
28 |
# policies, either expressed or implied, of GRNET S.A.
|
29 |
#
|
30 |
"""Reconciliation management command
|
31 |
|
32 |
Management command to reconcile the contents of the Synnefo DB with
|
33 |
the state of the Ganeti backend. See docstring on top of
|
34 |
logic/reconciliation.py for a description of reconciliation rules.
|
35 |
|
36 |
"""
|
37 |
import sys |
38 |
import datetime |
39 |
|
40 |
from optparse import make_option |
41 |
|
42 |
from django.core.management.base import BaseCommand, CommandError |
43 |
|
44 |
from synnefo.db.models import (Backend, VirtualMachine, Network, |
45 |
pooled_rapi_client) |
46 |
from synnefo.logic import reconciliation, utils |
47 |
from synnefo.logic import backend as backend_mod |
48 |
from synnefo.util.mac2eui64 import mac2eui64 |
49 |
from synnefo.management.common import get_backend |
50 |
|
51 |
|
52 |
class Command(BaseCommand): |
53 |
can_import_settings = True
|
54 |
|
55 |
help = 'Reconcile contents of Synnefo DB with state of Ganeti backend'
|
56 |
output_transaction = True # The management command runs inside |
57 |
# an SQL transaction
|
58 |
option_list = BaseCommand.option_list + ( |
59 |
make_option('--detect-stale', action='store_true', dest='detect_stale', |
60 |
default=False, help='Detect stale VM entries in DB'), |
61 |
make_option('--detect-orphans', action='store_true', |
62 |
dest='detect_orphans',
|
63 |
default=False, help='Detect orphan instances in Ganeti'), |
64 |
make_option('--detect-unsynced', action='store_true', |
65 |
dest='detect_unsynced',
|
66 |
default=False, help='Detect unsynced operstate between ' + |
67 |
'DB and Ganeti'),
|
68 |
make_option('--detect-build-errors', action='store_true', |
69 |
dest='detect_build_errors', default=False, |
70 |
help='Detect instances with build error'),
|
71 |
make_option('--detect-unsynced-nics', action='store_true', |
72 |
dest='detect_unsynced_nics', default=False, |
73 |
help='Detect unsynced nics between DB and Ganeti'),
|
74 |
make_option('--detect-unsynced-flavors', action='store_true', |
75 |
dest='detect_unsynced_flavors', default=False, |
76 |
help='Detect unsynced flavors between DB and Ganeti'),
|
77 |
make_option('--detect-all', action='store_true', |
78 |
dest='detect_all',
|
79 |
default=False, help='Enable all --detect-* arguments'), |
80 |
make_option('--fix-stale', action='store_true', dest='fix_stale', |
81 |
default=False, help='Fix (remove) stale DB entries in DB'), |
82 |
make_option('--fix-orphans', action='store_true', dest='fix_orphans', |
83 |
default=False, help='Fix (remove) orphan Ganeti VMs'), |
84 |
make_option('--fix-unsynced', action='store_true', dest='fix_unsynced', |
85 |
default=False, help='Fix server operstate in DB, set ' + |
86 |
'from Ganeti'),
|
87 |
make_option('--fix-build-errors', action='store_true', |
88 |
dest='fix_build_errors', default=False, |
89 |
help='Fix (remove) instances with build errors'),
|
90 |
make_option('--fix-unsynced-nics', action='store_true', |
91 |
dest='fix_unsynced_nics', default=False, |
92 |
help='Fix unsynced nics between DB and Ganeti'),
|
93 |
make_option('--fix-unsynced-flavors', action='store_true', |
94 |
dest='fix_unsynced_flavors', default=False, |
95 |
help='Fix unsynced flavors between DB and Ganeti'),
|
96 |
make_option('--fix-all', action='store_true', dest='fix_all', |
97 |
default=False, help='Enable all --fix-* arguments'), |
98 |
make_option('--backend-id', default=None, dest='backend-id', |
99 |
help='Reconcilie VMs only for this backend'),
|
100 |
) |
101 |
|
102 |
def _process_args(self, options): |
103 |
keys_detect = [k for k in options.keys() if k.startswith('detect_')] |
104 |
keys_fix = [k for k in options.keys() if k.startswith('fix_')] |
105 |
|
106 |
if not reduce(lambda x, y: x or y, |
107 |
map(lambda x: options[x], keys_detect)): |
108 |
options['detect_all'] = True |
109 |
|
110 |
if options['detect_all']: |
111 |
for kd in keys_detect: |
112 |
options[kd] = True
|
113 |
if options['fix_all']: |
114 |
for kf in keys_fix: |
115 |
options[kf] = True
|
116 |
|
117 |
for kf in keys_fix: |
118 |
kd = kf.replace('fix_', 'detect_', 1) |
119 |
if (options[kf] and not options[kd]): |
120 |
raise CommandError("Cannot use --%s without corresponding " |
121 |
"--%s argument" % (kf, kd))
|
122 |
|
123 |
def handle(self, **options): |
124 |
verbosity = int(options['verbosity']) |
125 |
self._process_args(options)
|
126 |
backend_id = options['backend-id']
|
127 |
if backend_id:
|
128 |
backends = [get_backend(backend_id)] |
129 |
else:
|
130 |
backends = Backend.objects.filter(offline=False)
|
131 |
|
132 |
with_nics = options["detect_unsynced_nics"]
|
133 |
|
134 |
DBVMs = reconciliation.get_servers_from_db(backends, with_nics) |
135 |
GanetiVMs = reconciliation.get_instances_from_ganeti(backends) |
136 |
|
137 |
#
|
138 |
# Detect problems
|
139 |
#
|
140 |
if options['detect_stale']: |
141 |
stale = reconciliation.stale_servers_in_db(DBVMs, GanetiVMs) |
142 |
if len(stale) > 0: |
143 |
print >> sys.stderr, "Found the following stale server IDs: " |
144 |
print " " + "\n ".join( |
145 |
[str(x) for x in stale]) |
146 |
elif verbosity == 2: |
147 |
print >> sys.stderr, "Found no stale server IDs in DB." |
148 |
|
149 |
if options['detect_orphans']: |
150 |
orphans = reconciliation.orphan_instances_in_ganeti(DBVMs, |
151 |
GanetiVMs) |
152 |
if len(orphans) > 0: |
153 |
print >> sys.stderr, "Found orphan Ganeti instances with IDs: " |
154 |
print " " + "\n ".join( |
155 |
[str(x) for x in orphans]) |
156 |
elif verbosity == 2: |
157 |
print >> sys.stderr, "Found no orphan Ganeti instances." |
158 |
|
159 |
if options['detect_unsynced']: |
160 |
unsynced = reconciliation.unsynced_operstate(DBVMs, GanetiVMs) |
161 |
if len(unsynced) > 0: |
162 |
print >> sys.stderr, "The operstate of the following server" \ |
163 |
" IDs is out-of-sync:"
|
164 |
print " " + "\n ".join( |
165 |
["%d is %s in DB, %s in Ganeti" %
|
166 |
(x[0], x[1], ('UP' if x[2] else 'DOWN')) |
167 |
for x in unsynced]) |
168 |
elif verbosity == 2: |
169 |
print >> sys.stderr, "The operstate of all servers is in sync." |
170 |
|
171 |
if options['detect_build_errors']: |
172 |
build_errors = reconciliation.\ |
173 |
instances_with_build_errors(DBVMs, GanetiVMs) |
174 |
if len(build_errors) > 0: |
175 |
msg = "The os for the following server IDs was not build"\
|
176 |
" successfully:"
|
177 |
print >> sys.stderr, msg
|
178 |
print " " + "\n ".join( |
179 |
["%d" % x for x in build_errors]) |
180 |
elif verbosity == 2: |
181 |
print >> sys.stderr, "Found no instances with build errors." |
182 |
|
183 |
if options['detect_unsynced_nics']: |
184 |
def pretty_print_nics(nics): |
185 |
if not nics: |
186 |
print ''.ljust(18) + 'None' |
187 |
for index, info in nics.items(): |
188 |
print ''.ljust(18) + 'nic/' + str(index) +\ |
189 |
': MAC: %s, IP: %s, Network: %s' % \
|
190 |
(info['mac'], info['ipv4'], info['network']) |
191 |
|
192 |
unsynced_nics = reconciliation.unsynced_nics(DBVMs, GanetiVMs) |
193 |
if len(unsynced_nics) > 0: |
194 |
msg = "The NICs of the servers with the following IDs are"\
|
195 |
" unsynced:"
|
196 |
print >> sys.stderr, msg
|
197 |
for id, nics in unsynced_nics.items(): |
198 |
print ''.ljust(2) + '%6d:' % id |
199 |
print ''.ljust(8) + '%8s:' % 'DB' |
200 |
pretty_print_nics(nics[0])
|
201 |
print ''.ljust(8) + '%8s:' % 'Ganeti' |
202 |
pretty_print_nics(nics[1])
|
203 |
elif verbosity == 2: |
204 |
print >> sys.stderr, "All instance nics are synced." |
205 |
|
206 |
if options["detect_unsynced_flavors"]: |
207 |
unsynced_flavors = reconciliation.unsynced_flavors(DBVMs, |
208 |
GanetiVMs) |
209 |
if len(unsynced_flavors) > 0: |
210 |
print >> sys.stderr, "The flavor of the following server" \ |
211 |
" IDs is out-of-sync:"
|
212 |
print " " + "\n ".join( |
213 |
["%d is %s in DB, %s in Ganeti" %
|
214 |
(x[0], x[1], x[2]) |
215 |
for x in unsynced_flavors]) |
216 |
elif verbosity == 2: |
217 |
print >> sys.stderr, "All instance flavors are synced." |
218 |
|
219 |
#
|
220 |
# Then fix them
|
221 |
#
|
222 |
if options['fix_stale'] and len(stale) > 0: |
223 |
print >> sys.stderr, \
|
224 |
"Simulating successful Ganeti removal for %d " \
|
225 |
"servers in the DB:" % len(stale) |
226 |
for vm in VirtualMachine.objects.filter(pk__in=stale): |
227 |
event_time = datetime.datetime.now() |
228 |
backend_mod.process_op_status( |
229 |
vm=vm, |
230 |
etime=event_time, |
231 |
jobid=-0,
|
232 |
opcode='OP_INSTANCE_REMOVE', status='success', |
233 |
logmsg='Reconciliation: simulated Ganeti event')
|
234 |
print >> sys.stderr, " ...done" |
235 |
|
236 |
if options['fix_orphans'] and len(orphans) > 0: |
237 |
print >> sys.stderr, \
|
238 |
"Issuing OP_INSTANCE_REMOVE for %d Ganeti instances:" % \
|
239 |
len(orphans)
|
240 |
for id in orphans: |
241 |
try:
|
242 |
vm = VirtualMachine.objects.get(pk=id)
|
243 |
with pooled_rapi_client(vm) as client: |
244 |
client.DeleteInstance(utils.id_to_instance_name(id))
|
245 |
except VirtualMachine.DoesNotExist:
|
246 |
print >> sys.stderr, "No entry for VM %d in DB !!" % id |
247 |
print >> sys.stderr, " ...done" |
248 |
|
249 |
if options['fix_unsynced'] and len(unsynced) > 0: |
250 |
print >> sys.stderr, "Setting the state of %d out-of-sync VMs:" % \ |
251 |
len(unsynced)
|
252 |
for id, db_state, ganeti_up in unsynced: |
253 |
vm = VirtualMachine.objects.get(pk=id)
|
254 |
opcode = "OP_INSTANCE_REBOOT" if ganeti_up \ |
255 |
else "OP_INSTANCE_SHUTDOWN" |
256 |
event_time = datetime.datetime.now() |
257 |
backend_mod.process_op_status( |
258 |
vm=vm, etime=event_time, jobid=-0,
|
259 |
opcode=opcode, status='success',
|
260 |
logmsg='Reconciliation: simulated Ganeti event')
|
261 |
print >> sys.stderr, " ...done" |
262 |
|
263 |
if options['fix_build_errors'] and len(build_errors) > 0: |
264 |
print >> sys.stderr, "Setting the state of %d build-errors VMs:" %\ |
265 |
len(build_errors)
|
266 |
for id in build_errors: |
267 |
vm = VirtualMachine.objects.get(pk=id)
|
268 |
event_time = datetime.datetime.now() |
269 |
backend_mod.process_op_status( |
270 |
vm=vm, etime=event_time, jobid=-0,
|
271 |
opcode="OP_INSTANCE_CREATE", status='error', |
272 |
logmsg='Reconciliation: simulated Ganeti event')
|
273 |
print >> sys.stderr, " ...done" |
274 |
|
275 |
if options['fix_unsynced_nics'] and len(unsynced_nics) > 0: |
276 |
print >> sys.stderr, "Setting the nics of %d out-of-sync VMs:" % \ |
277 |
len(unsynced_nics)
|
278 |
for id, nics in unsynced_nics.items(): |
279 |
vm = VirtualMachine.objects.get(pk=id)
|
280 |
nics = nics[1] # Ganeti nics |
281 |
if nics == {}: # No nics |
282 |
vm.nics.all.delete() |
283 |
continue
|
284 |
for index, nic in nics.items(): |
285 |
net_id = utils.id_from_network_name(nic['network'])
|
286 |
subnet6 = Network.objects.get(id=net_id).subnet6 |
287 |
# Produce ipv6
|
288 |
ipv6 = subnet6 and mac2eui64(nic['mac'], subnet6) or None |
289 |
nic['ipv6'] = ipv6
|
290 |
# Rename ipv4 to ip
|
291 |
nic['ip'] = nic['ipv4'] |
292 |
# Dict to sorted list
|
293 |
final_nics = [] |
294 |
nics_keys = nics.keys() |
295 |
nics_keys.sort() |
296 |
for i in nics_keys: |
297 |
if nics[i]['network']: |
298 |
final_nics.append(nics[i]) |
299 |
else:
|
300 |
print 'Network of nic %d of vm %s is None. ' \ |
301 |
'Can not reconcile' % (i, vm.backend_vm_id)
|
302 |
event_time = datetime.datetime.now() |
303 |
backend_mod.process_net_status(vm=vm, etime=event_time, |
304 |
nics=final_nics) |
305 |
print >> sys.stderr, " ...done" |
306 |
if options["fix_unsynced_flavors"] and len(unsynced_flavors) > 0: |
307 |
print >> sys.stderr, "Setting the flavor of %d unsynced VMs:" % \ |
308 |
len(unsynced_flavors)
|
309 |
for id, db_flavor, gnt_flavor in unsynced_flavors: |
310 |
vm = VirtualMachine.objects.get(pk=id)
|
311 |
old_state = vm.operstate |
312 |
opcode = "OP_INSTANCE_SET_PARAMS"
|
313 |
beparams = {"vcpus": gnt_flavor.cpu,
|
314 |
"minmem": gnt_flavor.ram,
|
315 |
"maxmem": gnt_flavor.ram}
|
316 |
event_time = datetime.datetime.now() |
317 |
backend_mod.process_op_status( |
318 |
vm=vm, etime=event_time, jobid=-0,
|
319 |
opcode=opcode, status='success',
|
320 |
beparams=beparams, |
321 |
logmsg='Reconciliation: simulated Ganeti event')
|
322 |
# process_op_status with beparams will set the vmstate to
|
323 |
# shutdown. Fix this be returning it to old state
|
324 |
vm = VirtualMachine.objects.get(pk=id)
|
325 |
vm.operstate = old_state |
326 |
vm.save() |
327 |
print >> sys.stderr, " ...done" |