root / snf-cyclades-app / synnefo / logic / management / commands / reconcile-networks.py @ 9115d567
History | View | Annotate | Download (12.7 kB)
1 |
# Copyright 2011-2012 GRNET S.A. All rights reserved.
|
---|---|
2 |
#
|
3 |
# Redistribution and use in source and binary forms, with or without
|
4 |
# modification, are permitted provided that the following conditions
|
5 |
# are met:
|
6 |
#
|
7 |
# 1. Redistributions of source code must retain the above copyright
|
8 |
# notice, this list of conditions and the following disclaimer.
|
9 |
#
|
10 |
# 2. Redistributions in binary form must reproduce the above copyright
|
11 |
# notice, this list of conditions and the following disclaimer in the
|
12 |
# documentation and/or other materials provided with the distribution.
|
13 |
#
|
14 |
# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
|
15 |
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
16 |
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
17 |
# ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
|
18 |
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
19 |
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
20 |
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
21 |
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
22 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
23 |
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
24 |
# SUCH DAMAGE.
|
25 |
#
|
26 |
# The views and conclusions contained in the software and documentation are
|
27 |
# those of the authors and should not be interpreted as representing official
|
28 |
# policies, either expressed or implied, of GRNET S.A.
|
29 |
#
|
30 |
"""Reconciliation management command
|
31 |
|
32 |
Management command to reconcile the contents of the Synnefo DB with
|
33 |
the state of the Ganeti backend. See docstring on top of
|
34 |
logic/reconciliation.py for a description of reconciliation rules.
|
35 |
|
36 |
"""
|
37 |
import sys |
38 |
import datetime |
39 |
import bitarray |
40 |
|
41 |
from optparse import make_option |
42 |
|
43 |
from synnefo.settings import PUBLIC_USE_POOL |
44 |
from django.core.management.base import BaseCommand |
45 |
from django.db import transaction |
46 |
|
47 |
from synnefo.db.models import Backend, Network, BackendNetwork |
48 |
from synnefo.db.pools import IPPool |
49 |
from synnefo.logic import reconciliation, utils |
50 |
from synnefo.logic import backend as backend_mod |
51 |
|
52 |
fix = False
|
53 |
write = sys.stdout.write |
54 |
|
55 |
|
56 |
class Command(BaseCommand): |
57 |
help = """Reconcile contents of Synnefo DB with state of Ganeti backend
|
58 |
|
59 |
Network reconciliation can detect and fix the following cases:
|
60 |
- Missing database entries for a network in a Ganeti backend
|
61 |
- Stale database networks, which do no exist in the Ganeti backend
|
62 |
- Missing Ganeti networks
|
63 |
- Ganeti networks that are not connected to all Ganeti nodegroups
|
64 |
- Networks that have unsynced state
|
65 |
- Networks that have unsynced IP pools
|
66 |
- Orphan networks in the Ganeti backend
|
67 |
"""
|
68 |
|
69 |
can_import_settings = True
|
70 |
output_transaction = True # The management command runs inside |
71 |
# an SQL transaction
|
72 |
option_list = BaseCommand.option_list + ( |
73 |
make_option('--fix-all', action='store_true', |
74 |
dest='fix', default=False, |
75 |
help='Fix all issues.'),
|
76 |
make_option('--conflicting-ips', action='store_true', |
77 |
dest='conflicting_ips', default=False, |
78 |
help='Detect conflicting ips')
|
79 |
) |
80 |
|
81 |
def handle(self, **options): |
82 |
global fix, write
|
83 |
fix = options['fix']
|
84 |
write = self.stdout.write
|
85 |
self.verbosity = int(options['verbosity']) |
86 |
conflicting_ips = options['conflicting_ips']
|
87 |
reconcile_networks(conflicting_ips) |
88 |
|
89 |
|
90 |
def reconcile_networks(conflicting_ips=False): |
91 |
# Get models from DB
|
92 |
backends = Backend.objects.exclude(offline=True)
|
93 |
networks = Network.objects.filter(deleted=False)
|
94 |
|
95 |
# Get info from all ganeti backends
|
96 |
ganeti_networks = {} |
97 |
ganeti_hanging_networks = {} |
98 |
for b in backends: |
99 |
g_nets = reconciliation.get_networks_from_ganeti(b) |
100 |
ganeti_networks[b] = g_nets |
101 |
g_hanging_nets = reconciliation.hanging_networks(b, g_nets) |
102 |
ganeti_hanging_networks[b] = g_hanging_nets |
103 |
|
104 |
# Perform reconciliation for each network
|
105 |
for network in networks: |
106 |
ip_available_maps = [] |
107 |
ip_reserved_maps = [] |
108 |
uses_pool = not network.public or PUBLIC_USE_POOL |
109 |
for bend in backends: |
110 |
bnet = get_backend_network(network, bend) |
111 |
gnet = ganeti_networks[bend].get(network.id) |
112 |
if not bnet: |
113 |
if network.floating_ip_pool:
|
114 |
# Network is a floating IP pool and does not exist in
|
115 |
# backend. We need to create it
|
116 |
bnet = reconcile_parted_network(network, bend) |
117 |
elif not gnet: |
118 |
# Network does not exist either in Ganeti nor in BD.
|
119 |
continue
|
120 |
else:
|
121 |
# Network exists in Ganeti and not in DB.
|
122 |
if network.action != "DESTROY" and not network.public: |
123 |
bnet = reconcile_parted_network(network, bend) |
124 |
|
125 |
if not gnet: |
126 |
# Network does not exist in Ganeti. If the network action is
|
127 |
# DESTROY, we have to mark as deleted in DB, else we have to
|
128 |
# create it in Ganeti.
|
129 |
if network.action == "DESTROY": |
130 |
if bnet.operstate != "DELETED": |
131 |
reconcile_stale_network(bnet) |
132 |
else:
|
133 |
reconcile_missing_network(network, bend) |
134 |
# Skip rest reconciliation!
|
135 |
continue
|
136 |
|
137 |
try:
|
138 |
hanging_groups = ganeti_hanging_networks[bend][network.id] |
139 |
except KeyError: |
140 |
# Network is connected to all nodegroups
|
141 |
hanging_groups = [] |
142 |
|
143 |
if hanging_groups:
|
144 |
# CASE-3: Ganeti networks not connected to all nodegroups
|
145 |
reconcile_hanging_groups(network, bend, hanging_groups) |
146 |
continue
|
147 |
|
148 |
if bnet.operstate != 'ACTIVE': |
149 |
# CASE-4: Unsynced network state. At this point the network
|
150 |
# exists and is connected to all nodes so is must be active!
|
151 |
reconcile_unsynced_network(network, bend, bnet) |
152 |
|
153 |
if uses_pool:
|
154 |
# Get ganeti IP Pools
|
155 |
available_map, reserved_map = get_network_pool(gnet) |
156 |
ip_available_maps.append(available_map) |
157 |
ip_reserved_maps.append(reserved_map) |
158 |
|
159 |
if uses_pool and (ip_available_maps or ip_reserved_maps): |
160 |
# CASE-5: Unsynced IP Pools
|
161 |
reconcile_ip_pools(network, ip_available_maps, ip_reserved_maps) |
162 |
|
163 |
if conflicting_ips:
|
164 |
detect_conflicting_ips() |
165 |
|
166 |
# CASE-6: Orphan networks
|
167 |
reconcile_orphan_networks(networks, ganeti_networks) |
168 |
|
169 |
|
170 |
def get_backend_network(network, backend): |
171 |
try:
|
172 |
return BackendNetwork.objects.get(network=network, backend=backend)
|
173 |
except BackendNetwork.DoesNotExist:
|
174 |
return None |
175 |
|
176 |
|
177 |
def reconcile_parted_network(network, backend): |
178 |
write("D: Missing DB entry for network %s in backend %s\n" %
|
179 |
(network, backend)) |
180 |
if fix:
|
181 |
network.create_backend_network(backend) |
182 |
write("F: Created DB entry\n")
|
183 |
bnet = get_backend_network(network, backend) |
184 |
return bnet
|
185 |
|
186 |
|
187 |
def reconcile_stale_network(backend_network): |
188 |
write("D: Stale DB entry for network %s in backend %s\n" %
|
189 |
(backend_network.network, backend_network.backend)) |
190 |
if fix:
|
191 |
etime = datetime.datetime.now() |
192 |
backend_mod.process_network_status(backend_network, etime, 0,
|
193 |
"OP_NETWORK_REMOVE",
|
194 |
"success",
|
195 |
"Reconciliation simulated event")
|
196 |
write("F: Reconciled event: OP_NETWORK_REMOVE\n")
|
197 |
|
198 |
|
199 |
def reconcile_missing_network(network, backend): |
200 |
write("D: Missing Ganeti network %s in backend %s\n" %
|
201 |
(network, backend)) |
202 |
if fix:
|
203 |
backend_mod.create_network(network, backend) |
204 |
write("F: Issued OP_NETWORK_CONNECT\n")
|
205 |
|
206 |
|
207 |
def reconcile_hanging_groups(network, backend, hanging_groups): |
208 |
write('D: Network %s in backend %s is not connected to '
|
209 |
'the following groups:\n' % (network, backend))
|
210 |
write('- ' + '\n- '.join(hanging_groups) + '\n') |
211 |
if fix:
|
212 |
for group in hanging_groups: |
213 |
write('F: Connecting network %s to nodegroup %s\n'
|
214 |
% (network, group)) |
215 |
backend_mod.connect_network(network, backend, depends=[], |
216 |
group=group) |
217 |
|
218 |
|
219 |
def reconcile_unsynced_network(network, backend, backend_network): |
220 |
write("D: Unsynced network %s in backend %s\n" % (network, backend))
|
221 |
if fix:
|
222 |
write("F: Issuing OP_NETWORK_CONNECT\n")
|
223 |
etime = datetime.datetime.now() |
224 |
backend_mod.process_network_status(backend_network, etime, 0,
|
225 |
"OP_NETWORK_CONNECT",
|
226 |
"success",
|
227 |
"Reconciliation simulated eventd")
|
228 |
|
229 |
|
230 |
@transaction.commit_on_success
|
231 |
def reconcile_ip_pools(network, available_maps, reserved_maps): |
232 |
available_map = reduce(lambda x, y: x & y, available_maps) |
233 |
reserved_map = reduce(lambda x, y: x & y, reserved_maps) |
234 |
|
235 |
pool = network.get_pool() |
236 |
# Temporary release unused floating IPs
|
237 |
temp_pool = network.get_pool() |
238 |
used_ips = network.nics.values_list("ipv4", flat=True) |
239 |
unused_static_ips = network.floating_ips.exclude(ipv4__in=used_ips) |
240 |
map(lambda ip: temp_pool.put(ip.ipv4), unused_static_ips) |
241 |
if temp_pool.available != available_map:
|
242 |
write("D: Unsynced available map of network %s:\n"
|
243 |
"\tDB: %r\n\tGB: %r\n" %
|
244 |
(network, temp_pool.available.to01(), available_map.to01())) |
245 |
if fix:
|
246 |
pool.available = available_map |
247 |
# Release unsued floating IPs, as they are not included in the
|
248 |
# available map
|
249 |
map(lambda ip: pool.reserve(ip.ipv4), unused_static_ips) |
250 |
pool.save() |
251 |
if pool.reserved != reserved_map:
|
252 |
write("D: Unsynced reserved map of network %s:\n"
|
253 |
"\tDB: %r\n\tGB: %r\n" %
|
254 |
(network, pool.reserved.to01(), reserved_map.to01())) |
255 |
if fix:
|
256 |
pool.reserved = reserved_map |
257 |
pool.save() |
258 |
|
259 |
|
260 |
def detect_conflicting_ips(network): |
261 |
"""Detect NIC's that have the same IP in the same network."""
|
262 |
machine_ips = network.nics.all().values_list('ipv4', 'machine') |
263 |
ips = map(lambda x: x[0], machine_ips) |
264 |
distinct_ips = set(ips)
|
265 |
if len(distinct_ips) < len(ips): |
266 |
for i in distinct_ips: |
267 |
ips.remove(i) |
268 |
for i in ips: |
269 |
machines = [utils.id_to_instance_name(x[1])
|
270 |
for x in machine_ips if x[0] == i] |
271 |
write('D: Conflicting IP:%s Machines: %s\n' %
|
272 |
(i, ', '.join(machines)))
|
273 |
|
274 |
|
275 |
def reconcile_orphan_networks(db_networks, ganeti_networks): |
276 |
# Detect Orphan Networks in Ganeti
|
277 |
db_network_ids = set([net.id for net in db_networks]) |
278 |
for back_end, ganeti_networks in ganeti_networks.items(): |
279 |
ganeti_network_ids = set(ganeti_networks.keys())
|
280 |
orphans = ganeti_network_ids - db_network_ids |
281 |
|
282 |
if len(orphans) > 0: |
283 |
write('D: Orphan Networks in backend %s:\n' % back_end.clustername)
|
284 |
write('- ' + '\n- '.join([str(o) for o in orphans]) + '\n') |
285 |
if fix:
|
286 |
for net_id in orphans: |
287 |
write('Disconnecting and deleting network %d\n' % net_id)
|
288 |
try:
|
289 |
network = Network.objects.get(id=net_id) |
290 |
backend_mod.delete_network(network, |
291 |
backend=back_end) |
292 |
except Network.DoesNotExist:
|
293 |
write("Not entry for network %s in DB !!\n" % net_id)
|
294 |
|
295 |
|
296 |
def get_network_pool(gnet): |
297 |
"""Return available and reserved IP maps.
|
298 |
|
299 |
Extract the available and reserved IP map from the info return from Ganeti
|
300 |
for a network.
|
301 |
|
302 |
"""
|
303 |
converter = IPPool(Foo(gnet['network']))
|
304 |
a_map = bitarray_from_map(gnet['map'])
|
305 |
a_map.invert() |
306 |
reserved = gnet['external_reservations']
|
307 |
r_map = a_map.copy() |
308 |
r_map.setall(True)
|
309 |
for address in reserved.split(','): |
310 |
index = converter.value_to_index(address) |
311 |
a_map[index] = True
|
312 |
r_map[index] = False
|
313 |
return a_map, r_map
|
314 |
|
315 |
|
316 |
def bitarray_from_map(bitmap): |
317 |
return bitarray.bitarray(bitmap.replace("X", "1").replace(".", "0")) |
318 |
|
319 |
|
320 |
class Foo(): |
321 |
def __init__(self, subnet): |
322 |
self.available_map = '' |
323 |
self.reserved_map = '' |
324 |
self.size = 0 |
325 |
self.network = Foo.Foo1(subnet)
|
326 |
|
327 |
class Foo1(): |
328 |
def __init__(self, subnet): |
329 |
self.subnet = subnet
|
330 |
self.gateway = None |