Revision 89b2b908 snf-cyclades-app/synnefo/logic/management/commands/reconcile-networks.py
b/snf-cyclades-app/synnefo/logic/management/commands/reconcile-networks.py | ||
---|---|---|
1 |
# Copyright 2011-2012 GRNET S.A. All rights reserved.
|
|
1 |
# Copyright 2011-2013 GRNET S.A. All rights reserved.
|
|
2 | 2 |
# |
3 | 3 |
# Redistribution and use in source and binary forms, with or without |
4 | 4 |
# modification, are permitted provided that the following conditions |
... | ... | |
34 | 34 |
logic/reconciliation.py for a description of reconciliation rules. |
35 | 35 |
|
36 | 36 |
""" |
37 |
import datetime |
|
38 |
import bitarray |
|
39 | 37 |
import logging |
40 |
|
|
41 | 38 |
from optparse import make_option |
42 |
|
|
43 | 39 |
from django.core.management.base import BaseCommand |
44 |
from django.db import transaction |
|
45 |
|
|
46 |
from synnefo.db.models import Backend, Network, BackendNetwork |
|
47 |
from synnefo.db.pools import IPPool |
|
48 |
from synnefo.logic import reconciliation, utils |
|
49 |
from synnefo.logic import backend as backend_mod |
|
40 |
from synnefo.logic import reconciliation |
|
50 | 41 |
|
51 | 42 |
|
52 | 43 |
class Command(BaseCommand): |
... | ... | |
93 | 84 |
logger.setLevel(logging.WARNING) |
94 | 85 |
|
95 | 86 |
logger.addHandler(log_handler) |
96 |
reconciler = NetworkReconciler(logger=logger, fix=fix, |
|
97 |
conflicting_ips=conflicting_ips) |
|
87 |
reconciler = reconciliation.NetworkReconciler( |
|
88 |
logger=logger, |
|
89 |
fix=fix, |
|
90 |
conflicting_ips=conflicting_ips) |
|
98 | 91 |
reconciler.reconcile_networks() |
99 |
|
|
100 |
|
|
101 |
class NetworkReconciler(object): |
|
102 |
def __init__(self, logger, fix=False, conflicting_ips=False): |
|
103 |
self.log = logger |
|
104 |
self.conflicting_ips = conflicting_ips |
|
105 |
self.fix = fix |
|
106 |
|
|
107 |
@transaction.commit_on_success |
|
108 |
def reconcile_networks(self): |
|
109 |
# Get models from DB |
|
110 |
backends = Backend.objects.exclude(offline=True) |
|
111 |
networks = Network.objects.filter(deleted=False) |
|
112 |
|
|
113 |
# Get info from all ganeti backends |
|
114 |
ganeti_networks = {} |
|
115 |
ganeti_hanging_networks = {} |
|
116 |
for b in backends: |
|
117 |
g_nets = reconciliation.get_networks_from_ganeti(b) |
|
118 |
ganeti_networks[b] = g_nets |
|
119 |
g_hanging_nets = reconciliation.hanging_networks(b, g_nets) |
|
120 |
ganeti_hanging_networks[b] = g_hanging_nets |
|
121 |
|
|
122 |
# Perform reconciliation for each network |
|
123 |
for network in networks: |
|
124 |
ip_available_maps = [] |
|
125 |
ip_reserved_maps = [] |
|
126 |
for bend in backends: |
|
127 |
bnet = get_backend_network(network, bend) |
|
128 |
gnet = ganeti_networks[bend].get(network.id) |
|
129 |
if not bnet: |
|
130 |
if network.floating_ip_pool: |
|
131 |
# Network is a floating IP pool and does not exist in |
|
132 |
# backend. We need to create it |
|
133 |
bnet = self.reconcile_parted_network(network, bend) |
|
134 |
elif not gnet: |
|
135 |
# Network does not exist either in Ganeti nor in BD. |
|
136 |
continue |
|
137 |
else: |
|
138 |
# Network exists in Ganeti and not in DB. |
|
139 |
if network.action != "DESTROY" and not network.public: |
|
140 |
bnet = self.reconcile_parted_network(network, bend) |
|
141 |
|
|
142 |
if not gnet: |
|
143 |
# Network does not exist in Ganeti. If the network action |
|
144 |
# is DESTROY, we have to mark as deleted in DB, else we |
|
145 |
# have to create it in Ganeti. |
|
146 |
if network.action == "DESTROY": |
|
147 |
if bnet.operstate != "DELETED": |
|
148 |
self.reconcile_stale_network(bnet) |
|
149 |
else: |
|
150 |
self.reconcile_missing_network(network, bend) |
|
151 |
# Skip rest reconciliation! |
|
152 |
continue |
|
153 |
|
|
154 |
try: |
|
155 |
hanging_groups = ganeti_hanging_networks[bend][network.id] |
|
156 |
except KeyError: |
|
157 |
# Network is connected to all nodegroups |
|
158 |
hanging_groups = [] |
|
159 |
|
|
160 |
if hanging_groups: |
|
161 |
# CASE-3: Ganeti networks not connected to all nodegroups |
|
162 |
self.reconcile_hanging_groups(network, bend, |
|
163 |
hanging_groups) |
|
164 |
continue |
|
165 |
|
|
166 |
if bnet.operstate != 'ACTIVE': |
|
167 |
# CASE-4: Unsynced network state. At this point the network |
|
168 |
# exists and is connected to all nodes so is must be |
|
169 |
# active! |
|
170 |
self.reconcile_unsynced_network(network, bend, bnet) |
|
171 |
|
|
172 |
# Get ganeti IP Pools |
|
173 |
available_map, reserved_map = get_network_pool(gnet) |
|
174 |
ip_available_maps.append(available_map) |
|
175 |
ip_reserved_maps.append(reserved_map) |
|
176 |
|
|
177 |
if ip_available_maps or ip_reserved_maps: |
|
178 |
# CASE-5: Unsynced IP Pools |
|
179 |
self.reconcile_ip_pools(network, ip_available_maps, |
|
180 |
ip_reserved_maps) |
|
181 |
|
|
182 |
if self.conflicting_ips: |
|
183 |
self.detect_conflicting_ips() |
|
184 |
|
|
185 |
# CASE-6: Orphan networks |
|
186 |
self.reconcile_orphan_networks(networks, ganeti_networks) |
|
187 |
|
|
188 |
def reconcile_parted_network(self, network, backend): |
|
189 |
self.log.info("D: Missing DB entry for network %s in backend %s", |
|
190 |
network, backend) |
|
191 |
if self.fix: |
|
192 |
network.create_backend_network(backend) |
|
193 |
self.log.info("F: Created DB entry") |
|
194 |
bnet = get_backend_network(network, backend) |
|
195 |
return bnet |
|
196 |
|
|
197 |
def reconcile_stale_network(self, backend_network): |
|
198 |
self.log.info("D: Stale DB entry for network %s in backend %s", |
|
199 |
backend_network.network, backend_network.backend) |
|
200 |
if self.fix: |
|
201 |
etime = datetime.datetime.now() |
|
202 |
backend_mod.process_network_status( |
|
203 |
backend_network, etime, 0, |
|
204 |
"OP_NETWORK_REMOVE", |
|
205 |
"success", |
|
206 |
"Reconciliation simulated event") |
|
207 |
self.log.info("F: Reconciled event: OP_NETWORK_REMOVE") |
|
208 |
|
|
209 |
def reconcile_missing_network(self, network, backend): |
|
210 |
self.log.info("D: Missing Ganeti network %s in backend %s", |
|
211 |
network, backend) |
|
212 |
if self.fix: |
|
213 |
backend_mod.create_network(network, backend) |
|
214 |
self.log.info("F: Issued OP_NETWORK_CONNECT") |
|
215 |
|
|
216 |
def reconcile_hanging_groups(self, network, backend, hanging_groups): |
|
217 |
self.log.info('D: Network %s in backend %s is not connected to ' |
|
218 |
'the following groups:', network, backend) |
|
219 |
self.log.info('- ' + '\n- '.join(hanging_groups)) |
|
220 |
if self.fix: |
|
221 |
for group in hanging_groups: |
|
222 |
self.log.info('F: Connecting network %s to nodegroup %s', |
|
223 |
network, group) |
|
224 |
backend_mod.connect_network(network, backend, depends=[], |
|
225 |
group=group) |
|
226 |
|
|
227 |
def reconcile_unsynced_network(self, network, backend, backend_network): |
|
228 |
self.log.info("D: Unsynced network %s in backend %s", network, backend) |
|
229 |
if self.fix: |
|
230 |
self.log.info("F: Issuing OP_NETWORK_CONNECT") |
|
231 |
etime = datetime.datetime.now() |
|
232 |
backend_mod.process_network_status( |
|
233 |
backend_network, etime, 0, |
|
234 |
"OP_NETWORK_CONNECT", |
|
235 |
"success", |
|
236 |
"Reconciliation simulated eventd") |
|
237 |
|
|
238 |
def reconcile_ip_pools(self, network, available_maps, reserved_maps): |
|
239 |
available_map = reduce(lambda x, y: x & y, available_maps) |
|
240 |
reserved_map = reduce(lambda x, y: x & y, reserved_maps) |
|
241 |
|
|
242 |
pool = network.get_pool() |
|
243 |
# Temporary release unused floating IPs |
|
244 |
temp_pool = network.get_pool() |
|
245 |
used_ips = network.nics.values_list("ipv4", flat=True) |
|
246 |
unused_static_ips = network.floating_ips.exclude(ipv4__in=used_ips) |
|
247 |
map(lambda ip: temp_pool.put(ip.ipv4), unused_static_ips) |
|
248 |
if temp_pool.available != available_map: |
|
249 |
self.log.info("D: Unsynced available map of network %s:\n" |
|
250 |
"\tDB: %r\n\tGB: %r", network, |
|
251 |
temp_pool.available.to01(), |
|
252 |
available_map.to01()) |
|
253 |
if self.fix: |
|
254 |
pool.available = available_map |
|
255 |
# Release unsued floating IPs, as they are not included in the |
|
256 |
# available map |
|
257 |
map(lambda ip: pool.reserve(ip.ipv4), unused_static_ips) |
|
258 |
pool.save() |
|
259 |
if pool.reserved != reserved_map: |
|
260 |
self.log.info("D: Unsynced reserved map of network %s:\n" |
|
261 |
"\tDB: %r\n\tGB: %r", network, pool.reserved.to01(), |
|
262 |
reserved_map.to01()) |
|
263 |
if self.fix: |
|
264 |
pool.reserved = reserved_map |
|
265 |
pool.save() |
|
266 |
|
|
267 |
def detect_conflicting_ips(self, network): |
|
268 |
"""Detect NIC's that have the same IP in the same network.""" |
|
269 |
machine_ips = network.nics.all().values_list('ipv4', 'machine') |
|
270 |
ips = map(lambda x: x[0], machine_ips) |
|
271 |
distinct_ips = set(ips) |
|
272 |
if len(distinct_ips) < len(ips): |
|
273 |
for i in distinct_ips: |
|
274 |
ips.remove(i) |
|
275 |
for i in ips: |
|
276 |
machines = [utils.id_to_instance_name(x[1]) |
|
277 |
for x in machine_ips if x[0] == i] |
|
278 |
self.log.info('D: Conflicting IP:%s Machines: %s', |
|
279 |
i, ', '.join(machines)) |
|
280 |
|
|
281 |
def reconcile_orphan_networks(self, db_networks, ganeti_networks): |
|
282 |
# Detect Orphan Networks in Ganeti |
|
283 |
db_network_ids = set([net.id for net in db_networks]) |
|
284 |
for back_end, ganeti_networks in ganeti_networks.items(): |
|
285 |
ganeti_network_ids = set(ganeti_networks.keys()) |
|
286 |
orphans = ganeti_network_ids - db_network_ids |
|
287 |
|
|
288 |
if len(orphans) > 0: |
|
289 |
self.log.info('D: Orphan Networks in backend %s:', |
|
290 |
back_end.clustername) |
|
291 |
self.log.info('- ' + '\n- '.join([str(o) for o in orphans])) |
|
292 |
if self.fix: |
|
293 |
for net_id in orphans: |
|
294 |
self.log.info('Disconnecting and deleting network %d', |
|
295 |
net_id) |
|
296 |
try: |
|
297 |
network = Network.objects.get(id=net_id) |
|
298 |
backend_mod.delete_network(network, |
|
299 |
backend=back_end) |
|
300 |
except Network.DoesNotExist: |
|
301 |
self.log.info("Not entry for network %s in DB !!", |
|
302 |
net_id) |
|
303 |
|
|
304 |
|
|
305 |
def get_backend_network(network, backend): |
|
306 |
try: |
|
307 |
return BackendNetwork.objects.get(network=network, backend=backend) |
|
308 |
except BackendNetwork.DoesNotExist: |
|
309 |
return None |
|
310 |
|
|
311 |
|
|
312 |
def get_network_pool(gnet): |
|
313 |
"""Return available and reserved IP maps. |
|
314 |
|
|
315 |
Extract the available and reserved IP map from the info return from Ganeti |
|
316 |
for a network. |
|
317 |
|
|
318 |
""" |
|
319 |
converter = IPPool(Foo(gnet['network'])) |
|
320 |
a_map = bitarray_from_map(gnet['map']) |
|
321 |
a_map.invert() |
|
322 |
reserved = gnet['external_reservations'] |
|
323 |
r_map = a_map.copy() |
|
324 |
r_map.setall(True) |
|
325 |
for address in reserved.split(','): |
|
326 |
index = converter.value_to_index(address.strip()) |
|
327 |
a_map[index] = True |
|
328 |
r_map[index] = False |
|
329 |
return a_map, r_map |
|
330 |
|
|
331 |
|
|
332 |
def bitarray_from_map(bitmap): |
|
333 |
return bitarray.bitarray(bitmap.replace("X", "1").replace(".", "0")) |
|
334 |
|
|
335 |
|
|
336 |
class Foo(): |
|
337 |
def __init__(self, subnet): |
|
338 |
self.available_map = '' |
|
339 |
self.reserved_map = '' |
|
340 |
self.size = 0 |
|
341 |
self.network = Foo.Foo1(subnet) |
|
342 |
|
|
343 |
class Foo1(): |
|
344 |
def __init__(self, subnet): |
|
345 |
self.subnet = subnet |
|
346 |
self.gateway = None |
Also available in: Unified diff