root / snf-cyclades-app / synnefo / logic / management / commands / reconcile-networks.py @ ad297723
History | View | Annotate | Download (9.7 kB)
1 |
# Copyright 2011-2012 GRNET S.A. All rights reserved.
|
---|---|
2 |
#
|
3 |
# Redistribution and use in source and binary forms, with or without
|
4 |
# modification, are permitted provided that the following conditions
|
5 |
# are met:
|
6 |
#
|
7 |
# 1. Redistributions of source code must retain the above copyright
|
8 |
# notice, this list of conditions and the following disclaimer.
|
9 |
#
|
10 |
# 2. Redistributions in binary form must reproduce the above copyright
|
11 |
# notice, this list of conditions and the following disclaimer in the
|
12 |
# documentation and/or other materials provided with the distribution.
|
13 |
#
|
14 |
# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
|
15 |
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
16 |
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
17 |
# ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
|
18 |
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
19 |
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
20 |
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
21 |
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
22 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
23 |
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
24 |
# SUCH DAMAGE.
|
25 |
#
|
26 |
# The views and conclusions contained in the software and documentation are
|
27 |
# those of the authors and should not be interpreted as representing official
|
28 |
# policies, either expressed or implied, of GRNET S.A.
|
29 |
#
|
30 |
"""Reconciliation management command
|
31 |
|
32 |
Management command to reconcile the contents of the Synnefo DB with
|
33 |
the state of the Ganeti backend. See docstring on top of
|
34 |
logic/reconciliation.py for a description of reconciliation rules.
|
35 |
|
36 |
"""
|
37 |
import datetime |
38 |
import bitarray |
39 |
|
40 |
from optparse import make_option |
41 |
|
42 |
from synnefo.settings import PUBLIC_ROUTED_USE_POOL |
43 |
from django.core.management.base import BaseCommand |
44 |
from django.db import transaction |
45 |
|
46 |
from synnefo.db.models import Backend, Network, BackendNetwork |
47 |
from synnefo.logic import reconciliation, backend, utils |
48 |
|
49 |
|
50 |
class Command(BaseCommand): |
51 |
help = 'Reconcile contents of Synnefo DB with state of Ganeti backend'
|
52 |
can_import_settings = True
|
53 |
output_transaction = True # The management command runs inside |
54 |
# an SQL transaction
|
55 |
option_list = BaseCommand.option_list + ( |
56 |
make_option('--fix-all', action='store_true', |
57 |
dest='fix', default=False, |
58 |
help='Fix all issues.'),
|
59 |
make_option('--conflicting-ips', action='store_true', |
60 |
dest='conflicting_ips', default=False, |
61 |
help='Detect conflicting ips')
|
62 |
) |
63 |
|
64 |
def handle(self, **options): |
65 |
self.verbosity = int(options['verbosity']) |
66 |
fix = options['fix']
|
67 |
conflicting_ips = options['conflicting_ips']
|
68 |
reconcile_networks(self.stdout, fix, conflicting_ips)
|
69 |
|
70 |
|
71 |
def reconcile_networks(out, fix, conflicting_ips): |
72 |
# Get models from DB
|
73 |
backends = Backend.objects.exclude(offline=True)
|
74 |
networks = Network.objects.filter(deleted=False)
|
75 |
|
76 |
# Get info from all ganeti backends
|
77 |
ganeti_networks = {} |
78 |
ganeti_hanging_networks = {} |
79 |
for b in backends: |
80 |
g_nets = reconciliation.get_networks_from_ganeti(b) |
81 |
ganeti_networks[b] = g_nets |
82 |
g_hanging_nets = reconciliation.hanging_networks(b, g_nets) |
83 |
ganeti_hanging_networks[b] = g_hanging_nets |
84 |
|
85 |
# Perform reconciliation for each network
|
86 |
for network in networks: |
87 |
net_id = network.id |
88 |
destroying = network.action == 'DESTROY'
|
89 |
uses_pool = not (network.type == 'PUBLIC_ROUTED' and (not |
90 |
PUBLIC_ROUTED_USE_POOL)) |
91 |
ip_address_maps = [] |
92 |
|
93 |
# Perform reconcilliation for each backend
|
94 |
for b in backends: |
95 |
info = (net_id, b.clustername) |
96 |
back_network = None
|
97 |
|
98 |
try:
|
99 |
# Get the model describing the network to this backend
|
100 |
back_network = BackendNetwork.objects.get(network=network, |
101 |
backend=b) |
102 |
except BackendNetwork.DoesNotExist:
|
103 |
out.write('D: No DB entry for network %d in backend %s\n' % info)
|
104 |
if fix:
|
105 |
out.write('F: Created entry in DB\n')
|
106 |
back_network = \ |
107 |
BackendNetwork.objects.create(network=network, |
108 |
backend=b) |
109 |
|
110 |
try:
|
111 |
# Get the info from backend
|
112 |
ganeti_networks[b][net_id] |
113 |
except KeyError: |
114 |
# Stale network does not exist in backend
|
115 |
if destroying:
|
116 |
out.write('D: Stale network %d in backend %s\n' % info)
|
117 |
if fix:
|
118 |
out.write("F: Issued OP_NETWORK_REMOVE'\n")
|
119 |
etime = datetime.datetime.now() |
120 |
backend.process_network_status(back_network, etime, |
121 |
0, 'OP_NETWORK_REMOVE', 'success', |
122 |
'Reconciliation simulated event.')
|
123 |
continue
|
124 |
else:
|
125 |
# Pending network
|
126 |
out.write('D: Pending network %d in backend %s\n' % info)
|
127 |
if fix:
|
128 |
out.write('F: Creating network in backend.\n')
|
129 |
backend.create_network(network, [b]) |
130 |
# Skip rest reconciliation as the network is just
|
131 |
# being created
|
132 |
continue
|
133 |
|
134 |
try:
|
135 |
hanging_groups = ganeti_hanging_networks[b][net_id] |
136 |
except KeyError: |
137 |
# Network is connected to all nodegroups
|
138 |
hanging_groups = [] |
139 |
|
140 |
if hanging_groups and not destroying: |
141 |
# Hanging network = not connected to all nodegroups of backend
|
142 |
out.write('D: Network %d in backend %s is not connected to '
|
143 |
'the following groups:\n' % info)
|
144 |
out.write('- ' + '\n- '.join(hanging_groups) + '\n') |
145 |
if fix:
|
146 |
for group in hanging_groups: |
147 |
out.write('F: Connecting network %d to nodegroup %s\n'
|
148 |
% (net_id, group)) |
149 |
backend.connect_network_group(b, network, group) |
150 |
elif back_network and back_network.operstate != 'ACTIVE': |
151 |
# Network is active
|
152 |
out.write('D: Unsynced network %d in backend %s\n' % info)
|
153 |
if fix:
|
154 |
out.write("F: Issued OP_NETWORK_CONNECT\n")
|
155 |
etime = datetime.datetime.now() |
156 |
backend.process_network_status(back_network, etime, |
157 |
0, 'OP_NETWORK_CONNECT', 'success', |
158 |
'Reconciliation simulated event.')
|
159 |
|
160 |
if uses_pool:
|
161 |
# Reconcile IP Pools
|
162 |
ip_map = ganeti_networks[b][net_id]['map']
|
163 |
ip_address_maps.append(bitarray_from_o1(ip_map)) |
164 |
|
165 |
if ip_address_maps and uses_pool: |
166 |
network_bitarray = reduce(lambda x, y: x | y, ip_address_maps) |
167 |
if not network.pool.reservations == network_bitarray: |
168 |
out.write('D: Unsynced pool of network %d\n' % net_id)
|
169 |
out.write('\t DB:\t%s\n' % network.pool.reservations.to01())
|
170 |
out.write('\t Ganeti:%s\n' % network_bitarray.to01())
|
171 |
if fix:
|
172 |
update_network_reservations(network, network_bitarray) |
173 |
out.write('F: Synchronized network pools\n')
|
174 |
|
175 |
# Detect conflicting IPs: Detect NIC's that have the same IP
|
176 |
# in the same network.
|
177 |
if conflicting_ips:
|
178 |
machine_ips = network.nics.all().values_list('ipv4', 'machine') |
179 |
ips = map(lambda x: x[0], machine_ips) |
180 |
distinct_ips = set(ips)
|
181 |
if len(distinct_ips) < len(ips): |
182 |
out.write('D: Conflicting IP in network %s.\n' % net_id)
|
183 |
conflicts = ips |
184 |
for i in distinct_ips: |
185 |
conflicts.remove(i) |
186 |
for i in conflicts: |
187 |
machines = [utils.id_to_instance_name(x[1]) \
|
188 |
for x in machine_ips if x[0] == i] |
189 |
out.write('\tIP:%s Machines: %s\n' %
|
190 |
(i, ', '.join(machines)))
|
191 |
if fix:
|
192 |
out.write('F: Can not fix it. Manually resolve the'
|
193 |
' conflict.\n')
|
194 |
|
195 |
# Detect Orphan Networks in Ganeti
|
196 |
db_network_ids = set([net.id for net in networks]) |
197 |
for back_end, ganeti_networks in ganeti_networks.items(): |
198 |
ganeti_network_ids = set(ganeti_networks.keys())
|
199 |
orphans = ganeti_network_ids - db_network_ids |
200 |
|
201 |
if len(orphans) > 0: |
202 |
out.write('D: Orphan Networks in backend %s:\n' % back_end.clustername)
|
203 |
out.write('- ' + '\n- '.join([str(o) for o in orphans]) + '\n') |
204 |
client = back_end.client |
205 |
if fix:
|
206 |
#XXX:Move this to backend
|
207 |
for id in orphans: |
208 |
out.write('Disconnecting and deleting network %d\n' % id) |
209 |
network = utils.id_to_network_name(id)
|
210 |
for group in client.GetGroups(): |
211 |
client.DisconnectNetwork(network, group) |
212 |
client.DeleteNetwork(network) |
213 |
|
214 |
|
215 |
def bitarray_from_o1(bitmap): |
216 |
return bitarray.bitarray(bitmap.replace("X", "1").replace(".", "0")) |
217 |
|
218 |
|
219 |
@transaction.commit_on_success
|
220 |
def update_network_reservations(network, reservations): |
221 |
network.pool.reservations = reservations |
222 |
network.pool.save() |