root / snf-cyclades-app / synnefo / logic / management / commands / reconcile-networks.py @ 3308a83f
History | View | Annotate | Download (11.2 kB)
1 | 0e9a423f | Christos Stavrakakis | # Copyright 2011-2012 GRNET S.A. All rights reserved.
|
---|---|---|---|
2 | 0e9a423f | Christos Stavrakakis | #
|
3 | 0e9a423f | Christos Stavrakakis | # Redistribution and use in source and binary forms, with or without
|
4 | 0e9a423f | Christos Stavrakakis | # modification, are permitted provided that the following conditions
|
5 | 0e9a423f | Christos Stavrakakis | # are met:
|
6 | 0e9a423f | Christos Stavrakakis | #
|
7 | 0e9a423f | Christos Stavrakakis | # 1. Redistributions of source code must retain the above copyright
|
8 | 0e9a423f | Christos Stavrakakis | # notice, this list of conditions and the following disclaimer.
|
9 | 0e9a423f | Christos Stavrakakis | #
|
10 | 0e9a423f | Christos Stavrakakis | # 2. Redistributions in binary form must reproduce the above copyright
|
11 | 0e9a423f | Christos Stavrakakis | # notice, this list of conditions and the following disclaimer in the
|
12 | 0e9a423f | Christos Stavrakakis | # documentation and/or other materials provided with the distribution.
|
13 | 0e9a423f | Christos Stavrakakis | #
|
14 | 0e9a423f | Christos Stavrakakis | # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
|
15 | 0e9a423f | Christos Stavrakakis | # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
16 | 0e9a423f | Christos Stavrakakis | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
17 | 0e9a423f | Christos Stavrakakis | # ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
|
18 | 0e9a423f | Christos Stavrakakis | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
19 | 0e9a423f | Christos Stavrakakis | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
20 | 0e9a423f | Christos Stavrakakis | # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
21 | 0e9a423f | Christos Stavrakakis | # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
22 | 0e9a423f | Christos Stavrakakis | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
23 | 0e9a423f | Christos Stavrakakis | # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
24 | 0e9a423f | Christos Stavrakakis | # SUCH DAMAGE.
|
25 | 0e9a423f | Christos Stavrakakis | #
|
26 | 0e9a423f | Christos Stavrakakis | # The views and conclusions contained in the software and documentation are
|
27 | 0e9a423f | Christos Stavrakakis | # those of the authors and should not be interpreted as representing official
|
28 | 0e9a423f | Christos Stavrakakis | # policies, either expressed or implied, of GRNET S.A.
|
29 | 0e9a423f | Christos Stavrakakis | #
|
30 | 0e9a423f | Christos Stavrakakis | """Reconciliation management command
|
31 | 0e9a423f | Christos Stavrakakis |
|
32 | 0e9a423f | Christos Stavrakakis | Management command to reconcile the contents of the Synnefo DB with
|
33 | 0e9a423f | Christos Stavrakakis | the state of the Ganeti backend. See docstring on top of
|
34 | 0e9a423f | Christos Stavrakakis | logic/reconciliation.py for a description of reconciliation rules.
|
35 | 0e9a423f | Christos Stavrakakis |
|
36 | 0e9a423f | Christos Stavrakakis | """
|
37 | 0e9a423f | Christos Stavrakakis | import datetime |
38 | c55b0cdc | Christos Stavrakakis | import bitarray |
39 | 0e9a423f | Christos Stavrakakis | |
40 | 0e9a423f | Christos Stavrakakis | from optparse import make_option |
41 | 0e9a423f | Christos Stavrakakis | |
42 | ad297723 | Christos Stavrakakis | from synnefo.settings import PUBLIC_ROUTED_USE_POOL |
43 | c55b0cdc | Christos Stavrakakis | from django.core.management.base import BaseCommand |
44 | c55b0cdc | Christos Stavrakakis | from django.db import transaction |
45 | 0e9a423f | Christos Stavrakakis | |
46 | 0e9a423f | Christos Stavrakakis | from synnefo.db.models import Backend, Network, BackendNetwork |
47 | fdc94944 | Christos Stavrakakis | from synnefo.db.pools import IPPool |
48 | d30f29aa | Christos Stavrakakis | from synnefo.logic import reconciliation, backend, utils |
49 | 0e9a423f | Christos Stavrakakis | |
50 | 0e9a423f | Christos Stavrakakis | |
51 | 0e9a423f | Christos Stavrakakis | class Command(BaseCommand): |
52 | 0e9a423f | Christos Stavrakakis | help = 'Reconcile contents of Synnefo DB with state of Ganeti backend'
|
53 | ad297723 | Christos Stavrakakis | can_import_settings = True
|
54 | 0e9a423f | Christos Stavrakakis | output_transaction = True # The management command runs inside |
55 | 0e9a423f | Christos Stavrakakis | # an SQL transaction
|
56 | 0e9a423f | Christos Stavrakakis | option_list = BaseCommand.option_list + ( |
57 | 0e9a423f | Christos Stavrakakis | make_option('--fix-all', action='store_true', |
58 | 0e9a423f | Christos Stavrakakis | dest='fix', default=False, |
59 | 0e9a423f | Christos Stavrakakis | help='Fix all issues.'),
|
60 | c55b0cdc | Christos Stavrakakis | make_option('--conflicting-ips', action='store_true', |
61 | c55b0cdc | Christos Stavrakakis | dest='conflicting_ips', default=False, |
62 | c55b0cdc | Christos Stavrakakis | help='Detect conflicting ips')
|
63 | 0e9a423f | Christos Stavrakakis | ) |
64 | 0e9a423f | Christos Stavrakakis | |
65 | 0e9a423f | Christos Stavrakakis | def handle(self, **options): |
66 | 0e9a423f | Christos Stavrakakis | self.verbosity = int(options['verbosity']) |
67 | 0e9a423f | Christos Stavrakakis | fix = options['fix']
|
68 | c55b0cdc | Christos Stavrakakis | conflicting_ips = options['conflicting_ips']
|
69 | c55b0cdc | Christos Stavrakakis | reconcile_networks(self.stdout, fix, conflicting_ips)
|
70 | 0e9a423f | Christos Stavrakakis | |
71 | 0e9a423f | Christos Stavrakakis | |
72 | c55b0cdc | Christos Stavrakakis | def reconcile_networks(out, fix, conflicting_ips): |
73 | 0e9a423f | Christos Stavrakakis | # Get models from DB
|
74 | 0e9a423f | Christos Stavrakakis | backends = Backend.objects.exclude(offline=True)
|
75 | 0e9a423f | Christos Stavrakakis | networks = Network.objects.filter(deleted=False)
|
76 | 0e9a423f | Christos Stavrakakis | |
77 | 0e9a423f | Christos Stavrakakis | # Get info from all ganeti backends
|
78 | 0e9a423f | Christos Stavrakakis | ganeti_networks = {} |
79 | 0e9a423f | Christos Stavrakakis | ganeti_hanging_networks = {} |
80 | 0e9a423f | Christos Stavrakakis | for b in backends: |
81 | 0e9a423f | Christos Stavrakakis | g_nets = reconciliation.get_networks_from_ganeti(b) |
82 | 0e9a423f | Christos Stavrakakis | ganeti_networks[b] = g_nets |
83 | 0e9a423f | Christos Stavrakakis | g_hanging_nets = reconciliation.hanging_networks(b, g_nets) |
84 | 0e9a423f | Christos Stavrakakis | ganeti_hanging_networks[b] = g_hanging_nets |
85 | 0e9a423f | Christos Stavrakakis | |
86 | 0e9a423f | Christos Stavrakakis | # Perform reconciliation for each network
|
87 | 0e9a423f | Christos Stavrakakis | for network in networks: |
88 | 0e9a423f | Christos Stavrakakis | net_id = network.id |
89 | 0e9a423f | Christos Stavrakakis | destroying = network.action == 'DESTROY'
|
90 | ad297723 | Christos Stavrakakis | uses_pool = not (network.type == 'PUBLIC_ROUTED' and (not |
91 | ad297723 | Christos Stavrakakis | PUBLIC_ROUTED_USE_POOL)) |
92 | fdc94944 | Christos Stavrakakis | ip_available_maps = [] |
93 | fdc94944 | Christos Stavrakakis | ip_reserved_maps = [] |
94 | 0e9a423f | Christos Stavrakakis | |
95 | 0e9a423f | Christos Stavrakakis | # Perform reconcilliation for each backend
|
96 | 0e9a423f | Christos Stavrakakis | for b in backends: |
97 | 7fede91e | Christos Stavrakakis | if network.public and not \ |
98 | 7fede91e | Christos Stavrakakis | BackendNetwork.objects.filter(network=network, |
99 | 7fede91e | Christos Stavrakakis | backend=b).exists(): |
100 | 7fede91e | Christos Stavrakakis | continue
|
101 | 7fede91e | Christos Stavrakakis | |
102 | 0e9a423f | Christos Stavrakakis | info = (net_id, b.clustername) |
103 | 0e9a423f | Christos Stavrakakis | back_network = None
|
104 | 0e9a423f | Christos Stavrakakis | |
105 | 0e9a423f | Christos Stavrakakis | try:
|
106 | 0e9a423f | Christos Stavrakakis | # Get the model describing the network to this backend
|
107 | 0e9a423f | Christos Stavrakakis | back_network = BackendNetwork.objects.get(network=network, |
108 | 0e9a423f | Christos Stavrakakis | backend=b) |
109 | 0e9a423f | Christos Stavrakakis | except BackendNetwork.DoesNotExist:
|
110 | 0e9a423f | Christos Stavrakakis | out.write('D: No DB entry for network %d in backend %s\n' % info)
|
111 | 0e9a423f | Christos Stavrakakis | if fix:
|
112 | 0e9a423f | Christos Stavrakakis | out.write('F: Created entry in DB\n')
|
113 | 0e9a423f | Christos Stavrakakis | back_network = \ |
114 | 0e9a423f | Christos Stavrakakis | BackendNetwork.objects.create(network=network, |
115 | 0e9a423f | Christos Stavrakakis | backend=b) |
116 | 3308a83f | Christos Stavrakakis | else:
|
117 | 3308a83f | Christos Stavrakakis | continue
|
118 | 0e9a423f | Christos Stavrakakis | |
119 | 0e9a423f | Christos Stavrakakis | try:
|
120 | 0e9a423f | Christos Stavrakakis | # Get the info from backend
|
121 | 0e9a423f | Christos Stavrakakis | ganeti_networks[b][net_id] |
122 | 0e9a423f | Christos Stavrakakis | except KeyError: |
123 | 0e9a423f | Christos Stavrakakis | # Stale network does not exist in backend
|
124 | 0e9a423f | Christos Stavrakakis | if destroying:
|
125 | 6883dcf3 | Christos Stavrakakis | if back_network.operstate != "DELETED": |
126 | 6883dcf3 | Christos Stavrakakis | out.write('D: Stale network %d in backend %s\n' % info)
|
127 | 6883dcf3 | Christos Stavrakakis | if fix:
|
128 | 6883dcf3 | Christos Stavrakakis | out.write("F: Issued OP_NETWORK_REMOVE'\n")
|
129 | 6883dcf3 | Christos Stavrakakis | etime = datetime.datetime.now() |
130 | 6883dcf3 | Christos Stavrakakis | backend.process_network_status(back_network, etime, |
131 | 6883dcf3 | Christos Stavrakakis | 0, 'OP_NETWORK_REMOVE', 'success', |
132 | 6883dcf3 | Christos Stavrakakis | 'Reconciliation simulated event.')
|
133 | 0e9a423f | Christos Stavrakakis | continue
|
134 | 0e9a423f | Christos Stavrakakis | else:
|
135 | 0e9a423f | Christos Stavrakakis | # Pending network
|
136 | 0e9a423f | Christos Stavrakakis | out.write('D: Pending network %d in backend %s\n' % info)
|
137 | 0e9a423f | Christos Stavrakakis | if fix:
|
138 | 0e9a423f | Christos Stavrakakis | out.write('F: Creating network in backend.\n')
|
139 | 0e9a423f | Christos Stavrakakis | backend.create_network(network, [b]) |
140 | 0e9a423f | Christos Stavrakakis | # Skip rest reconciliation as the network is just
|
141 | 0e9a423f | Christos Stavrakakis | # being created
|
142 | 0e9a423f | Christos Stavrakakis | continue
|
143 | 0e9a423f | Christos Stavrakakis | |
144 | 0e9a423f | Christos Stavrakakis | try:
|
145 | 0e9a423f | Christos Stavrakakis | hanging_groups = ganeti_hanging_networks[b][net_id] |
146 | 0e9a423f | Christos Stavrakakis | except KeyError: |
147 | 0e9a423f | Christos Stavrakakis | # Network is connected to all nodegroups
|
148 | 0e9a423f | Christos Stavrakakis | hanging_groups = [] |
149 | 0e9a423f | Christos Stavrakakis | |
150 | 0e9a423f | Christos Stavrakakis | if hanging_groups and not destroying: |
151 | 0e9a423f | Christos Stavrakakis | # Hanging network = not connected to all nodegroups of backend
|
152 | 0e9a423f | Christos Stavrakakis | out.write('D: Network %d in backend %s is not connected to '
|
153 | 0e9a423f | Christos Stavrakakis | 'the following groups:\n' % info)
|
154 | 0e9a423f | Christos Stavrakakis | out.write('- ' + '\n- '.join(hanging_groups) + '\n') |
155 | 0e9a423f | Christos Stavrakakis | if fix:
|
156 | 0e9a423f | Christos Stavrakakis | for group in hanging_groups: |
157 | 0e9a423f | Christos Stavrakakis | out.write('F: Connecting network %d to nodegroup %s\n'
|
158 | 0e9a423f | Christos Stavrakakis | % (net_id, group)) |
159 | 122c4019 | Christos Stavrakakis | backend.connect_network(network, b, group=group) |
160 | 0e9a423f | Christos Stavrakakis | elif back_network and back_network.operstate != 'ACTIVE': |
161 | 0e9a423f | Christos Stavrakakis | # Network is active
|
162 | 0e9a423f | Christos Stavrakakis | out.write('D: Unsynced network %d in backend %s\n' % info)
|
163 | 0e9a423f | Christos Stavrakakis | if fix:
|
164 | 0e9a423f | Christos Stavrakakis | out.write("F: Issued OP_NETWORK_CONNECT\n")
|
165 | 0e9a423f | Christos Stavrakakis | etime = datetime.datetime.now() |
166 | 0e9a423f | Christos Stavrakakis | backend.process_network_status(back_network, etime, |
167 | 0e9a423f | Christos Stavrakakis | 0, 'OP_NETWORK_CONNECT', 'success', |
168 | 0e9a423f | Christos Stavrakakis | 'Reconciliation simulated event.')
|
169 | ec22dfc4 | Christos Stavrakakis | network = Network.objects.get(id=network.id) |
170 | 0e9a423f | Christos Stavrakakis | |
171 | ad297723 | Christos Stavrakakis | if uses_pool:
|
172 | ad297723 | Christos Stavrakakis | # Reconcile IP Pools
|
173 | fdc94944 | Christos Stavrakakis | gnet = ganeti_networks[b][net_id] |
174 | fdc94944 | Christos Stavrakakis | converter = IPPool(Foo(gnet['network']))
|
175 | fdc94944 | Christos Stavrakakis | a_map = bitarray_from_map(gnet['map'])
|
176 | ec22dfc4 | Christos Stavrakakis | a_map.invert() |
177 | fdc94944 | Christos Stavrakakis | reserved = gnet['external_reservations']
|
178 | fdc94944 | Christos Stavrakakis | r_map = a_map.copy() |
179 | fdc94944 | Christos Stavrakakis | r_map.setall(True)
|
180 | fdc94944 | Christos Stavrakakis | for address in reserved.split(','): |
181 | fdc94944 | Christos Stavrakakis | index = converter.value_to_index(address) |
182 | fdc94944 | Christos Stavrakakis | a_map[index] = True
|
183 | fdc94944 | Christos Stavrakakis | r_map[index] = False
|
184 | fdc94944 | Christos Stavrakakis | ip_available_maps.append(a_map) |
185 | fdc94944 | Christos Stavrakakis | ip_reserved_maps.append(r_map) |
186 | fdc94944 | Christos Stavrakakis | |
187 | fdc94944 | Christos Stavrakakis | if uses_pool and (ip_available_maps or ip_reserved_maps): |
188 | 6883dcf3 | Christos Stavrakakis | available_map = reduce(lambda x, y: x & y, ip_available_maps) |
189 | 6883dcf3 | Christos Stavrakakis | reserved_map = reduce(lambda x, y: x & y, ip_reserved_maps) |
190 | fdc94944 | Christos Stavrakakis | |
191 | fdc94944 | Christos Stavrakakis | pool = network.get_pool() |
192 | fdc94944 | Christos Stavrakakis | un_available = pool.available != available_map |
193 | fdc94944 | Christos Stavrakakis | un_reserved = pool.reserved != reserved_map |
194 | fdc94944 | Christos Stavrakakis | if un_available or un_reserved: |
195 | fdc94944 | Christos Stavrakakis | out.write("Detected unsynchronized pool for network %r:\n" %
|
196 | fdc94944 | Christos Stavrakakis | network.id) |
197 | fdc94944 | Christos Stavrakakis | if un_available:
|
198 | fdc94944 | Christos Stavrakakis | out.write("Available:\n\tDB: %r\n\tGB: %r\n" %
|
199 | fdc94944 | Christos Stavrakakis | (pool.available.to01(), available_map.to01())) |
200 | fdc94944 | Christos Stavrakakis | if fix:
|
201 | fdc94944 | Christos Stavrakakis | pool.available = available_map |
202 | fdc94944 | Christos Stavrakakis | if un_reserved:
|
203 | fdc94944 | Christos Stavrakakis | out.write("Reserved:\n\tDB: %r\n\tGB: %r\n" %
|
204 | fdc94944 | Christos Stavrakakis | (pool.reserved.to01(), reserved_map.to01())) |
205 | fdc94944 | Christos Stavrakakis | if fix:
|
206 | fdc94944 | Christos Stavrakakis | pool.reserved = reserved_map |
207 | d5b7fd7f | Christos Stavrakakis | if fix:
|
208 | fdc94944 | Christos Stavrakakis | out.write("Synchronized pools for network %r.\n" % network.id)
|
209 | fdc94944 | Christos Stavrakakis | pool.save() |
210 | c55b0cdc | Christos Stavrakakis | |
211 | ec22dfc4 | Christos Stavrakakis | |
212 | c55b0cdc | Christos Stavrakakis | # Detect conflicting IPs: Detect NIC's that have the same IP
|
213 | c55b0cdc | Christos Stavrakakis | # in the same network.
|
214 | c55b0cdc | Christos Stavrakakis | if conflicting_ips:
|
215 | c55b0cdc | Christos Stavrakakis | machine_ips = network.nics.all().values_list('ipv4', 'machine') |
216 | c55b0cdc | Christos Stavrakakis | ips = map(lambda x: x[0], machine_ips) |
217 | c55b0cdc | Christos Stavrakakis | distinct_ips = set(ips)
|
218 | c55b0cdc | Christos Stavrakakis | if len(distinct_ips) < len(ips): |
219 | c55b0cdc | Christos Stavrakakis | out.write('D: Conflicting IP in network %s.\n' % net_id)
|
220 | c55b0cdc | Christos Stavrakakis | conflicts = ips |
221 | c55b0cdc | Christos Stavrakakis | for i in distinct_ips: |
222 | c55b0cdc | Christos Stavrakakis | conflicts.remove(i) |
223 | c55b0cdc | Christos Stavrakakis | for i in conflicts: |
224 | c55b0cdc | Christos Stavrakakis | machines = [utils.id_to_instance_name(x[1]) \
|
225 | c55b0cdc | Christos Stavrakakis | for x in machine_ips if x[0] == i] |
226 | c55b0cdc | Christos Stavrakakis | out.write('\tIP:%s Machines: %s\n' %
|
227 | c55b0cdc | Christos Stavrakakis | (i, ', '.join(machines)))
|
228 | c55b0cdc | Christos Stavrakakis | if fix:
|
229 | c55b0cdc | Christos Stavrakakis | out.write('F: Can not fix it. Manually resolve the'
|
230 | c55b0cdc | Christos Stavrakakis | ' conflict.\n')
|
231 | c55b0cdc | Christos Stavrakakis | |
232 | 0e9a423f | Christos Stavrakakis | # Detect Orphan Networks in Ganeti
|
233 | 0e9a423f | Christos Stavrakakis | db_network_ids = set([net.id for net in networks]) |
234 | 0e9a423f | Christos Stavrakakis | for back_end, ganeti_networks in ganeti_networks.items(): |
235 | 0e9a423f | Christos Stavrakakis | ganeti_network_ids = set(ganeti_networks.keys())
|
236 | 0e9a423f | Christos Stavrakakis | orphans = ganeti_network_ids - db_network_ids |
237 | 0e9a423f | Christos Stavrakakis | |
238 | 0e9a423f | Christos Stavrakakis | if len(orphans) > 0: |
239 | 0e9a423f | Christos Stavrakakis | out.write('D: Orphan Networks in backend %s:\n' % back_end.clustername)
|
240 | 0e9a423f | Christos Stavrakakis | out.write('- ' + '\n- '.join([str(o) for o in orphans]) + '\n') |
241 | 0e9a423f | Christos Stavrakakis | if fix:
|
242 | fdc94944 | Christos Stavrakakis | for net_id in orphans: |
243 | fdc94944 | Christos Stavrakakis | out.write('Disconnecting and deleting network %d\n' % net_id)
|
244 | fdc94944 | Christos Stavrakakis | network = Network.objects.get(id=net_id) |
245 | fdc94944 | Christos Stavrakakis | backend.delete_network(network, backends=[back_end]) |
246 | c55b0cdc | Christos Stavrakakis | |
247 | c55b0cdc | Christos Stavrakakis | |
248 | fdc94944 | Christos Stavrakakis | def bitarray_from_map(bitmap): |
249 | c55b0cdc | Christos Stavrakakis | return bitarray.bitarray(bitmap.replace("X", "1").replace(".", "0")) |
250 | c55b0cdc | Christos Stavrakakis | |
251 | c55b0cdc | Christos Stavrakakis | |
252 | fdc94944 | Christos Stavrakakis | |
253 | fdc94944 | Christos Stavrakakis | class Foo(): |
254 | fdc94944 | Christos Stavrakakis | def __init__(self, subnet): |
255 | fdc94944 | Christos Stavrakakis | self.available_map = '' |
256 | fdc94944 | Christos Stavrakakis | self.reserved_map = '' |
257 | fdc94944 | Christos Stavrakakis | self.size = 0 |
258 | fdc94944 | Christos Stavrakakis | self.network = Foo.Foo1(subnet)
|
259 | fdc94944 | Christos Stavrakakis | |
260 | fdc94944 | Christos Stavrakakis | class Foo1(): |
261 | fdc94944 | Christos Stavrakakis | def __init__(self, subnet): |
262 | fdc94944 | Christos Stavrakakis | self.subnet = subnet
|
263 | fdc94944 | Christos Stavrakakis | self.gateway = None |