root / snf-cyclades-app / synnefo / logic / management / commands / reconcile-networks.py @ d7841399
History | View | Annotate | Download (11.1 kB)
1 | 0e9a423f | Christos Stavrakakis | # Copyright 2011-2012 GRNET S.A. All rights reserved.
|
---|---|---|---|
2 | 0e9a423f | Christos Stavrakakis | #
|
3 | 0e9a423f | Christos Stavrakakis | # Redistribution and use in source and binary forms, with or without
|
4 | 0e9a423f | Christos Stavrakakis | # modification, are permitted provided that the following conditions
|
5 | 0e9a423f | Christos Stavrakakis | # are met:
|
6 | 0e9a423f | Christos Stavrakakis | #
|
7 | 0e9a423f | Christos Stavrakakis | # 1. Redistributions of source code must retain the above copyright
|
8 | 0e9a423f | Christos Stavrakakis | # notice, this list of conditions and the following disclaimer.
|
9 | 0e9a423f | Christos Stavrakakis | #
|
10 | 0e9a423f | Christos Stavrakakis | # 2. Redistributions in binary form must reproduce the above copyright
|
11 | 0e9a423f | Christos Stavrakakis | # notice, this list of conditions and the following disclaimer in the
|
12 | 0e9a423f | Christos Stavrakakis | # documentation and/or other materials provided with the distribution.
|
13 | 0e9a423f | Christos Stavrakakis | #
|
14 | 0e9a423f | Christos Stavrakakis | # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
|
15 | 0e9a423f | Christos Stavrakakis | # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
16 | 0e9a423f | Christos Stavrakakis | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
17 | 0e9a423f | Christos Stavrakakis | # ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
|
18 | 0e9a423f | Christos Stavrakakis | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
19 | 0e9a423f | Christos Stavrakakis | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
20 | 0e9a423f | Christos Stavrakakis | # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
21 | 0e9a423f | Christos Stavrakakis | # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
22 | 0e9a423f | Christos Stavrakakis | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
23 | 0e9a423f | Christos Stavrakakis | # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
24 | 0e9a423f | Christos Stavrakakis | # SUCH DAMAGE.
|
25 | 0e9a423f | Christos Stavrakakis | #
|
26 | 0e9a423f | Christos Stavrakakis | # The views and conclusions contained in the software and documentation are
|
27 | 0e9a423f | Christos Stavrakakis | # those of the authors and should not be interpreted as representing official
|
28 | 0e9a423f | Christos Stavrakakis | # policies, either expressed or implied, of GRNET S.A.
|
29 | 0e9a423f | Christos Stavrakakis | #
|
30 | 0e9a423f | Christos Stavrakakis | """Reconciliation management command
|
31 | 0e9a423f | Christos Stavrakakis |
|
32 | 0e9a423f | Christos Stavrakakis | Management command to reconcile the contents of the Synnefo DB with
|
33 | 0e9a423f | Christos Stavrakakis | the state of the Ganeti backend. See docstring on top of
|
34 | 0e9a423f | Christos Stavrakakis | logic/reconciliation.py for a description of reconciliation rules.
|
35 | 0e9a423f | Christos Stavrakakis |
|
36 | 0e9a423f | Christos Stavrakakis | """
|
37 | 0e9a423f | Christos Stavrakakis | import datetime |
38 | c55b0cdc | Christos Stavrakakis | import bitarray |
39 | 0e9a423f | Christos Stavrakakis | |
40 | 0e9a423f | Christos Stavrakakis | from optparse import make_option |
41 | 0e9a423f | Christos Stavrakakis | |
42 | d1b1e683 | Dimitris Aragiorgis | from synnefo.settings import PUBLIC_USE_POOL |
43 | c55b0cdc | Christos Stavrakakis | from django.core.management.base import BaseCommand |
44 | c55b0cdc | Christos Stavrakakis | from django.db import transaction |
45 | 0e9a423f | Christos Stavrakakis | |
46 | 0e9a423f | Christos Stavrakakis | from synnefo.db.models import Backend, Network, BackendNetwork |
47 | fdc94944 | Christos Stavrakakis | from synnefo.db.pools import IPPool |
48 | d30f29aa | Christos Stavrakakis | from synnefo.logic import reconciliation, backend, utils |
49 | 0e9a423f | Christos Stavrakakis | |
50 | 0e9a423f | Christos Stavrakakis | |
51 | 0e9a423f | Christos Stavrakakis | class Command(BaseCommand): |
52 | 0e9a423f | Christos Stavrakakis | help = 'Reconcile contents of Synnefo DB with state of Ganeti backend'
|
53 | ad297723 | Christos Stavrakakis | can_import_settings = True
|
54 | 0e9a423f | Christos Stavrakakis | output_transaction = True # The management command runs inside |
55 | 0e9a423f | Christos Stavrakakis | # an SQL transaction
|
56 | 0e9a423f | Christos Stavrakakis | option_list = BaseCommand.option_list + ( |
57 | 0e9a423f | Christos Stavrakakis | make_option('--fix-all', action='store_true', |
58 | 0e9a423f | Christos Stavrakakis | dest='fix', default=False, |
59 | 0e9a423f | Christos Stavrakakis | help='Fix all issues.'),
|
60 | c55b0cdc | Christos Stavrakakis | make_option('--conflicting-ips', action='store_true', |
61 | c55b0cdc | Christos Stavrakakis | dest='conflicting_ips', default=False, |
62 | c55b0cdc | Christos Stavrakakis | help='Detect conflicting ips')
|
63 | 0e9a423f | Christos Stavrakakis | ) |
64 | 0e9a423f | Christos Stavrakakis | |
65 | 0e9a423f | Christos Stavrakakis | def handle(self, **options): |
66 | 0e9a423f | Christos Stavrakakis | self.verbosity = int(options['verbosity']) |
67 | 0e9a423f | Christos Stavrakakis | fix = options['fix']
|
68 | c55b0cdc | Christos Stavrakakis | conflicting_ips = options['conflicting_ips']
|
69 | c55b0cdc | Christos Stavrakakis | reconcile_networks(self.stdout, fix, conflicting_ips)
|
70 | 0e9a423f | Christos Stavrakakis | |
71 | 0e9a423f | Christos Stavrakakis | |
72 | c55b0cdc | Christos Stavrakakis | def reconcile_networks(out, fix, conflicting_ips): |
73 | 0e9a423f | Christos Stavrakakis | # Get models from DB
|
74 | 0e9a423f | Christos Stavrakakis | backends = Backend.objects.exclude(offline=True)
|
75 | 0e9a423f | Christos Stavrakakis | networks = Network.objects.filter(deleted=False)
|
76 | 0e9a423f | Christos Stavrakakis | |
77 | 0e9a423f | Christos Stavrakakis | # Get info from all ganeti backends
|
78 | 0e9a423f | Christos Stavrakakis | ganeti_networks = {} |
79 | 0e9a423f | Christos Stavrakakis | ganeti_hanging_networks = {} |
80 | 0e9a423f | Christos Stavrakakis | for b in backends: |
81 | 0e9a423f | Christos Stavrakakis | g_nets = reconciliation.get_networks_from_ganeti(b) |
82 | 0e9a423f | Christos Stavrakakis | ganeti_networks[b] = g_nets |
83 | 0e9a423f | Christos Stavrakakis | g_hanging_nets = reconciliation.hanging_networks(b, g_nets) |
84 | 0e9a423f | Christos Stavrakakis | ganeti_hanging_networks[b] = g_hanging_nets |
85 | 0e9a423f | Christos Stavrakakis | |
86 | 0e9a423f | Christos Stavrakakis | # Perform reconciliation for each network
|
87 | 0e9a423f | Christos Stavrakakis | for network in networks: |
88 | 0e9a423f | Christos Stavrakakis | net_id = network.id |
89 | 0e9a423f | Christos Stavrakakis | destroying = network.action == 'DESTROY'
|
90 | d1b1e683 | Dimitris Aragiorgis | uses_pool = not network.public or PUBLIC_USE_POOL |
91 | fdc94944 | Christos Stavrakakis | ip_available_maps = [] |
92 | fdc94944 | Christos Stavrakakis | ip_reserved_maps = [] |
93 | 0e9a423f | Christos Stavrakakis | |
94 | 0e9a423f | Christos Stavrakakis | # Perform reconcilliation for each backend
|
95 | 0e9a423f | Christos Stavrakakis | for b in backends: |
96 | 7fede91e | Christos Stavrakakis | if network.public and not \ |
97 | 7fede91e | Christos Stavrakakis | BackendNetwork.objects.filter(network=network, |
98 | 7fede91e | Christos Stavrakakis | backend=b).exists(): |
99 | 7fede91e | Christos Stavrakakis | continue
|
100 | 7fede91e | Christos Stavrakakis | |
101 | 0e9a423f | Christos Stavrakakis | info = (net_id, b.clustername) |
102 | 0e9a423f | Christos Stavrakakis | back_network = None
|
103 | 0e9a423f | Christos Stavrakakis | |
104 | 0e9a423f | Christos Stavrakakis | try:
|
105 | 0e9a423f | Christos Stavrakakis | # Get the model describing the network to this backend
|
106 | 0e9a423f | Christos Stavrakakis | back_network = BackendNetwork.objects.get(network=network, |
107 | 0e9a423f | Christos Stavrakakis | backend=b) |
108 | 0e9a423f | Christos Stavrakakis | except BackendNetwork.DoesNotExist:
|
109 | 0e9a423f | Christos Stavrakakis | out.write('D: No DB entry for network %d in backend %s\n' % info)
|
110 | 0e9a423f | Christos Stavrakakis | if fix:
|
111 | 0e9a423f | Christos Stavrakakis | out.write('F: Created entry in DB\n')
|
112 | 0e9a423f | Christos Stavrakakis | back_network = \ |
113 | 0e9a423f | Christos Stavrakakis | BackendNetwork.objects.create(network=network, |
114 | 0e9a423f | Christos Stavrakakis | backend=b) |
115 | 3308a83f | Christos Stavrakakis | else:
|
116 | 3308a83f | Christos Stavrakakis | continue
|
117 | 0e9a423f | Christos Stavrakakis | |
118 | 0e9a423f | Christos Stavrakakis | try:
|
119 | 0e9a423f | Christos Stavrakakis | # Get the info from backend
|
120 | 0e9a423f | Christos Stavrakakis | ganeti_networks[b][net_id] |
121 | 0e9a423f | Christos Stavrakakis | except KeyError: |
122 | 0e9a423f | Christos Stavrakakis | # Stale network does not exist in backend
|
123 | 0e9a423f | Christos Stavrakakis | if destroying:
|
124 | 6883dcf3 | Christos Stavrakakis | if back_network.operstate != "DELETED": |
125 | 6883dcf3 | Christos Stavrakakis | out.write('D: Stale network %d in backend %s\n' % info)
|
126 | 6883dcf3 | Christos Stavrakakis | if fix:
|
127 | 6883dcf3 | Christos Stavrakakis | out.write("F: Issued OP_NETWORK_REMOVE'\n")
|
128 | 6883dcf3 | Christos Stavrakakis | etime = datetime.datetime.now() |
129 | 6883dcf3 | Christos Stavrakakis | backend.process_network_status(back_network, etime, |
130 | 6883dcf3 | Christos Stavrakakis | 0, 'OP_NETWORK_REMOVE', 'success', |
131 | 6883dcf3 | Christos Stavrakakis | 'Reconciliation simulated event.')
|
132 | 0e9a423f | Christos Stavrakakis | continue
|
133 | 0e9a423f | Christos Stavrakakis | else:
|
134 | 0e9a423f | Christos Stavrakakis | # Pending network
|
135 | 0e9a423f | Christos Stavrakakis | out.write('D: Pending network %d in backend %s\n' % info)
|
136 | 0e9a423f | Christos Stavrakakis | if fix:
|
137 | 0e9a423f | Christos Stavrakakis | out.write('F: Creating network in backend.\n')
|
138 | 0e9a423f | Christos Stavrakakis | backend.create_network(network, [b]) |
139 | 0e9a423f | Christos Stavrakakis | # Skip rest reconciliation as the network is just
|
140 | 0e9a423f | Christos Stavrakakis | # being created
|
141 | 0e9a423f | Christos Stavrakakis | continue
|
142 | 0e9a423f | Christos Stavrakakis | |
143 | 0e9a423f | Christos Stavrakakis | try:
|
144 | 0e9a423f | Christos Stavrakakis | hanging_groups = ganeti_hanging_networks[b][net_id] |
145 | 0e9a423f | Christos Stavrakakis | except KeyError: |
146 | 0e9a423f | Christos Stavrakakis | # Network is connected to all nodegroups
|
147 | 0e9a423f | Christos Stavrakakis | hanging_groups = [] |
148 | 0e9a423f | Christos Stavrakakis | |
149 | 0e9a423f | Christos Stavrakakis | if hanging_groups and not destroying: |
150 | 0e9a423f | Christos Stavrakakis | # Hanging network = not connected to all nodegroups of backend
|
151 | 0e9a423f | Christos Stavrakakis | out.write('D: Network %d in backend %s is not connected to '
|
152 | 0e9a423f | Christos Stavrakakis | 'the following groups:\n' % info)
|
153 | 0e9a423f | Christos Stavrakakis | out.write('- ' + '\n- '.join(hanging_groups) + '\n') |
154 | 0e9a423f | Christos Stavrakakis | if fix:
|
155 | 0e9a423f | Christos Stavrakakis | for group in hanging_groups: |
156 | 0e9a423f | Christos Stavrakakis | out.write('F: Connecting network %d to nodegroup %s\n'
|
157 | 0e9a423f | Christos Stavrakakis | % (net_id, group)) |
158 | 122c4019 | Christos Stavrakakis | backend.connect_network(network, b, group=group) |
159 | 0e9a423f | Christos Stavrakakis | elif back_network and back_network.operstate != 'ACTIVE': |
160 | 0e9a423f | Christos Stavrakakis | # Network is active
|
161 | 0e9a423f | Christos Stavrakakis | out.write('D: Unsynced network %d in backend %s\n' % info)
|
162 | 0e9a423f | Christos Stavrakakis | if fix:
|
163 | 0e9a423f | Christos Stavrakakis | out.write("F: Issued OP_NETWORK_CONNECT\n")
|
164 | 0e9a423f | Christos Stavrakakis | etime = datetime.datetime.now() |
165 | 0e9a423f | Christos Stavrakakis | backend.process_network_status(back_network, etime, |
166 | 0e9a423f | Christos Stavrakakis | 0, 'OP_NETWORK_CONNECT', 'success', |
167 | 0e9a423f | Christos Stavrakakis | 'Reconciliation simulated event.')
|
168 | ec22dfc4 | Christos Stavrakakis | network = Network.objects.get(id=network.id) |
169 | 0e9a423f | Christos Stavrakakis | |
170 | ad297723 | Christos Stavrakakis | if uses_pool:
|
171 | ad297723 | Christos Stavrakakis | # Reconcile IP Pools
|
172 | fdc94944 | Christos Stavrakakis | gnet = ganeti_networks[b][net_id] |
173 | fdc94944 | Christos Stavrakakis | converter = IPPool(Foo(gnet['network']))
|
174 | fdc94944 | Christos Stavrakakis | a_map = bitarray_from_map(gnet['map'])
|
175 | ec22dfc4 | Christos Stavrakakis | a_map.invert() |
176 | fdc94944 | Christos Stavrakakis | reserved = gnet['external_reservations']
|
177 | fdc94944 | Christos Stavrakakis | r_map = a_map.copy() |
178 | fdc94944 | Christos Stavrakakis | r_map.setall(True)
|
179 | fdc94944 | Christos Stavrakakis | for address in reserved.split(','): |
180 | fdc94944 | Christos Stavrakakis | index = converter.value_to_index(address) |
181 | fdc94944 | Christos Stavrakakis | a_map[index] = True
|
182 | fdc94944 | Christos Stavrakakis | r_map[index] = False
|
183 | fdc94944 | Christos Stavrakakis | ip_available_maps.append(a_map) |
184 | fdc94944 | Christos Stavrakakis | ip_reserved_maps.append(r_map) |
185 | fdc94944 | Christos Stavrakakis | |
186 | fdc94944 | Christos Stavrakakis | if uses_pool and (ip_available_maps or ip_reserved_maps): |
187 | 6883dcf3 | Christos Stavrakakis | available_map = reduce(lambda x, y: x & y, ip_available_maps) |
188 | 6883dcf3 | Christos Stavrakakis | reserved_map = reduce(lambda x, y: x & y, ip_reserved_maps) |
189 | fdc94944 | Christos Stavrakakis | |
190 | fdc94944 | Christos Stavrakakis | pool = network.get_pool() |
191 | fdc94944 | Christos Stavrakakis | un_available = pool.available != available_map |
192 | fdc94944 | Christos Stavrakakis | un_reserved = pool.reserved != reserved_map |
193 | fdc94944 | Christos Stavrakakis | if un_available or un_reserved: |
194 | fdc94944 | Christos Stavrakakis | out.write("Detected unsynchronized pool for network %r:\n" %
|
195 | fdc94944 | Christos Stavrakakis | network.id) |
196 | fdc94944 | Christos Stavrakakis | if un_available:
|
197 | fdc94944 | Christos Stavrakakis | out.write("Available:\n\tDB: %r\n\tGB: %r\n" %
|
198 | fdc94944 | Christos Stavrakakis | (pool.available.to01(), available_map.to01())) |
199 | fdc94944 | Christos Stavrakakis | if fix:
|
200 | fdc94944 | Christos Stavrakakis | pool.available = available_map |
201 | fdc94944 | Christos Stavrakakis | if un_reserved:
|
202 | fdc94944 | Christos Stavrakakis | out.write("Reserved:\n\tDB: %r\n\tGB: %r\n" %
|
203 | fdc94944 | Christos Stavrakakis | (pool.reserved.to01(), reserved_map.to01())) |
204 | fdc94944 | Christos Stavrakakis | if fix:
|
205 | fdc94944 | Christos Stavrakakis | pool.reserved = reserved_map |
206 | d5b7fd7f | Christos Stavrakakis | if fix:
|
207 | fdc94944 | Christos Stavrakakis | out.write("Synchronized pools for network %r.\n" % network.id)
|
208 | fdc94944 | Christos Stavrakakis | pool.save() |
209 | c55b0cdc | Christos Stavrakakis | |
210 | ec22dfc4 | Christos Stavrakakis | |
211 | c55b0cdc | Christos Stavrakakis | # Detect conflicting IPs: Detect NIC's that have the same IP
|
212 | c55b0cdc | Christos Stavrakakis | # in the same network.
|
213 | c55b0cdc | Christos Stavrakakis | if conflicting_ips:
|
214 | c55b0cdc | Christos Stavrakakis | machine_ips = network.nics.all().values_list('ipv4', 'machine') |
215 | c55b0cdc | Christos Stavrakakis | ips = map(lambda x: x[0], machine_ips) |
216 | c55b0cdc | Christos Stavrakakis | distinct_ips = set(ips)
|
217 | c55b0cdc | Christos Stavrakakis | if len(distinct_ips) < len(ips): |
218 | c55b0cdc | Christos Stavrakakis | out.write('D: Conflicting IP in network %s.\n' % net_id)
|
219 | c55b0cdc | Christos Stavrakakis | conflicts = ips |
220 | c55b0cdc | Christos Stavrakakis | for i in distinct_ips: |
221 | c55b0cdc | Christos Stavrakakis | conflicts.remove(i) |
222 | c55b0cdc | Christos Stavrakakis | for i in conflicts: |
223 | c55b0cdc | Christos Stavrakakis | machines = [utils.id_to_instance_name(x[1]) \
|
224 | c55b0cdc | Christos Stavrakakis | for x in machine_ips if x[0] == i] |
225 | c55b0cdc | Christos Stavrakakis | out.write('\tIP:%s Machines: %s\n' %
|
226 | c55b0cdc | Christos Stavrakakis | (i, ', '.join(machines)))
|
227 | c55b0cdc | Christos Stavrakakis | if fix:
|
228 | c55b0cdc | Christos Stavrakakis | out.write('F: Can not fix it. Manually resolve the'
|
229 | c55b0cdc | Christos Stavrakakis | ' conflict.\n')
|
230 | c55b0cdc | Christos Stavrakakis | |
231 | 0e9a423f | Christos Stavrakakis | # Detect Orphan Networks in Ganeti
|
232 | 0e9a423f | Christos Stavrakakis | db_network_ids = set([net.id for net in networks]) |
233 | 0e9a423f | Christos Stavrakakis | for back_end, ganeti_networks in ganeti_networks.items(): |
234 | 0e9a423f | Christos Stavrakakis | ganeti_network_ids = set(ganeti_networks.keys())
|
235 | 0e9a423f | Christos Stavrakakis | orphans = ganeti_network_ids - db_network_ids |
236 | 0e9a423f | Christos Stavrakakis | |
237 | 0e9a423f | Christos Stavrakakis | if len(orphans) > 0: |
238 | 0e9a423f | Christos Stavrakakis | out.write('D: Orphan Networks in backend %s:\n' % back_end.clustername)
|
239 | 0e9a423f | Christos Stavrakakis | out.write('- ' + '\n- '.join([str(o) for o in orphans]) + '\n') |
240 | 0e9a423f | Christos Stavrakakis | if fix:
|
241 | fdc94944 | Christos Stavrakakis | for net_id in orphans: |
242 | fdc94944 | Christos Stavrakakis | out.write('Disconnecting and deleting network %d\n' % net_id)
|
243 | fdc94944 | Christos Stavrakakis | network = Network.objects.get(id=net_id) |
244 | fdc94944 | Christos Stavrakakis | backend.delete_network(network, backends=[back_end]) |
245 | c55b0cdc | Christos Stavrakakis | |
246 | c55b0cdc | Christos Stavrakakis | |
247 | fdc94944 | Christos Stavrakakis | def bitarray_from_map(bitmap): |
248 | c55b0cdc | Christos Stavrakakis | return bitarray.bitarray(bitmap.replace("X", "1").replace(".", "0")) |
249 | c55b0cdc | Christos Stavrakakis | |
250 | c55b0cdc | Christos Stavrakakis | |
251 | fdc94944 | Christos Stavrakakis | |
252 | fdc94944 | Christos Stavrakakis | class Foo(): |
253 | fdc94944 | Christos Stavrakakis | def __init__(self, subnet): |
254 | fdc94944 | Christos Stavrakakis | self.available_map = '' |
255 | fdc94944 | Christos Stavrakakis | self.reserved_map = '' |
256 | fdc94944 | Christos Stavrakakis | self.size = 0 |
257 | fdc94944 | Christos Stavrakakis | self.network = Foo.Foo1(subnet)
|
258 | fdc94944 | Christos Stavrakakis | |
259 | fdc94944 | Christos Stavrakakis | class Foo1(): |
260 | fdc94944 | Christos Stavrakakis | def __init__(self, subnet): |
261 | fdc94944 | Christos Stavrakakis | self.subnet = subnet
|
262 | fdc94944 | Christos Stavrakakis | self.gateway = None |