Statistics
| Branch: | Tag: | Revision:

root / snf-cyclades-app / synnefo / logic / management / commands / reconcile-networks.py @ ec22dfc4

History | View | Annotate | Download (11.1 kB)

1
# Copyright 2011-2012 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or without
4
# modification, are permitted provided that the following conditions
5
# are met:
6
#
7
#   1. Redistributions of source code must retain the above copyright
8
#      notice, this list of conditions and the following disclaimer.
9
#
10
#  2. Redistributions in binary form must reproduce the above copyright
11
#     notice, this list of conditions and the following disclaimer in the
12
#     documentation and/or other materials provided with the distribution.
13
#
14
# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
15
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17
# ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
18
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24
# SUCH DAMAGE.
25
#
26
# The views and conclusions contained in the software and documentation are
27
# those of the authors and should not be interpreted as representing official
28
# policies, either expressed or implied, of GRNET S.A.
29
#
30
"""Reconciliation management command
31

32
Management command to reconcile the contents of the Synnefo DB with
33
the state of the Ganeti backend. See docstring on top of
34
logic/reconciliation.py for a description of reconciliation rules.
35

36
"""
37
import datetime
38
import bitarray
39

    
40
from optparse import make_option
41

    
42
from synnefo.settings import PUBLIC_ROUTED_USE_POOL
43
from django.core.management.base import BaseCommand
44
from django.db import transaction
45

    
46
from synnefo.db.models import Backend, Network, BackendNetwork
47
from synnefo.db.pools import IPPool
48
from synnefo.logic import reconciliation, backend, utils
49

    
50

    
51
class Command(BaseCommand):
52
    help = 'Reconcile contents of Synnefo DB with state of Ganeti backend'
53
    can_import_settings = True
54
    output_transaction = True  # The management command runs inside
55
                               # an SQL transaction
56
    option_list = BaseCommand.option_list + (
57
        make_option('--fix-all', action='store_true',
58
                    dest='fix', default=False,
59
                    help='Fix all issues.'),
60
        make_option('--conflicting-ips', action='store_true',
61
                    dest='conflicting_ips', default=False,
62
                    help='Detect conflicting ips')
63
        )
64

    
65
    def handle(self, **options):
66
        self.verbosity = int(options['verbosity'])
67
        fix = options['fix']
68
        conflicting_ips = options['conflicting_ips']
69
        reconcile_networks(self.stdout, fix, conflicting_ips)
70

    
71

    
72
def reconcile_networks(out, fix, conflicting_ips):
73
    # Get models from DB
74
    backends = Backend.objects.exclude(offline=True)
75
    networks = Network.objects.filter(deleted=False)
76

    
77
    # Get info from all ganeti backends
78
    ganeti_networks = {}
79
    ganeti_hanging_networks = {}
80
    for b in backends:
81
        g_nets = reconciliation.get_networks_from_ganeti(b)
82
        ganeti_networks[b] = g_nets
83
        g_hanging_nets = reconciliation.hanging_networks(b, g_nets)
84
        ganeti_hanging_networks[b] = g_hanging_nets
85

    
86
    # Perform reconciliation for each network
87
    for network in networks:
88
        net_id = network.id
89
        destroying = network.action == 'DESTROY'
90
        uses_pool = not (network.type == 'PUBLIC_ROUTED' and (not
91
                        PUBLIC_ROUTED_USE_POOL))
92
        ip_available_maps = []
93
        ip_reserved_maps = []
94

    
95
        # Perform reconcilliation for each backend
96
        for b in backends:
97
            if network.public and not \
98
                BackendNetwork.objects.filter(network=network,
99
                                              backend=b).exists():
100
                    continue
101

    
102
            info = (net_id, b.clustername)
103
            back_network = None
104

    
105
            try:
106
                # Get the model describing the network to this backend
107
                back_network = BackendNetwork.objects.get(network=network,
108
                                                          backend=b)
109
            except BackendNetwork.DoesNotExist:
110
                out.write('D: No DB entry for network %d in backend %s\n' % info)
111
                if fix:
112
                    out.write('F: Created entry in DB\n')
113
                    back_network = \
114
                        BackendNetwork.objects.create(network=network,
115
                                                      backend=b)
116

    
117
            try:
118
                # Get the info from backend
119
                ganeti_networks[b][net_id]
120
            except KeyError:
121
                # Stale network does not exist in backend
122
                if destroying:
123
                    out.write('D: Stale network %d in backend %s\n' % info)
124
                    if fix:
125
                        out.write("F: Issued OP_NETWORK_REMOVE'\n")
126
                        etime = datetime.datetime.now()
127
                        backend.process_network_status(back_network, etime,
128
                                            0, 'OP_NETWORK_REMOVE', 'success',
129
                                            'Reconciliation simulated event.')
130
                    continue
131
                else:
132
                    # Pending network
133
                    out.write('D: Pending network %d in backend %s\n' % info)
134
                    if fix:
135
                        out.write('F: Creating network in backend.\n')
136
                        backend.create_network(network, [b])
137
                        # Skip rest reconciliation as the network is just
138
                        # being created
139
                    continue
140

    
141
            try:
142
                hanging_groups = ganeti_hanging_networks[b][net_id]
143
            except KeyError:
144
                # Network is connected to all nodegroups
145
                hanging_groups = []
146

    
147
            if hanging_groups and not destroying:
148
                # Hanging network = not connected to all nodegroups of backend
149
                out.write('D: Network %d in backend %s is not connected to '
150
                          'the following groups:\n' % info)
151
                out.write('-  ' + '\n-  '.join(hanging_groups) + '\n')
152
                if fix:
153
                    for group in hanging_groups:
154
                        out.write('F: Connecting network %d to nodegroup %s\n'
155
                                  % (net_id, group))
156
                        backend.connect_network_group(b, network, group)
157
            elif back_network and back_network.operstate != 'ACTIVE':
158
                # Network is active
159
                out.write('D: Unsynced network %d in backend %s\n' % info)
160
                if fix:
161
                    out.write("F: Issued OP_NETWORK_CONNECT\n")
162
                    etime = datetime.datetime.now()
163
                    backend.process_network_status(back_network, etime,
164
                                        0, 'OP_NETWORK_CONNECT', 'success',
165
                                        'Reconciliation simulated event.')
166
                    network = Network.objects.get(id=network.id)
167

    
168
            if uses_pool:
169
                # Reconcile IP Pools
170
                gnet = ganeti_networks[b][net_id]
171
                converter = IPPool(Foo(gnet['network']))
172
                a_map = bitarray_from_map(gnet['map'])
173
                a_map.invert()
174
                reserved = gnet['external_reservations']
175
                r_map = a_map.copy()
176
                r_map.setall(True)
177
                for address in reserved.split(','):
178
                    index = converter.value_to_index(address)
179
                    a_map[index] = True
180
                    r_map[index] = False
181
                ip_available_maps.append(a_map)
182
                ip_reserved_maps.append(r_map)
183

    
184
        if uses_pool and (ip_available_maps or ip_reserved_maps):
185
            available_map = reduce(lambda x, y: x | y, ip_available_maps)
186
            reserved_map = reduce(lambda x, y: x | y, ip_reserved_maps)
187

    
188
            pool = network.get_pool()
189
            un_available = pool.available != available_map
190
            un_reserved = pool.reserved != reserved_map
191
            if un_available or un_reserved:
192
                out.write("Detected unsynchronized pool for network %r:\n" %
193
                          network.id)
194
                if un_available:
195
                    out.write("Available:\n\tDB: %r\n\tGB: %r\n" %
196
                             (pool.available.to01(), available_map.to01()))
197
                    if fix:
198
                        pool.available = available_map
199
                if un_reserved:
200
                    out.write("Reserved:\n\tDB: %r\n\tGB: %r\n" %
201
                             (pool.reserved.to01(), reserved_map.to01()))
202
                    if fix:
203
                        pool.reserved = reserved_map
204
                if fix:
205
                    out.write("Synchronized pools for network %r.\n" % network.id)
206
            pool.save()
207

    
208

    
209
        # Detect conflicting IPs: Detect NIC's that have the same IP
210
        # in the same network.
211
        if conflicting_ips:
212
            machine_ips = network.nics.all().values_list('ipv4', 'machine')
213
            ips = map(lambda x: x[0], machine_ips)
214
            distinct_ips = set(ips)
215
            if len(distinct_ips) < len(ips):
216
                out.write('D: Conflicting IP in network %s.\n' % net_id)
217
                conflicts = ips
218
                for i in distinct_ips:
219
                    conflicts.remove(i)
220
                for i in conflicts:
221
                    machines = [utils.id_to_instance_name(x[1]) \
222
                                for x in machine_ips if x[0] == i]
223
                    out.write('\tIP:%s Machines: %s\n' %
224
                              (i, ', '.join(machines)))
225
                if fix:
226
                    out.write('F: Can not fix it. Manually resolve the'
227
                              ' conflict.\n')
228

    
229
    # Detect Orphan Networks in Ganeti
230
    db_network_ids = set([net.id for net in networks])
231
    for back_end, ganeti_networks in ganeti_networks.items():
232
        ganeti_network_ids = set(ganeti_networks.keys())
233
        orphans = ganeti_network_ids - db_network_ids
234

    
235
        if len(orphans) > 0:
236
            out.write('D: Orphan Networks in backend %s:\n' % back_end.clustername)
237
            out.write('-  ' + '\n-  '.join([str(o) for o in orphans]) + '\n')
238
            if fix:
239
                for net_id in orphans:
240
                    out.write('Disconnecting and deleting network %d\n' % net_id)
241
                    network = Network.objects.get(id=net_id)
242
                    backend.delete_network(network, backends=[back_end])
243

    
244

    
245
def bitarray_from_map(bitmap):
246
    return bitarray.bitarray(bitmap.replace("X", "1").replace(".", "0"))
247

    
248

    
249

    
250
class Foo():
251
    def __init__(self, subnet):
252
        self.available_map = ''
253
        self.reserved_map = ''
254
        self.size = 0
255
        self.network = Foo.Foo1(subnet)
256

    
257
    class Foo1():
258
        def __init__(self, subnet):
259
            self.subnet = subnet
260
            self.gateway = None