Statistics
| Branch: | Tag: | Revision:

root / snf-cyclades-app / synnefo / logic / reconciliation.py @ 6e9255ab

History | View | Annotate | Download (27.2 kB)

1 9fea53cc Vangelis Koukis
# -*- coding: utf-8 -*-
2 9fea53cc Vangelis Koukis
#
3 75dc539e Christos Stavrakakis
# Copyright 2011-2013 GRNET S.A. All rights reserved.
4 9fea53cc Vangelis Koukis
#
5 9fea53cc Vangelis Koukis
# Redistribution and use in source and binary forms, with or
6 9fea53cc Vangelis Koukis
# without modification, are permitted provided that the following
7 9fea53cc Vangelis Koukis
# conditions are met:
8 9fea53cc Vangelis Koukis
#
9 9fea53cc Vangelis Koukis
#   1. Redistributions of source code must retain the above
10 9fea53cc Vangelis Koukis
#      copyright notice, this list of conditions and the following
11 9fea53cc Vangelis Koukis
#      disclaimer.
12 9fea53cc Vangelis Koukis
#
13 9fea53cc Vangelis Koukis
#   2. Redistributions in binary form must reproduce the above
14 9fea53cc Vangelis Koukis
#      copyright notice, this list of conditions and the following
15 9fea53cc Vangelis Koukis
#      disclaimer in the documentation and/or other materials
16 9fea53cc Vangelis Koukis
#      provided with the distribution.
17 9fea53cc Vangelis Koukis
#
18 9fea53cc Vangelis Koukis
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19 9fea53cc Vangelis Koukis
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 9fea53cc Vangelis Koukis
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 9fea53cc Vangelis Koukis
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22 9fea53cc Vangelis Koukis
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 9fea53cc Vangelis Koukis
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 9fea53cc Vangelis Koukis
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25 9fea53cc Vangelis Koukis
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 9fea53cc Vangelis Koukis
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 9fea53cc Vangelis Koukis
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 9fea53cc Vangelis Koukis
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 9fea53cc Vangelis Koukis
# POSSIBILITY OF SUCH DAMAGE.
30 9fea53cc Vangelis Koukis
#
31 9fea53cc Vangelis Koukis
# The views and conclusions contained in the software and
32 9fea53cc Vangelis Koukis
# documentation are those of the authors and should not be
33 9fea53cc Vangelis Koukis
# interpreted as representing official policies, either expressed
34 9fea53cc Vangelis Koukis
# or implied, of GRNET S.A.
35 9fea53cc Vangelis Koukis
#
36 9fea53cc Vangelis Koukis
"""Business logic for reconciliation
37 9fea53cc Vangelis Koukis

38 9fea53cc Vangelis Koukis
Reconcile the contents of the DB with the actual state of the
39 9fea53cc Vangelis Koukis
Ganeti backend.
40 9fea53cc Vangelis Koukis

41 9fea53cc Vangelis Koukis
Let D be the set of VMs in the DB, G the set of VMs in Ganeti.
42 9fea53cc Vangelis Koukis
RULES:
43 9fea53cc Vangelis Koukis
    R1. Stale servers in DB:
44 9fea53cc Vangelis Koukis
            For any v in D but not in G:
45 9fea53cc Vangelis Koukis
            Set deleted=True.
46 9fea53cc Vangelis Koukis
    R2. Orphan instances in Ganet:
47 9fea53cc Vangelis Koukis
            For any v in G with deleted=True in D:
48 9fea53cc Vangelis Koukis
            Issue OP_INSTANCE_DESTROY.
49 9fea53cc Vangelis Koukis
    R3. Unsynced operstate:
50 9fea53cc Vangelis Koukis
            For any v whose operating state differs between G and V:
51 9fea53cc Vangelis Koukis
            Set the operating state in D based on the state in G.
52 9fea53cc Vangelis Koukis
In the code, D, G are Python dicts mapping instance ids to operating state.
53 9fea53cc Vangelis Koukis
For D, the operating state is chosen from VirtualMachine.OPER_STATES.
54 9fea53cc Vangelis Koukis
For G, the operating state is True if the machine is up, False otherwise.
55 9fea53cc Vangelis Koukis

56 9fea53cc Vangelis Koukis
"""
57 9fea53cc Vangelis Koukis
58 9fea53cc Vangelis Koukis
59 9fea53cc Vangelis Koukis
from django.core.management import setup_environ
60 9fea53cc Vangelis Koukis
try:
61 9fea53cc Vangelis Koukis
    from synnefo import settings
62 9fea53cc Vangelis Koukis
except ImportError:
63 9fea53cc Vangelis Koukis
    raise Exception("Cannot import settings, make sure PYTHONPATH contains "
64 9fea53cc Vangelis Koukis
                    "the parent directory of the Synnefo Django project.")
65 9fea53cc Vangelis Koukis
setup_environ(settings)
66 9fea53cc Vangelis Koukis
67 4161cb41 Christos Stavrakakis
68 75dc539e Christos Stavrakakis
import logging
69 75dc539e Christos Stavrakakis
import itertools
70 89b2b908 Christos Stavrakakis
import bitarray
71 4161cb41 Christos Stavrakakis
from datetime import datetime, timedelta
72 4161cb41 Christos Stavrakakis
73 75dc539e Christos Stavrakakis
from django.db import transaction
74 75dc539e Christos Stavrakakis
from synnefo.db.models import (Backend, VirtualMachine, Flavor,
75 89b2b908 Christos Stavrakakis
                               pooled_rapi_client, Network,
76 89b2b908 Christos Stavrakakis
                               BackendNetwork)
77 89b2b908 Christos Stavrakakis
from synnefo.db.pools import IPPool
78 75dc539e Christos Stavrakakis
from synnefo.logic import utils, backend as backend_mod
79 3524241a Christos Stavrakakis
from synnefo.logic.rapi import GanetiApiError
80 9fea53cc Vangelis Koukis
81 75dc539e Christos Stavrakakis
logger = logging.getLogger()
82 75dc539e Christos Stavrakakis
logging.basicConfig()
83 9e98ba3c Giorgos Verigakis
84 e63050ca Christos Stavrakakis
try:
85 e63050ca Christos Stavrakakis
    CHECK_INTERVAL = settings.RECONCILIATION_CHECK_INTERVAL
86 e63050ca Christos Stavrakakis
except AttributeError:
87 e63050ca Christos Stavrakakis
    CHECK_INTERVAL = 60
88 e63050ca Christos Stavrakakis
89 e63050ca Christos Stavrakakis
90 75dc539e Christos Stavrakakis
class BackendReconciler(object):
91 75dc539e Christos Stavrakakis
    def __init__(self, backend, logger, options=None):
92 75dc539e Christos Stavrakakis
        self.backend = backend
93 75dc539e Christos Stavrakakis
        self.log = logger
94 75dc539e Christos Stavrakakis
        self.client = backend.get_client()
95 75dc539e Christos Stavrakakis
        if options is None:
96 75dc539e Christos Stavrakakis
            self.options = {}
97 4161cb41 Christos Stavrakakis
        else:
98 75dc539e Christos Stavrakakis
            self.options = options
99 75dc539e Christos Stavrakakis
100 75dc539e Christos Stavrakakis
    def close(self):
101 75dc539e Christos Stavrakakis
        self.backend.put_client(self.client)
102 75dc539e Christos Stavrakakis
103 75dc539e Christos Stavrakakis
    @transaction.commit_on_success
104 75dc539e Christos Stavrakakis
    def reconcile(self):
105 75dc539e Christos Stavrakakis
        log = self.log
106 75dc539e Christos Stavrakakis
        backend = self.backend
107 75dc539e Christos Stavrakakis
        log.debug("Reconciling backend %s", backend)
108 75dc539e Christos Stavrakakis
109 75dc539e Christos Stavrakakis
        self.db_servers = get_database_servers(backend)
110 75dc539e Christos Stavrakakis
        self.db_servers_keys = set(self.db_servers.keys())
111 75dc539e Christos Stavrakakis
        log.debug("Got servers info from database.")
112 75dc539e Christos Stavrakakis
113 75dc539e Christos Stavrakakis
        self.gnt_servers = get_ganeti_servers(backend)
114 75dc539e Christos Stavrakakis
        self.gnt_servers_keys = set(self.gnt_servers.keys())
115 75dc539e Christos Stavrakakis
        log.debug("Got servers info from Ganeti backend.")
116 75dc539e Christos Stavrakakis
117 75dc539e Christos Stavrakakis
        self.event_time = datetime.now()
118 75dc539e Christos Stavrakakis
119 75dc539e Christos Stavrakakis
        self.stale_servers = self.reconcile_stale_servers()
120 75dc539e Christos Stavrakakis
        self.orphan_servers = self.reconcile_orphan_servers()
121 75dc539e Christos Stavrakakis
        self.unsynced_servers = self.reconcile_unsynced_servers()
122 75dc539e Christos Stavrakakis
        self.close()
123 75dc539e Christos Stavrakakis
124 75dc539e Christos Stavrakakis
    def get_build_status(self, db_server):
125 75dc539e Christos Stavrakakis
        job = db_server.backendjobid
126 75dc539e Christos Stavrakakis
        if job is None:
127 75dc539e Christos Stavrakakis
            created = db_server.created
128 75dc539e Christos Stavrakakis
            # Job has not yet been enqueued.
129 75dc539e Christos Stavrakakis
            if self.event_time < created + timedelta(seconds=60):
130 75dc539e Christos Stavrakakis
                return "RUNNING"
131 75dc539e Christos Stavrakakis
            else:
132 75dc539e Christos Stavrakakis
                return "ERROR"
133 75dc539e Christos Stavrakakis
        else:
134 75dc539e Christos Stavrakakis
            updated = db_server.backendtime
135 75dc539e Christos Stavrakakis
            if self.event_time >= updated + timedelta(seconds=60):
136 75dc539e Christos Stavrakakis
                try:
137 75dc539e Christos Stavrakakis
                    job_info = self.client.GetJobStatus(job_id=job)
138 75dc539e Christos Stavrakakis
                    finalized = ["success", "error", "cancelled"]
139 75dc539e Christos Stavrakakis
                    if job_info["status"] == "error":
140 75dc539e Christos Stavrakakis
                        return "ERROR"
141 75dc539e Christos Stavrakakis
                    elif job_info["status"] not in finalized:
142 75dc539e Christos Stavrakakis
                        return "RUNNING"
143 75dc539e Christos Stavrakakis
                    else:
144 75dc539e Christos Stavrakakis
                        return "FINALIZED"
145 75dc539e Christos Stavrakakis
                except GanetiApiError:
146 75dc539e Christos Stavrakakis
                    return "ERROR"
147 75dc539e Christos Stavrakakis
            else:
148 75dc539e Christos Stavrakakis
                self.log.debug("Pending build for server '%s'", db_server.id)
149 75dc539e Christos Stavrakakis
                return "RUNNING"
150 75dc539e Christos Stavrakakis
151 75dc539e Christos Stavrakakis
    def reconcile_stale_servers(self):
152 75dc539e Christos Stavrakakis
        # Detect stale servers
153 75dc539e Christos Stavrakakis
        stale = []
154 75dc539e Christos Stavrakakis
        stale_keys = self.db_servers_keys - self.gnt_servers_keys
155 75dc539e Christos Stavrakakis
        for server_id in stale_keys:
156 75dc539e Christos Stavrakakis
            db_server = self.db_servers[server_id]
157 75dc539e Christos Stavrakakis
            if db_server.operstate == "BUILD":
158 75dc539e Christos Stavrakakis
                build_status = self.get_build_status(db_server)
159 75dc539e Christos Stavrakakis
                if build_status == "ERROR":
160 75dc539e Christos Stavrakakis
                    # Special handling of BUILD eerrors
161 75dc539e Christos Stavrakakis
                    self.reconcile_building_server(db_server)
162 75dc539e Christos Stavrakakis
                elif build_status != "RUNNING":
163 75dc539e Christos Stavrakakis
                    stale.append(server_id)
164 75dc539e Christos Stavrakakis
            else:
165 75dc539e Christos Stavrakakis
                stale.append(server_id)
166 75dc539e Christos Stavrakakis
167 75dc539e Christos Stavrakakis
        # Report them
168 75dc539e Christos Stavrakakis
        if stale:
169 75dc539e Christos Stavrakakis
            self.log.info("Found stale servers %s at backend %s",
170 75dc539e Christos Stavrakakis
                          ", ".join(map(str, stale)), self.backend)
171 75dc539e Christos Stavrakakis
        else:
172 75dc539e Christos Stavrakakis
            self.log.debug("No stale servers at backend %s", self.backend)
173 75dc539e Christos Stavrakakis
174 75dc539e Christos Stavrakakis
        # Fix them
175 75dc539e Christos Stavrakakis
        if stale and self.options["fix_stale"]:
176 75dc539e Christos Stavrakakis
            for server_id in stale:
177 75dc539e Christos Stavrakakis
                db_server = self.db_servers[server_id]
178 75dc539e Christos Stavrakakis
                backend_mod.process_op_status(
179 75dc539e Christos Stavrakakis
                    vm=db_server,
180 75dc539e Christos Stavrakakis
                    etime=self.event_time,
181 75dc539e Christos Stavrakakis
                    jobid=-0,
182 75dc539e Christos Stavrakakis
                    opcode='OP_INSTANCE_REMOVE', status='success',
183 75dc539e Christos Stavrakakis
                    logmsg='Reconciliation: simulated Ganeti event')
184 75dc539e Christos Stavrakakis
            self.log.debug("Simulated Ganeti removal for stale servers.")
185 75dc539e Christos Stavrakakis
186 75dc539e Christos Stavrakakis
    def reconcile_orphan_servers(self):
187 75dc539e Christos Stavrakakis
        orphans = self.gnt_servers_keys - self.db_servers_keys
188 75dc539e Christos Stavrakakis
        if orphans:
189 75dc539e Christos Stavrakakis
            self.log.info("Found orphan servers %s at backend %s",
190 75dc539e Christos Stavrakakis
                          ", ".join(map(str, orphans)), self.backend)
191 75dc539e Christos Stavrakakis
        else:
192 75dc539e Christos Stavrakakis
            self.log.debug("No orphan servers at backend %s", self.backend)
193 75dc539e Christos Stavrakakis
194 75dc539e Christos Stavrakakis
        if orphans and self.options["fix_orphans"]:
195 75dc539e Christos Stavrakakis
            for server_id in orphans:
196 75dc539e Christos Stavrakakis
                server_name = utils.id_to_instance_name(server_id)
197 75dc539e Christos Stavrakakis
                self.client.DeleteInstance(server_name)
198 75dc539e Christos Stavrakakis
            self.log.debug("Issued OP_INSTANCE_REMOVE for orphan servers.")
199 75dc539e Christos Stavrakakis
200 75dc539e Christos Stavrakakis
    def reconcile_unsynced_servers(self):
201 75dc539e Christos Stavrakakis
        #log = self.log
202 75dc539e Christos Stavrakakis
        for server_id in self.db_servers_keys & self.gnt_servers_keys:
203 75dc539e Christos Stavrakakis
            db_server = self.db_servers[server_id]
204 75dc539e Christos Stavrakakis
            gnt_server = self.gnt_servers[server_id]
205 75dc539e Christos Stavrakakis
            if db_server.operstate == "BUILD":
206 75dc539e Christos Stavrakakis
                build_status = self.get_build_status(db_server)
207 75dc539e Christos Stavrakakis
                if build_status == "RUNNING":
208 75dc539e Christos Stavrakakis
                    # Do not reconcile building VMs
209 75dc539e Christos Stavrakakis
                    continue
210 75dc539e Christos Stavrakakis
                elif build_status == "ERROR":
211 75dc539e Christos Stavrakakis
                    # Special handling of build errors
212 75dc539e Christos Stavrakakis
                    self.reconcile_building_server(db_server)
213 75dc539e Christos Stavrakakis
                    continue
214 75dc539e Christos Stavrakakis
215 75dc539e Christos Stavrakakis
            self.reconcile_unsynced_operstate(server_id, db_server,
216 75dc539e Christos Stavrakakis
                                              gnt_server)
217 75dc539e Christos Stavrakakis
            self.reconcile_unsynced_flavor(server_id, db_server,
218 75dc539e Christos Stavrakakis
                                           gnt_server)
219 75dc539e Christos Stavrakakis
            self.reconcile_unsynced_nics(server_id, db_server, gnt_server)
220 75dc539e Christos Stavrakakis
            self.reconcile_unsynced_disks(server_id, db_server, gnt_server)
221 75dc539e Christos Stavrakakis
222 75dc539e Christos Stavrakakis
    def reconcile_building_server(self, db_server):
223 75dc539e Christos Stavrakakis
        self.log.info("Server '%s' is BUILD in DB, but 'ERROR' in Ganeti.",
224 75dc539e Christos Stavrakakis
                      db_server.id)
225 75dc539e Christos Stavrakakis
        if self.options["fix_unsynced"]:
226 75dc539e Christos Stavrakakis
            fix_opcode = "OP_INSTANCE_CREATE"
227 75dc539e Christos Stavrakakis
            backend_mod.process_op_status(
228 75dc539e Christos Stavrakakis
                vm=db_server,
229 75dc539e Christos Stavrakakis
                etime=self.event_time,
230 75dc539e Christos Stavrakakis
                jobid=-0,
231 75dc539e Christos Stavrakakis
                opcode=fix_opcode, status='error',
232 75dc539e Christos Stavrakakis
                logmsg='Reconciliation: simulated Ganeti event')
233 75dc539e Christos Stavrakakis
            self.log.debug("Simulated Ganeti error build event for"
234 75dc539e Christos Stavrakakis
                           " server '%s'", db_server.id)
235 75dc539e Christos Stavrakakis
236 75dc539e Christos Stavrakakis
    def reconcile_unsynced_operstate(self, server_id, db_server, gnt_server):
237 75dc539e Christos Stavrakakis
        if db_server.operstate != gnt_server["state"]:
238 75dc539e Christos Stavrakakis
            self.log.info("Server '%s' is '%s' in DB and '%s' in Ganeti.",
239 75dc539e Christos Stavrakakis
                          server_id, db_server.operstate, gnt_server["state"])
240 75dc539e Christos Stavrakakis
            if self.options["fix_unsynced"]:
241 5d3e597a Christos Stavrakakis
                # If server is in building state, you will have first to
242 5d3e597a Christos Stavrakakis
                # reconcile it's creation, to avoid wrong quotas
243 5d3e597a Christos Stavrakakis
                if db_server.operstate == "BUILD":
244 5d3e597a Christos Stavrakakis
                    backend_mod.process_op_status(
245 5d3e597a Christos Stavrakakis
                        vm=db_server, etime=self.event_time, jobid=-0,
246 5d3e597a Christos Stavrakakis
                        opcode="OP_INSTANCE_CREATE", status='success',
247 5d3e597a Christos Stavrakakis
                        logmsg='Reconciliation: simulated Ganeti event')
248 5d3e597a Christos Stavrakakis
                fix_opcode = "OP_INSTANCE_STARTUP"\
249 5d3e597a Christos Stavrakakis
                    if gnt_server["state"] == "STARTED"\
250 75dc539e Christos Stavrakakis
                    else "OP_INSTANCE_SHUTDOWN"
251 75dc539e Christos Stavrakakis
                backend_mod.process_op_status(
252 5d3e597a Christos Stavrakakis
                    vm=db_server, etime=self.event_time, jobid=-0,
253 75dc539e Christos Stavrakakis
                    opcode=fix_opcode, status='success',
254 75dc539e Christos Stavrakakis
                    logmsg='Reconciliation: simulated Ganeti event')
255 75dc539e Christos Stavrakakis
                self.log.debug("Simulated Ganeti state event for server '%s'",
256 75dc539e Christos Stavrakakis
                               server_id)
257 75dc539e Christos Stavrakakis
258 75dc539e Christos Stavrakakis
    def reconcile_unsynced_flavor(self, server_id, db_server, gnt_server):
259 75dc539e Christos Stavrakakis
        db_flavor = db_server.flavor
260 75dc539e Christos Stavrakakis
        gnt_flavor = gnt_server["flavor"]
261 75dc539e Christos Stavrakakis
        if (db_flavor.ram != gnt_flavor["ram"] or
262 5d3e597a Christos Stavrakakis
           db_flavor.cpu != gnt_flavor["vcpus"]):
263 a67419d8 Christos Stavrakakis
            try:
264 a67419d8 Christos Stavrakakis
                gnt_flavor = Flavor.objects.get(
265 75dc539e Christos Stavrakakis
                    ram=gnt_flavor["ram"],
266 75dc539e Christos Stavrakakis
                    cpu=gnt_flavor["vcpus"],
267 75dc539e Christos Stavrakakis
                    disk=db_flavor.disk,
268 75dc539e Christos Stavrakakis
                    disk_template=db_flavor.disk_template)
269 a67419d8 Christos Stavrakakis
            except Flavor.DoesNotExist:
270 75dc539e Christos Stavrakakis
                self.log.warning("Server '%s' has unknown flavor.", server_id)
271 75dc539e Christos Stavrakakis
                return
272 75dc539e Christos Stavrakakis
273 75dc539e Christos Stavrakakis
            self.log.info("Server '%s' has flavor '%' in DB and '%s' in"
274 75dc539e Christos Stavrakakis
                          " Ganeti", server_id, db_flavor, gnt_flavor)
275 75dc539e Christos Stavrakakis
            if self.options["fix_unsynced_flavors"]:
276 75dc539e Christos Stavrakakis
                old_state = db_server.operstate
277 75dc539e Christos Stavrakakis
                opcode = "OP_INSTANCE_SET_PARAMS"
278 75dc539e Christos Stavrakakis
                beparams = {"vcpus": gnt_flavor.cpu,
279 75dc539e Christos Stavrakakis
                            "minmem": gnt_flavor.ram,
280 75dc539e Christos Stavrakakis
                            "maxmem": gnt_flavor.ram}
281 75dc539e Christos Stavrakakis
                backend_mod.process_op_status(
282 75dc539e Christos Stavrakakis
                    vm=db_server, etime=self.event_time, jobid=-0,
283 75dc539e Christos Stavrakakis
                    opcode=opcode, status='success',
284 75dc539e Christos Stavrakakis
                    beparams=beparams,
285 75dc539e Christos Stavrakakis
                    logmsg='Reconciliation: simulated Ganeti event')
286 75dc539e Christos Stavrakakis
                # process_op_status with beparams will set the vmstate to
287 75dc539e Christos Stavrakakis
                # shutdown. Fix this be returning it to old state
288 75dc539e Christos Stavrakakis
                vm = VirtualMachine.objects.get(pk=server_id)
289 75dc539e Christos Stavrakakis
                vm.operstate = old_state
290 75dc539e Christos Stavrakakis
                vm.save()
291 75dc539e Christos Stavrakakis
                self.log.debug("Simulated Ganeti flavor event for server '%s'",
292 75dc539e Christos Stavrakakis
                               server_id)
293 75dc539e Christos Stavrakakis
294 75dc539e Christos Stavrakakis
    def reconcile_unsynced_nics(self, server_id, db_server, gnt_server):
295 75dc539e Christos Stavrakakis
        db_nics = db_server.nics.order_by("index")
296 75dc539e Christos Stavrakakis
        gnt_nics = gnt_server["nics"]
297 75dc539e Christos Stavrakakis
        gnt_nics_parsed = backend_mod.process_ganeti_nics(gnt_nics)
298 75dc539e Christos Stavrakakis
        if backend_mod.nics_changed(db_nics, gnt_nics_parsed):
299 75dc539e Christos Stavrakakis
            msg = "Found unsynced NICs for server '%s'.\n\t"\
300 75dc539e Christos Stavrakakis
                  "DB: %s\n\tGaneti: %s"
301 75dc539e Christos Stavrakakis
            db_nics_str = ", ".join(map(format_db_nic, db_nics))
302 75dc539e Christos Stavrakakis
            gnt_nics_str = ", ".join(map(format_gnt_nic, gnt_nics_parsed))
303 75dc539e Christos Stavrakakis
            self.log.info(msg, server_id, db_nics_str, gnt_nics_str)
304 75dc539e Christos Stavrakakis
            if self.options["fix_unsynced_nics"]:
305 75dc539e Christos Stavrakakis
                backend_mod.process_net_status(vm=db_server,
306 75dc539e Christos Stavrakakis
                                               etime=self.event_time,
307 75dc539e Christos Stavrakakis
                                               nics=gnt_nics)
308 75dc539e Christos Stavrakakis
309 75dc539e Christos Stavrakakis
    def reconcile_unsynced_disks(self, server_id, db_server, gnt_server):
310 75dc539e Christos Stavrakakis
        pass
311 75dc539e Christos Stavrakakis
312 75dc539e Christos Stavrakakis
313 75dc539e Christos Stavrakakis
def format_db_nic(nic):
314 75dc539e Christos Stavrakakis
    return "Index: %s IP: %s Network: %s MAC: %s Firewall: %s" % (nic.index,
315 75dc539e Christos Stavrakakis
           nic.ipv4, nic.network_id, nic.mac, nic.firewall_profile)
316 75dc539e Christos Stavrakakis
317 75dc539e Christos Stavrakakis
318 75dc539e Christos Stavrakakis
def format_gnt_nic(nic):
319 75dc539e Christos Stavrakakis
    return "Index: %s IP: %s Network: %s MAC: %s Firewall: %s" %\
320 75dc539e Christos Stavrakakis
           (nic["index"], nic["ipv4"], nic["network"], nic["mac"],
321 75dc539e Christos Stavrakakis
            nic["firewall_profile"])
322 9fea53cc Vangelis Koukis
323 cc92b70f Christos Stavrakakis
324 0e9a423f Christos Stavrakakis
#
325 0e9a423f Christos Stavrakakis
# Networks
326 0e9a423f Christos Stavrakakis
#
327 3524241a Christos Stavrakakis
328 3524241a Christos Stavrakakis
329 0e9a423f Christos Stavrakakis
def get_networks_from_ganeti(backend):
330 44e2c577 Christos Stavrakakis
    prefix = settings.BACKEND_PREFIX_ID + 'net-'
331 0e9a423f Christos Stavrakakis
332 0e9a423f Christos Stavrakakis
    networks = {}
333 3524241a Christos Stavrakakis
    with pooled_rapi_client(backend) as c:
334 3524241a Christos Stavrakakis
        for net in c.GetNetworks(bulk=True):
335 3524241a Christos Stavrakakis
            if net['name'].startswith(prefix):
336 3524241a Christos Stavrakakis
                id = utils.id_from_network_name(net['name'])
337 3524241a Christos Stavrakakis
                networks[id] = net
338 0e9a423f Christos Stavrakakis
339 0e9a423f Christos Stavrakakis
    return networks
340 0e9a423f Christos Stavrakakis
341 0e9a423f Christos Stavrakakis
342 0e9a423f Christos Stavrakakis
def hanging_networks(backend, GNets):
343 0e9a423f Christos Stavrakakis
    """Get networks that are not connected to all Nodegroups.
344 0e9a423f Christos Stavrakakis

345 0e9a423f Christos Stavrakakis
    """
346 0e9a423f Christos Stavrakakis
    def get_network_groups(group_list):
347 0e9a423f Christos Stavrakakis
        groups = set()
348 0e9a423f Christos Stavrakakis
        for g in group_list:
349 0e9a423f Christos Stavrakakis
            g_name = g.split('(')[0]
350 0e9a423f Christos Stavrakakis
            groups.add(g_name)
351 0e9a423f Christos Stavrakakis
        return groups
352 0e9a423f Christos Stavrakakis
353 3524241a Christos Stavrakakis
    with pooled_rapi_client(backend) as c:
354 3524241a Christos Stavrakakis
        groups = set(c.GetGroups())
355 0e9a423f Christos Stavrakakis
356 0e9a423f Christos Stavrakakis
    hanging = {}
357 0e9a423f Christos Stavrakakis
    for id, info in GNets.items():
358 0e9a423f Christos Stavrakakis
        group_list = get_network_groups(info['group_list'])
359 0e9a423f Christos Stavrakakis
        if group_list != groups:
360 0e9a423f Christos Stavrakakis
            hanging[id] = groups - group_list
361 0e9a423f Christos Stavrakakis
    return hanging
362 0e9a423f Christos Stavrakakis
363 9fea53cc Vangelis Koukis
364 75dc539e Christos Stavrakakis
def get_online_backends():
365 75dc539e Christos Stavrakakis
    return Backend.objects.filter(offline=False)
366 75dc539e Christos Stavrakakis
367 75dc539e Christos Stavrakakis
368 75dc539e Christos Stavrakakis
def get_database_servers(backend):
369 75dc539e Christos Stavrakakis
    servers = backend.virtual_machines.select_related("nics", "flavor")\
370 75dc539e Christos Stavrakakis
                                      .filter(deleted=False)
371 75dc539e Christos Stavrakakis
    return dict([(s.id, s) for s in servers])
372 75dc539e Christos Stavrakakis
373 75dc539e Christos Stavrakakis
374 75dc539e Christos Stavrakakis
def get_ganeti_servers(backend):
375 75dc539e Christos Stavrakakis
    gnt_instances = backend_mod.get_instances(backend)
376 75dc539e Christos Stavrakakis
    # Filter out non-synnefo instances
377 75dc539e Christos Stavrakakis
    snf_backend_prefix = settings.BACKEND_PREFIX_ID
378 75dc539e Christos Stavrakakis
    gnt_instances = filter(lambda i: i["name"].startswith(snf_backend_prefix),
379 75dc539e Christos Stavrakakis
                           gnt_instances)
380 75dc539e Christos Stavrakakis
    gnt_instances = map(parse_gnt_instance, gnt_instances)
381 75dc539e Christos Stavrakakis
    return dict([(i["id"], i) for i in gnt_instances if i["id"] is not None])
382 75dc539e Christos Stavrakakis
383 75dc539e Christos Stavrakakis
384 75dc539e Christos Stavrakakis
def parse_gnt_instance(instance):
385 75dc539e Christos Stavrakakis
    try:
386 75dc539e Christos Stavrakakis
        instance_id = utils.id_from_instance_name(instance['name'])
387 75dc539e Christos Stavrakakis
    except Exception:
388 75dc539e Christos Stavrakakis
        logger.error("Ignoring instance with malformed name %s",
389 75dc539e Christos Stavrakakis
                     instance['name'])
390 75dc539e Christos Stavrakakis
        return (None, None)
391 75dc539e Christos Stavrakakis
392 75dc539e Christos Stavrakakis
    beparams = instance["beparams"]
393 75dc539e Christos Stavrakakis
394 75dc539e Christos Stavrakakis
    vcpus = beparams["vcpus"]
395 75dc539e Christos Stavrakakis
    ram = beparams["maxmem"]
396 75dc539e Christos Stavrakakis
    state = instance["oper_state"] and "STARTED" or "STOPPED"
397 75dc539e Christos Stavrakakis
398 75dc539e Christos Stavrakakis
    return {
399 75dc539e Christos Stavrakakis
        "id": instance_id,
400 75dc539e Christos Stavrakakis
        "state": state,  # FIX
401 75dc539e Christos Stavrakakis
        "updated": datetime.fromtimestamp(instance["mtime"]),
402 75dc539e Christos Stavrakakis
        "disks": disks_from_instance(instance),
403 75dc539e Christos Stavrakakis
        "nics": nics_from_instance(instance),
404 75dc539e Christos Stavrakakis
        "flavor": {"vcpus": vcpus,
405 75dc539e Christos Stavrakakis
                   "ram": ram},
406 75dc539e Christos Stavrakakis
        "tags": instance["tags"]
407 75dc539e Christos Stavrakakis
    }
408 75dc539e Christos Stavrakakis
409 75dc539e Christos Stavrakakis
410 75dc539e Christos Stavrakakis
def nics_from_instance(i):
411 75dc539e Christos Stavrakakis
    ips = zip(itertools.repeat('ip'), i['nic.ips'])
412 75dc539e Christos Stavrakakis
    macs = zip(itertools.repeat('mac'), i['nic.macs'])
413 75dc539e Christos Stavrakakis
    networks = zip(itertools.repeat('network'), i['nic.networks'])
414 75dc539e Christos Stavrakakis
    # modes = zip(itertools.repeat('mode'), i['nic.modes'])
415 75dc539e Christos Stavrakakis
    # links = zip(itertools.repeat('link'), i['nic.links'])
416 75dc539e Christos Stavrakakis
    # nics = zip(ips,macs,modes,networks,links)
417 75dc539e Christos Stavrakakis
    nics = zip(ips, macs, networks)
418 75dc539e Christos Stavrakakis
    nics = map(lambda x: dict(x), nics)
419 75dc539e Christos Stavrakakis
    #nics = dict(enumerate(nics))
420 75dc539e Christos Stavrakakis
    tags = i["tags"]
421 75dc539e Christos Stavrakakis
    for tag in tags:
422 75dc539e Christos Stavrakakis
        t = tag.split(":")
423 75dc539e Christos Stavrakakis
        if t[0:2] == ["synnefo", "network"]:
424 75dc539e Christos Stavrakakis
            if len(t) != 4:
425 75dc539e Christos Stavrakakis
                logger.error("Malformed synefo tag %s", tag)
426 75dc539e Christos Stavrakakis
                continue
427 75dc539e Christos Stavrakakis
            try:
428 75dc539e Christos Stavrakakis
                index = int(t[2])
429 75dc539e Christos Stavrakakis
                nics[index]['firewall'] = t[3]
430 75dc539e Christos Stavrakakis
            except ValueError:
431 75dc539e Christos Stavrakakis
                logger.error("Malformed synnefo tag %s", tag)
432 75dc539e Christos Stavrakakis
            except IndexError:
433 75dc539e Christos Stavrakakis
                logger.error("Found tag %s for non-existent NIC %d",
434 75dc539e Christos Stavrakakis
                             tag, index)
435 75dc539e Christos Stavrakakis
    return nics
436 9fea53cc Vangelis Koukis
437 9fea53cc Vangelis Koukis
438 75dc539e Christos Stavrakakis
def disks_from_instance(i):
439 75dc539e Christos Stavrakakis
    return dict([(index, {"size": size})
440 75dc539e Christos Stavrakakis
                 for index, size in enumerate(i["disk.sizes"])])
441 89b2b908 Christos Stavrakakis
442 89b2b908 Christos Stavrakakis
443 89b2b908 Christos Stavrakakis
class NetworkReconciler(object):
444 89b2b908 Christos Stavrakakis
    def __init__(self, logger, fix=False, conflicting_ips=False):
445 89b2b908 Christos Stavrakakis
        self.log = logger
446 89b2b908 Christos Stavrakakis
        self.conflicting_ips = conflicting_ips
447 89b2b908 Christos Stavrakakis
        self.fix = fix
448 89b2b908 Christos Stavrakakis
449 89b2b908 Christos Stavrakakis
    @transaction.commit_on_success
450 89b2b908 Christos Stavrakakis
    def reconcile_networks(self):
451 89b2b908 Christos Stavrakakis
        # Get models from DB
452 89b2b908 Christos Stavrakakis
        backends = Backend.objects.exclude(offline=True)
453 89b2b908 Christos Stavrakakis
        networks = Network.objects.filter(deleted=False)
454 89b2b908 Christos Stavrakakis
455 89b2b908 Christos Stavrakakis
        self.event_time = datetime.now()
456 89b2b908 Christos Stavrakakis
457 89b2b908 Christos Stavrakakis
        # Get info from all ganeti backends
458 89b2b908 Christos Stavrakakis
        ganeti_networks = {}
459 89b2b908 Christos Stavrakakis
        ganeti_hanging_networks = {}
460 89b2b908 Christos Stavrakakis
        for b in backends:
461 89b2b908 Christos Stavrakakis
            g_nets = get_networks_from_ganeti(b)
462 89b2b908 Christos Stavrakakis
            ganeti_networks[b] = g_nets
463 89b2b908 Christos Stavrakakis
            g_hanging_nets = hanging_networks(b, g_nets)
464 89b2b908 Christos Stavrakakis
            ganeti_hanging_networks[b] = g_hanging_nets
465 89b2b908 Christos Stavrakakis
466 89b2b908 Christos Stavrakakis
        # Perform reconciliation for each network
467 89b2b908 Christos Stavrakakis
        for network in networks:
468 89b2b908 Christos Stavrakakis
            ip_available_maps = []
469 89b2b908 Christos Stavrakakis
            ip_reserved_maps = []
470 89b2b908 Christos Stavrakakis
            for bend in backends:
471 89b2b908 Christos Stavrakakis
                bnet = get_backend_network(network, bend)
472 89b2b908 Christos Stavrakakis
                gnet = ganeti_networks[bend].get(network.id)
473 89b2b908 Christos Stavrakakis
                if not bnet:
474 89b2b908 Christos Stavrakakis
                    if network.floating_ip_pool:
475 89b2b908 Christos Stavrakakis
                        # Network is a floating IP pool and does not exist in
476 89b2b908 Christos Stavrakakis
                        # backend. We need to create it
477 89b2b908 Christos Stavrakakis
                        bnet = self.reconcile_parted_network(network, bend)
478 89b2b908 Christos Stavrakakis
                    elif not gnet:
479 89b2b908 Christos Stavrakakis
                        # Network does not exist either in Ganeti nor in BD.
480 89b2b908 Christos Stavrakakis
                        continue
481 89b2b908 Christos Stavrakakis
                    else:
482 89b2b908 Christos Stavrakakis
                        # Network exists in Ganeti and not in DB.
483 89b2b908 Christos Stavrakakis
                        if network.action != "DESTROY" and not network.public:
484 89b2b908 Christos Stavrakakis
                            bnet = self.reconcile_parted_network(network, bend)
485 89b2b908 Christos Stavrakakis
                        else:
486 89b2b908 Christos Stavrakakis
                            continue
487 89b2b908 Christos Stavrakakis
488 89b2b908 Christos Stavrakakis
                if not gnet:
489 89b2b908 Christos Stavrakakis
                    # Network does not exist in Ganeti. If the network action
490 89b2b908 Christos Stavrakakis
                    # is DESTROY, we have to mark as deleted in DB, else we
491 89b2b908 Christos Stavrakakis
                    # have to create it in Ganeti.
492 89b2b908 Christos Stavrakakis
                    if network.action == "DESTROY":
493 89b2b908 Christos Stavrakakis
                        if bnet.operstate != "DELETED":
494 89b2b908 Christos Stavrakakis
                            self.reconcile_stale_network(bnet)
495 89b2b908 Christos Stavrakakis
                    else:
496 89b2b908 Christos Stavrakakis
                        self.reconcile_missing_network(network, bend)
497 89b2b908 Christos Stavrakakis
                    # Skip rest reconciliation!
498 89b2b908 Christos Stavrakakis
                    continue
499 89b2b908 Christos Stavrakakis
500 89b2b908 Christos Stavrakakis
                try:
501 89b2b908 Christos Stavrakakis
                    hanging_groups = ganeti_hanging_networks[bend][network.id]
502 89b2b908 Christos Stavrakakis
                except KeyError:
503 89b2b908 Christos Stavrakakis
                    # Network is connected to all nodegroups
504 89b2b908 Christos Stavrakakis
                    hanging_groups = []
505 89b2b908 Christos Stavrakakis
506 89b2b908 Christos Stavrakakis
                if hanging_groups:
507 89b2b908 Christos Stavrakakis
                    # CASE-3: Ganeti networks not connected to all nodegroups
508 89b2b908 Christos Stavrakakis
                    self.reconcile_hanging_groups(network, bend,
509 89b2b908 Christos Stavrakakis
                                                  hanging_groups)
510 89b2b908 Christos Stavrakakis
                    continue
511 89b2b908 Christos Stavrakakis
512 89b2b908 Christos Stavrakakis
                if bnet.operstate != 'ACTIVE':
513 89b2b908 Christos Stavrakakis
                    # CASE-4: Unsynced network state. At this point the network
514 89b2b908 Christos Stavrakakis
                    # exists and is connected to all nodes so is must be
515 89b2b908 Christos Stavrakakis
                    # active!
516 89b2b908 Christos Stavrakakis
                    self.reconcile_unsynced_network(network, bend, bnet)
517 89b2b908 Christos Stavrakakis
518 89b2b908 Christos Stavrakakis
                # Get ganeti IP Pools
519 89b2b908 Christos Stavrakakis
                available_map, reserved_map = get_network_pool(gnet)
520 89b2b908 Christos Stavrakakis
                ip_available_maps.append(available_map)
521 89b2b908 Christos Stavrakakis
                ip_reserved_maps.append(reserved_map)
522 89b2b908 Christos Stavrakakis
523 89b2b908 Christos Stavrakakis
            if ip_available_maps or ip_reserved_maps:
524 89b2b908 Christos Stavrakakis
                # CASE-5: Unsynced IP Pools
525 89b2b908 Christos Stavrakakis
                self.reconcile_ip_pools(network, ip_available_maps,
526 89b2b908 Christos Stavrakakis
                                        ip_reserved_maps)
527 89b2b908 Christos Stavrakakis
528 89b2b908 Christos Stavrakakis
            if self.conflicting_ips:
529 89b2b908 Christos Stavrakakis
                self.detect_conflicting_ips()
530 89b2b908 Christos Stavrakakis
531 89b2b908 Christos Stavrakakis
        # CASE-6: Orphan networks
532 89b2b908 Christos Stavrakakis
        self.reconcile_orphan_networks(networks, ganeti_networks)
533 89b2b908 Christos Stavrakakis
534 89b2b908 Christos Stavrakakis
    def reconcile_parted_network(self, network, backend):
535 89b2b908 Christos Stavrakakis
        self.log.info("D: Missing DB entry for network %s in backend %s",
536 89b2b908 Christos Stavrakakis
                      network, backend)
537 89b2b908 Christos Stavrakakis
        if self.fix:
538 89b2b908 Christos Stavrakakis
            network.create_backend_network(backend)
539 89b2b908 Christos Stavrakakis
            self.log.info("F: Created DB entry")
540 89b2b908 Christos Stavrakakis
            bnet = get_backend_network(network, backend)
541 89b2b908 Christos Stavrakakis
            return bnet
542 89b2b908 Christos Stavrakakis
543 89b2b908 Christos Stavrakakis
    def reconcile_stale_network(self, backend_network):
544 89b2b908 Christos Stavrakakis
        self.log.info("D: Stale DB entry for network %s in backend %s",
545 89b2b908 Christos Stavrakakis
                      backend_network.network, backend_network.backend)
546 89b2b908 Christos Stavrakakis
        if self.fix:
547 89b2b908 Christos Stavrakakis
            backend_mod.process_network_status(
548 89b2b908 Christos Stavrakakis
                backend_network, self.event_time, 0,
549 89b2b908 Christos Stavrakakis
                "OP_NETWORK_REMOVE",
550 89b2b908 Christos Stavrakakis
                "success",
551 89b2b908 Christos Stavrakakis
                "Reconciliation simulated event")
552 89b2b908 Christos Stavrakakis
            self.log.info("F: Reconciled event: OP_NETWORK_REMOVE")
553 89b2b908 Christos Stavrakakis
554 89b2b908 Christos Stavrakakis
    def reconcile_missing_network(self, network, backend):
555 89b2b908 Christos Stavrakakis
        self.log.info("D: Missing Ganeti network %s in backend %s",
556 89b2b908 Christos Stavrakakis
                      network, backend)
557 89b2b908 Christos Stavrakakis
        if self.fix:
558 89b2b908 Christos Stavrakakis
            backend_mod.create_network(network, backend)
559 89b2b908 Christos Stavrakakis
            self.log.info("F: Issued OP_NETWORK_CONNECT")
560 89b2b908 Christos Stavrakakis
561 89b2b908 Christos Stavrakakis
    def reconcile_hanging_groups(self, network, backend, hanging_groups):
562 89b2b908 Christos Stavrakakis
        self.log.info('D: Network %s in backend %s is not connected to '
563 89b2b908 Christos Stavrakakis
                      'the following groups:', network, backend)
564 89b2b908 Christos Stavrakakis
        self.log.info('-  ' + '\n-  '.join(hanging_groups))
565 89b2b908 Christos Stavrakakis
        if self.fix:
566 89b2b908 Christos Stavrakakis
            for group in hanging_groups:
567 89b2b908 Christos Stavrakakis
                self.log.info('F: Connecting network %s to nodegroup %s',
568 89b2b908 Christos Stavrakakis
                              network, group)
569 89b2b908 Christos Stavrakakis
                backend_mod.connect_network(network, backend, depends=[],
570 89b2b908 Christos Stavrakakis
                                            group=group)
571 89b2b908 Christos Stavrakakis
572 89b2b908 Christos Stavrakakis
    def reconcile_unsynced_network(self, network, backend, backend_network):
573 89b2b908 Christos Stavrakakis
        self.log.info("D: Unsynced network %s in backend %s", network, backend)
574 89b2b908 Christos Stavrakakis
        if self.fix:
575 89b2b908 Christos Stavrakakis
            self.log.info("F: Issuing OP_NETWORK_CONNECT")
576 89b2b908 Christos Stavrakakis
            backend_mod.process_network_status(
577 89b2b908 Christos Stavrakakis
                backend_network, self.event_time, 0,
578 89b2b908 Christos Stavrakakis
                "OP_NETWORK_CONNECT",
579 89b2b908 Christos Stavrakakis
                "success",
580 89b2b908 Christos Stavrakakis
                "Reconciliation simulated eventd")
581 89b2b908 Christos Stavrakakis
582 89b2b908 Christos Stavrakakis
    def reconcile_ip_pools(self, network, available_maps, reserved_maps):
583 89b2b908 Christos Stavrakakis
        available_map = reduce(lambda x, y: x & y, available_maps)
584 89b2b908 Christos Stavrakakis
        reserved_map = reduce(lambda x, y: x & y, reserved_maps)
585 89b2b908 Christos Stavrakakis
586 89b2b908 Christos Stavrakakis
        pool = network.get_pool()
587 89b2b908 Christos Stavrakakis
        # Temporary release unused floating IPs
588 89b2b908 Christos Stavrakakis
        temp_pool = network.get_pool()
589 89b2b908 Christos Stavrakakis
        used_ips = network.nics.values_list("ipv4", flat=True)
590 89b2b908 Christos Stavrakakis
        unused_static_ips = network.floating_ips.exclude(ipv4__in=used_ips)
591 89b2b908 Christos Stavrakakis
        map(lambda ip: temp_pool.put(ip.ipv4), unused_static_ips)
592 89b2b908 Christos Stavrakakis
        if temp_pool.available != available_map:
593 89b2b908 Christos Stavrakakis
            self.log.info("D: Unsynced available map of network %s:\n"
594 89b2b908 Christos Stavrakakis
                          "\tDB: %r\n\tGB: %r", network,
595 89b2b908 Christos Stavrakakis
                          temp_pool.available.to01(),
596 89b2b908 Christos Stavrakakis
                          available_map.to01())
597 89b2b908 Christos Stavrakakis
            if self.fix:
598 89b2b908 Christos Stavrakakis
                pool.available = available_map
599 89b2b908 Christos Stavrakakis
                # Release unsued floating IPs, as they are not included in the
600 89b2b908 Christos Stavrakakis
                # available map
601 89b2b908 Christos Stavrakakis
                map(lambda ip: pool.reserve(ip.ipv4), unused_static_ips)
602 89b2b908 Christos Stavrakakis
                pool.save()
603 89b2b908 Christos Stavrakakis
        if pool.reserved != reserved_map:
604 89b2b908 Christos Stavrakakis
            self.log.info("D: Unsynced reserved map of network %s:\n"
605 89b2b908 Christos Stavrakakis
                          "\tDB: %r\n\tGB: %r", network, pool.reserved.to01(),
606 89b2b908 Christos Stavrakakis
                          reserved_map.to01())
607 89b2b908 Christos Stavrakakis
            if self.fix:
608 89b2b908 Christos Stavrakakis
                pool.reserved = reserved_map
609 89b2b908 Christos Stavrakakis
                pool.save()
610 89b2b908 Christos Stavrakakis
611 89b2b908 Christos Stavrakakis
    def detect_conflicting_ips(self, network):
612 89b2b908 Christos Stavrakakis
        """Detect NIC's that have the same IP in the same network."""
613 89b2b908 Christos Stavrakakis
        machine_ips = network.nics.all().values_list('ipv4', 'machine')
614 89b2b908 Christos Stavrakakis
        ips = map(lambda x: x[0], machine_ips)
615 89b2b908 Christos Stavrakakis
        distinct_ips = set(ips)
616 89b2b908 Christos Stavrakakis
        if len(distinct_ips) < len(ips):
617 89b2b908 Christos Stavrakakis
            for i in distinct_ips:
618 89b2b908 Christos Stavrakakis
                ips.remove(i)
619 89b2b908 Christos Stavrakakis
            for i in ips:
620 89b2b908 Christos Stavrakakis
                machines = [utils.id_to_instance_name(x[1])
621 89b2b908 Christos Stavrakakis
                            for x in machine_ips if x[0] == i]
622 89b2b908 Christos Stavrakakis
                self.log.info('D: Conflicting IP:%s Machines: %s',
623 89b2b908 Christos Stavrakakis
                              i, ', '.join(machines))
624 89b2b908 Christos Stavrakakis
625 89b2b908 Christos Stavrakakis
    def reconcile_orphan_networks(self, db_networks, ganeti_networks):
626 89b2b908 Christos Stavrakakis
        # Detect Orphan Networks in Ganeti
627 89b2b908 Christos Stavrakakis
        db_network_ids = set([net.id for net in db_networks])
628 89b2b908 Christos Stavrakakis
        for back_end, ganeti_networks in ganeti_networks.items():
629 89b2b908 Christos Stavrakakis
            ganeti_network_ids = set(ganeti_networks.keys())
630 89b2b908 Christos Stavrakakis
            orphans = ganeti_network_ids - db_network_ids
631 89b2b908 Christos Stavrakakis
632 89b2b908 Christos Stavrakakis
            if len(orphans) > 0:
633 89b2b908 Christos Stavrakakis
                self.log.info('D: Orphan Networks in backend %s:',
634 89b2b908 Christos Stavrakakis
                              back_end.clustername)
635 89b2b908 Christos Stavrakakis
                self.log.info('-  ' + '\n-  '.join([str(o) for o in orphans]))
636 89b2b908 Christos Stavrakakis
                if self.fix:
637 89b2b908 Christos Stavrakakis
                    for net_id in orphans:
638 89b2b908 Christos Stavrakakis
                        self.log.info('Disconnecting and deleting network %d',
639 89b2b908 Christos Stavrakakis
                                      net_id)
640 89b2b908 Christos Stavrakakis
                        try:
641 89b2b908 Christos Stavrakakis
                            network = Network.objects.get(id=net_id)
642 89b2b908 Christos Stavrakakis
                            backend_mod.delete_network(network,
643 89b2b908 Christos Stavrakakis
                                                       backend=back_end)
644 89b2b908 Christos Stavrakakis
                        except Network.DoesNotExist:
645 89b2b908 Christos Stavrakakis
                            self.log.info("Not entry for network %s in DB !!",
646 89b2b908 Christos Stavrakakis
                                          net_id)
647 89b2b908 Christos Stavrakakis
648 89b2b908 Christos Stavrakakis
649 89b2b908 Christos Stavrakakis
def get_backend_network(network, backend):
650 89b2b908 Christos Stavrakakis
    try:
651 89b2b908 Christos Stavrakakis
        return BackendNetwork.objects.get(network=network, backend=backend)
652 89b2b908 Christos Stavrakakis
    except BackendNetwork.DoesNotExist:
653 89b2b908 Christos Stavrakakis
        return None
654 89b2b908 Christos Stavrakakis
655 89b2b908 Christos Stavrakakis
656 89b2b908 Christos Stavrakakis
def get_network_pool(gnet):
657 89b2b908 Christos Stavrakakis
    """Return available and reserved IP maps.
658 89b2b908 Christos Stavrakakis

659 89b2b908 Christos Stavrakakis
    Extract the available and reserved IP map from the info return from Ganeti
660 89b2b908 Christos Stavrakakis
    for a network.
661 89b2b908 Christos Stavrakakis

662 89b2b908 Christos Stavrakakis
    """
663 89b2b908 Christos Stavrakakis
    converter = IPPool(Foo(gnet['network']))
664 89b2b908 Christos Stavrakakis
    a_map = bitarray_from_map(gnet['map'])
665 89b2b908 Christos Stavrakakis
    a_map.invert()
666 89b2b908 Christos Stavrakakis
    reserved = gnet['external_reservations']
667 89b2b908 Christos Stavrakakis
    r_map = a_map.copy()
668 89b2b908 Christos Stavrakakis
    r_map.setall(True)
669 89b2b908 Christos Stavrakakis
    if reserved:
670 89b2b908 Christos Stavrakakis
        for address in reserved.split(','):
671 89b2b908 Christos Stavrakakis
            index = converter.value_to_index(address.strip())
672 89b2b908 Christos Stavrakakis
            a_map[index] = True
673 89b2b908 Christos Stavrakakis
            r_map[index] = False
674 89b2b908 Christos Stavrakakis
    return a_map, r_map
675 89b2b908 Christos Stavrakakis
676 89b2b908 Christos Stavrakakis
677 89b2b908 Christos Stavrakakis
def bitarray_from_map(bitmap):
678 89b2b908 Christos Stavrakakis
    return bitarray.bitarray(bitmap.replace("X", "1").replace(".", "0"))
679 89b2b908 Christos Stavrakakis
680 89b2b908 Christos Stavrakakis
681 89b2b908 Christos Stavrakakis
class Foo():
682 89b2b908 Christos Stavrakakis
    def __init__(self, subnet):
683 89b2b908 Christos Stavrakakis
        self.available_map = ''
684 89b2b908 Christos Stavrakakis
        self.reserved_map = ''
685 89b2b908 Christos Stavrakakis
        self.size = 0
686 89b2b908 Christos Stavrakakis
        self.network = Foo.Foo1(subnet)
687 89b2b908 Christos Stavrakakis
688 89b2b908 Christos Stavrakakis
    class Foo1():
689 89b2b908 Christos Stavrakakis
        def __init__(self, subnet):
690 89b2b908 Christos Stavrakakis
            self.subnet = subnet
691 89b2b908 Christos Stavrakakis
            self.gateway = None