Statistics
| Branch: | Tag: | Revision:

root / snf-cyclades-app / synnefo / logic / reconciliation.py @ a633e247

History | View | Annotate | Download (27.8 kB)

1 9fea53cc Vangelis Koukis
# -*- coding: utf-8 -*-
2 9fea53cc Vangelis Koukis
#
3 75dc539e Christos Stavrakakis
# Copyright 2011-2013 GRNET S.A. All rights reserved.
4 9fea53cc Vangelis Koukis
#
5 9fea53cc Vangelis Koukis
# Redistribution and use in source and binary forms, with or
6 9fea53cc Vangelis Koukis
# without modification, are permitted provided that the following
7 9fea53cc Vangelis Koukis
# conditions are met:
8 9fea53cc Vangelis Koukis
#
9 9fea53cc Vangelis Koukis
#   1. Redistributions of source code must retain the above
10 9fea53cc Vangelis Koukis
#      copyright notice, this list of conditions and the following
11 9fea53cc Vangelis Koukis
#      disclaimer.
12 9fea53cc Vangelis Koukis
#
13 9fea53cc Vangelis Koukis
#   2. Redistributions in binary form must reproduce the above
14 9fea53cc Vangelis Koukis
#      copyright notice, this list of conditions and the following
15 9fea53cc Vangelis Koukis
#      disclaimer in the documentation and/or other materials
16 9fea53cc Vangelis Koukis
#      provided with the distribution.
17 9fea53cc Vangelis Koukis
#
18 9fea53cc Vangelis Koukis
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19 9fea53cc Vangelis Koukis
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 9fea53cc Vangelis Koukis
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 9fea53cc Vangelis Koukis
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22 9fea53cc Vangelis Koukis
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 9fea53cc Vangelis Koukis
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 9fea53cc Vangelis Koukis
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25 9fea53cc Vangelis Koukis
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 9fea53cc Vangelis Koukis
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 9fea53cc Vangelis Koukis
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 9fea53cc Vangelis Koukis
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 9fea53cc Vangelis Koukis
# POSSIBILITY OF SUCH DAMAGE.
30 9fea53cc Vangelis Koukis
#
31 9fea53cc Vangelis Koukis
# The views and conclusions contained in the software and
32 9fea53cc Vangelis Koukis
# documentation are those of the authors and should not be
33 9fea53cc Vangelis Koukis
# interpreted as representing official policies, either expressed
34 9fea53cc Vangelis Koukis
# or implied, of GRNET S.A.
35 9fea53cc Vangelis Koukis
#
36 9fea53cc Vangelis Koukis
"""Business logic for reconciliation
37 9fea53cc Vangelis Koukis

38 9fea53cc Vangelis Koukis
Reconcile the contents of the DB with the actual state of the
39 9fea53cc Vangelis Koukis
Ganeti backend.
40 9fea53cc Vangelis Koukis

41 9fea53cc Vangelis Koukis
Let D be the set of VMs in the DB, G the set of VMs in Ganeti.
42 9fea53cc Vangelis Koukis
RULES:
43 9fea53cc Vangelis Koukis
    R1. Stale servers in DB:
44 9fea53cc Vangelis Koukis
            For any v in D but not in G:
45 9fea53cc Vangelis Koukis
            Set deleted=True.
46 9fea53cc Vangelis Koukis
    R2. Orphan instances in Ganet:
47 9fea53cc Vangelis Koukis
            For any v in G with deleted=True in D:
48 9fea53cc Vangelis Koukis
            Issue OP_INSTANCE_DESTROY.
49 9fea53cc Vangelis Koukis
    R3. Unsynced operstate:
50 9fea53cc Vangelis Koukis
            For any v whose operating state differs between G and V:
51 9fea53cc Vangelis Koukis
            Set the operating state in D based on the state in G.
52 9fea53cc Vangelis Koukis
In the code, D, G are Python dicts mapping instance ids to operating state.
53 9fea53cc Vangelis Koukis
For D, the operating state is chosen from VirtualMachine.OPER_STATES.
54 9fea53cc Vangelis Koukis
For G, the operating state is True if the machine is up, False otherwise.
55 9fea53cc Vangelis Koukis

56 9fea53cc Vangelis Koukis
"""
57 9fea53cc Vangelis Koukis
58 9fea53cc Vangelis Koukis
59 9fea53cc Vangelis Koukis
from django.core.management import setup_environ
60 9fea53cc Vangelis Koukis
try:
61 9fea53cc Vangelis Koukis
    from synnefo import settings
62 9fea53cc Vangelis Koukis
except ImportError:
63 9fea53cc Vangelis Koukis
    raise Exception("Cannot import settings, make sure PYTHONPATH contains "
64 9fea53cc Vangelis Koukis
                    "the parent directory of the Synnefo Django project.")
65 9fea53cc Vangelis Koukis
setup_environ(settings)
66 9fea53cc Vangelis Koukis
67 4161cb41 Christos Stavrakakis
68 75dc539e Christos Stavrakakis
import logging
69 75dc539e Christos Stavrakakis
import itertools
70 89b2b908 Christos Stavrakakis
import bitarray
71 63f9eb8e Christos Stavrakakis
from datetime import datetime
72 4161cb41 Christos Stavrakakis
73 75dc539e Christos Stavrakakis
from django.db import transaction
74 75dc539e Christos Stavrakakis
from synnefo.db.models import (Backend, VirtualMachine, Flavor,
75 89b2b908 Christos Stavrakakis
                               pooled_rapi_client, Network,
76 89b2b908 Christos Stavrakakis
                               BackendNetwork)
77 89b2b908 Christos Stavrakakis
from synnefo.db.pools import IPPool
78 75dc539e Christos Stavrakakis
from synnefo.logic import utils, backend as backend_mod
79 9fea53cc Vangelis Koukis
80 75dc539e Christos Stavrakakis
logger = logging.getLogger()
81 75dc539e Christos Stavrakakis
logging.basicConfig()
82 9e98ba3c Giorgos Verigakis
83 e63050ca Christos Stavrakakis
try:
84 e63050ca Christos Stavrakakis
    CHECK_INTERVAL = settings.RECONCILIATION_CHECK_INTERVAL
85 e63050ca Christos Stavrakakis
except AttributeError:
86 e63050ca Christos Stavrakakis
    CHECK_INTERVAL = 60
87 e63050ca Christos Stavrakakis
88 63f9eb8e Christos Stavrakakis
GANETI_JOB_ERROR = "error"
89 63f9eb8e Christos Stavrakakis
GANETI_JOBS_PENDING = ["queued", "waiting", "running", "canceling"]
90 63f9eb8e Christos Stavrakakis
GANETI_JOBS_FINALIZED = ["success", "error", "canceled"]
91 63f9eb8e Christos Stavrakakis
92 e63050ca Christos Stavrakakis
93 75dc539e Christos Stavrakakis
class BackendReconciler(object):
94 75dc539e Christos Stavrakakis
    def __init__(self, backend, logger, options=None):
95 75dc539e Christos Stavrakakis
        self.backend = backend
96 75dc539e Christos Stavrakakis
        self.log = logger
97 75dc539e Christos Stavrakakis
        self.client = backend.get_client()
98 75dc539e Christos Stavrakakis
        if options is None:
99 75dc539e Christos Stavrakakis
            self.options = {}
100 4161cb41 Christos Stavrakakis
        else:
101 75dc539e Christos Stavrakakis
            self.options = options
102 75dc539e Christos Stavrakakis
103 75dc539e Christos Stavrakakis
    def close(self):
104 75dc539e Christos Stavrakakis
        self.backend.put_client(self.client)
105 75dc539e Christos Stavrakakis
106 75dc539e Christos Stavrakakis
    @transaction.commit_on_success
107 75dc539e Christos Stavrakakis
    def reconcile(self):
108 75dc539e Christos Stavrakakis
        log = self.log
109 75dc539e Christos Stavrakakis
        backend = self.backend
110 75dc539e Christos Stavrakakis
        log.debug("Reconciling backend %s", backend)
111 75dc539e Christos Stavrakakis
112 75dc539e Christos Stavrakakis
        self.db_servers = get_database_servers(backend)
113 75dc539e Christos Stavrakakis
        self.db_servers_keys = set(self.db_servers.keys())
114 75dc539e Christos Stavrakakis
        log.debug("Got servers info from database.")
115 75dc539e Christos Stavrakakis
116 75dc539e Christos Stavrakakis
        self.gnt_servers = get_ganeti_servers(backend)
117 75dc539e Christos Stavrakakis
        self.gnt_servers_keys = set(self.gnt_servers.keys())
118 75dc539e Christos Stavrakakis
        log.debug("Got servers info from Ganeti backend.")
119 75dc539e Christos Stavrakakis
120 63f9eb8e Christos Stavrakakis
        self.gnt_jobs = get_ganeti_jobs(backend)
121 63f9eb8e Christos Stavrakakis
        log.debug("Got jobs from Ganeti backend")
122 63f9eb8e Christos Stavrakakis
123 75dc539e Christos Stavrakakis
        self.event_time = datetime.now()
124 75dc539e Christos Stavrakakis
125 75dc539e Christos Stavrakakis
        self.stale_servers = self.reconcile_stale_servers()
126 75dc539e Christos Stavrakakis
        self.orphan_servers = self.reconcile_orphan_servers()
127 75dc539e Christos Stavrakakis
        self.unsynced_servers = self.reconcile_unsynced_servers()
128 75dc539e Christos Stavrakakis
        self.close()
129 75dc539e Christos Stavrakakis
130 75dc539e Christos Stavrakakis
    def get_build_status(self, db_server):
131 63f9eb8e Christos Stavrakakis
        job_id = db_server.backendjobid
132 63f9eb8e Christos Stavrakakis
        if job_id in self.gnt_jobs:
133 63f9eb8e Christos Stavrakakis
            gnt_job_status = self.gnt_jobs[job_id]["status"]
134 63f9eb8e Christos Stavrakakis
            if gnt_job_status == GANETI_JOB_ERROR:
135 63f9eb8e Christos Stavrakakis
                return "ERROR"
136 63f9eb8e Christos Stavrakakis
            elif gnt_job_status not in GANETI_JOBS_FINALIZED:
137 75dc539e Christos Stavrakakis
                return "RUNNING"
138 75dc539e Christos Stavrakakis
            else:
139 63f9eb8e Christos Stavrakakis
                return "FINALIZED"
140 75dc539e Christos Stavrakakis
        else:
141 63f9eb8e Christos Stavrakakis
            return "ERROR"
142 75dc539e Christos Stavrakakis
143 75dc539e Christos Stavrakakis
    def reconcile_stale_servers(self):
144 75dc539e Christos Stavrakakis
        # Detect stale servers
145 75dc539e Christos Stavrakakis
        stale = []
146 75dc539e Christos Stavrakakis
        stale_keys = self.db_servers_keys - self.gnt_servers_keys
147 75dc539e Christos Stavrakakis
        for server_id in stale_keys:
148 75dc539e Christos Stavrakakis
            db_server = self.db_servers[server_id]
149 75dc539e Christos Stavrakakis
            if db_server.operstate == "BUILD":
150 75dc539e Christos Stavrakakis
                build_status = self.get_build_status(db_server)
151 75dc539e Christos Stavrakakis
                if build_status == "ERROR":
152 75dc539e Christos Stavrakakis
                    # Special handling of BUILD eerrors
153 75dc539e Christos Stavrakakis
                    self.reconcile_building_server(db_server)
154 75dc539e Christos Stavrakakis
                elif build_status != "RUNNING":
155 75dc539e Christos Stavrakakis
                    stale.append(server_id)
156 75dc539e Christos Stavrakakis
            else:
157 75dc539e Christos Stavrakakis
                stale.append(server_id)
158 75dc539e Christos Stavrakakis
159 75dc539e Christos Stavrakakis
        # Report them
160 75dc539e Christos Stavrakakis
        if stale:
161 75dc539e Christos Stavrakakis
            self.log.info("Found stale servers %s at backend %s",
162 75dc539e Christos Stavrakakis
                          ", ".join(map(str, stale)), self.backend)
163 75dc539e Christos Stavrakakis
        else:
164 75dc539e Christos Stavrakakis
            self.log.debug("No stale servers at backend %s", self.backend)
165 75dc539e Christos Stavrakakis
166 75dc539e Christos Stavrakakis
        # Fix them
167 75dc539e Christos Stavrakakis
        if stale and self.options["fix_stale"]:
168 75dc539e Christos Stavrakakis
            for server_id in stale:
169 75dc539e Christos Stavrakakis
                db_server = self.db_servers[server_id]
170 75dc539e Christos Stavrakakis
                backend_mod.process_op_status(
171 75dc539e Christos Stavrakakis
                    vm=db_server,
172 75dc539e Christos Stavrakakis
                    etime=self.event_time,
173 75dc539e Christos Stavrakakis
                    jobid=-0,
174 75dc539e Christos Stavrakakis
                    opcode='OP_INSTANCE_REMOVE', status='success',
175 75dc539e Christos Stavrakakis
                    logmsg='Reconciliation: simulated Ganeti event')
176 75dc539e Christos Stavrakakis
            self.log.debug("Simulated Ganeti removal for stale servers.")
177 75dc539e Christos Stavrakakis
178 75dc539e Christos Stavrakakis
    def reconcile_orphan_servers(self):
179 75dc539e Christos Stavrakakis
        orphans = self.gnt_servers_keys - self.db_servers_keys
180 75dc539e Christos Stavrakakis
        if orphans:
181 75dc539e Christos Stavrakakis
            self.log.info("Found orphan servers %s at backend %s",
182 75dc539e Christos Stavrakakis
                          ", ".join(map(str, orphans)), self.backend)
183 75dc539e Christos Stavrakakis
        else:
184 75dc539e Christos Stavrakakis
            self.log.debug("No orphan servers at backend %s", self.backend)
185 75dc539e Christos Stavrakakis
186 75dc539e Christos Stavrakakis
        if orphans and self.options["fix_orphans"]:
187 75dc539e Christos Stavrakakis
            for server_id in orphans:
188 75dc539e Christos Stavrakakis
                server_name = utils.id_to_instance_name(server_id)
189 75dc539e Christos Stavrakakis
                self.client.DeleteInstance(server_name)
190 75dc539e Christos Stavrakakis
            self.log.debug("Issued OP_INSTANCE_REMOVE for orphan servers.")
191 75dc539e Christos Stavrakakis
192 75dc539e Christos Stavrakakis
    def reconcile_unsynced_servers(self):
193 75dc539e Christos Stavrakakis
        #log = self.log
194 75dc539e Christos Stavrakakis
        for server_id in self.db_servers_keys & self.gnt_servers_keys:
195 75dc539e Christos Stavrakakis
            db_server = self.db_servers[server_id]
196 75dc539e Christos Stavrakakis
            gnt_server = self.gnt_servers[server_id]
197 75dc539e Christos Stavrakakis
            if db_server.operstate == "BUILD":
198 75dc539e Christos Stavrakakis
                build_status = self.get_build_status(db_server)
199 75dc539e Christos Stavrakakis
                if build_status == "RUNNING":
200 75dc539e Christos Stavrakakis
                    # Do not reconcile building VMs
201 75dc539e Christos Stavrakakis
                    continue
202 75dc539e Christos Stavrakakis
                elif build_status == "ERROR":
203 75dc539e Christos Stavrakakis
                    # Special handling of build errors
204 75dc539e Christos Stavrakakis
                    self.reconcile_building_server(db_server)
205 75dc539e Christos Stavrakakis
                    continue
206 75dc539e Christos Stavrakakis
207 75dc539e Christos Stavrakakis
            self.reconcile_unsynced_operstate(server_id, db_server,
208 75dc539e Christos Stavrakakis
                                              gnt_server)
209 75dc539e Christos Stavrakakis
            self.reconcile_unsynced_flavor(server_id, db_server,
210 75dc539e Christos Stavrakakis
                                           gnt_server)
211 75dc539e Christos Stavrakakis
            self.reconcile_unsynced_nics(server_id, db_server, gnt_server)
212 75dc539e Christos Stavrakakis
            self.reconcile_unsynced_disks(server_id, db_server, gnt_server)
213 63f9eb8e Christos Stavrakakis
            if db_server.task is not None:
214 63f9eb8e Christos Stavrakakis
                self.reconcile_pending_task(server_id, db_server)
215 75dc539e Christos Stavrakakis
216 75dc539e Christos Stavrakakis
    def reconcile_building_server(self, db_server):
217 75dc539e Christos Stavrakakis
        self.log.info("Server '%s' is BUILD in DB, but 'ERROR' in Ganeti.",
218 75dc539e Christos Stavrakakis
                      db_server.id)
219 75dc539e Christos Stavrakakis
        if self.options["fix_unsynced"]:
220 75dc539e Christos Stavrakakis
            fix_opcode = "OP_INSTANCE_CREATE"
221 75dc539e Christos Stavrakakis
            backend_mod.process_op_status(
222 75dc539e Christos Stavrakakis
                vm=db_server,
223 75dc539e Christos Stavrakakis
                etime=self.event_time,
224 75dc539e Christos Stavrakakis
                jobid=-0,
225 75dc539e Christos Stavrakakis
                opcode=fix_opcode, status='error',
226 75dc539e Christos Stavrakakis
                logmsg='Reconciliation: simulated Ganeti event')
227 75dc539e Christos Stavrakakis
            self.log.debug("Simulated Ganeti error build event for"
228 75dc539e Christos Stavrakakis
                           " server '%s'", db_server.id)
229 75dc539e Christos Stavrakakis
230 75dc539e Christos Stavrakakis
    def reconcile_unsynced_operstate(self, server_id, db_server, gnt_server):
231 75dc539e Christos Stavrakakis
        if db_server.operstate != gnt_server["state"]:
232 75dc539e Christos Stavrakakis
            self.log.info("Server '%s' is '%s' in DB and '%s' in Ganeti.",
233 75dc539e Christos Stavrakakis
                          server_id, db_server.operstate, gnt_server["state"])
234 75dc539e Christos Stavrakakis
            if self.options["fix_unsynced"]:
235 5d3e597a Christos Stavrakakis
                # If server is in building state, you will have first to
236 5d3e597a Christos Stavrakakis
                # reconcile it's creation, to avoid wrong quotas
237 5d3e597a Christos Stavrakakis
                if db_server.operstate == "BUILD":
238 5d3e597a Christos Stavrakakis
                    backend_mod.process_op_status(
239 5d3e597a Christos Stavrakakis
                        vm=db_server, etime=self.event_time, jobid=-0,
240 5d3e597a Christos Stavrakakis
                        opcode="OP_INSTANCE_CREATE", status='success',
241 5d3e597a Christos Stavrakakis
                        logmsg='Reconciliation: simulated Ganeti event')
242 5d3e597a Christos Stavrakakis
                fix_opcode = "OP_INSTANCE_STARTUP"\
243 5d3e597a Christos Stavrakakis
                    if gnt_server["state"] == "STARTED"\
244 75dc539e Christos Stavrakakis
                    else "OP_INSTANCE_SHUTDOWN"
245 75dc539e Christos Stavrakakis
                backend_mod.process_op_status(
246 5d3e597a Christos Stavrakakis
                    vm=db_server, etime=self.event_time, jobid=-0,
247 75dc539e Christos Stavrakakis
                    opcode=fix_opcode, status='success',
248 75dc539e Christos Stavrakakis
                    logmsg='Reconciliation: simulated Ganeti event')
249 75dc539e Christos Stavrakakis
                self.log.debug("Simulated Ganeti state event for server '%s'",
250 75dc539e Christos Stavrakakis
                               server_id)
251 75dc539e Christos Stavrakakis
252 75dc539e Christos Stavrakakis
    def reconcile_unsynced_flavor(self, server_id, db_server, gnt_server):
253 75dc539e Christos Stavrakakis
        db_flavor = db_server.flavor
254 75dc539e Christos Stavrakakis
        gnt_flavor = gnt_server["flavor"]
255 75dc539e Christos Stavrakakis
        if (db_flavor.ram != gnt_flavor["ram"] or
256 5d3e597a Christos Stavrakakis
           db_flavor.cpu != gnt_flavor["vcpus"]):
257 a67419d8 Christos Stavrakakis
            try:
258 a67419d8 Christos Stavrakakis
                gnt_flavor = Flavor.objects.get(
259 75dc539e Christos Stavrakakis
                    ram=gnt_flavor["ram"],
260 75dc539e Christos Stavrakakis
                    cpu=gnt_flavor["vcpus"],
261 75dc539e Christos Stavrakakis
                    disk=db_flavor.disk,
262 75dc539e Christos Stavrakakis
                    disk_template=db_flavor.disk_template)
263 a67419d8 Christos Stavrakakis
            except Flavor.DoesNotExist:
264 75dc539e Christos Stavrakakis
                self.log.warning("Server '%s' has unknown flavor.", server_id)
265 75dc539e Christos Stavrakakis
                return
266 75dc539e Christos Stavrakakis
267 75dc539e Christos Stavrakakis
            self.log.info("Server '%s' has flavor '%' in DB and '%s' in"
268 75dc539e Christos Stavrakakis
                          " Ganeti", server_id, db_flavor, gnt_flavor)
269 75dc539e Christos Stavrakakis
            if self.options["fix_unsynced_flavors"]:
270 75dc539e Christos Stavrakakis
                old_state = db_server.operstate
271 75dc539e Christos Stavrakakis
                opcode = "OP_INSTANCE_SET_PARAMS"
272 75dc539e Christos Stavrakakis
                beparams = {"vcpus": gnt_flavor.cpu,
273 75dc539e Christos Stavrakakis
                            "minmem": gnt_flavor.ram,
274 75dc539e Christos Stavrakakis
                            "maxmem": gnt_flavor.ram}
275 75dc539e Christos Stavrakakis
                backend_mod.process_op_status(
276 75dc539e Christos Stavrakakis
                    vm=db_server, etime=self.event_time, jobid=-0,
277 75dc539e Christos Stavrakakis
                    opcode=opcode, status='success',
278 75dc539e Christos Stavrakakis
                    beparams=beparams,
279 75dc539e Christos Stavrakakis
                    logmsg='Reconciliation: simulated Ganeti event')
280 75dc539e Christos Stavrakakis
                # process_op_status with beparams will set the vmstate to
281 75dc539e Christos Stavrakakis
                # shutdown. Fix this be returning it to old state
282 75dc539e Christos Stavrakakis
                vm = VirtualMachine.objects.get(pk=server_id)
283 75dc539e Christos Stavrakakis
                vm.operstate = old_state
284 75dc539e Christos Stavrakakis
                vm.save()
285 75dc539e Christos Stavrakakis
                self.log.debug("Simulated Ganeti flavor event for server '%s'",
286 75dc539e Christos Stavrakakis
                               server_id)
287 75dc539e Christos Stavrakakis
288 75dc539e Christos Stavrakakis
    def reconcile_unsynced_nics(self, server_id, db_server, gnt_server):
289 75dc539e Christos Stavrakakis
        db_nics = db_server.nics.order_by("index")
290 75dc539e Christos Stavrakakis
        gnt_nics = gnt_server["nics"]
291 75dc539e Christos Stavrakakis
        gnt_nics_parsed = backend_mod.process_ganeti_nics(gnt_nics)
292 75dc539e Christos Stavrakakis
        if backend_mod.nics_changed(db_nics, gnt_nics_parsed):
293 75dc539e Christos Stavrakakis
            msg = "Found unsynced NICs for server '%s'.\n\t"\
294 75dc539e Christos Stavrakakis
                  "DB: %s\n\tGaneti: %s"
295 75dc539e Christos Stavrakakis
            db_nics_str = ", ".join(map(format_db_nic, db_nics))
296 75dc539e Christos Stavrakakis
            gnt_nics_str = ", ".join(map(format_gnt_nic, gnt_nics_parsed))
297 75dc539e Christos Stavrakakis
            self.log.info(msg, server_id, db_nics_str, gnt_nics_str)
298 75dc539e Christos Stavrakakis
            if self.options["fix_unsynced_nics"]:
299 75dc539e Christos Stavrakakis
                backend_mod.process_net_status(vm=db_server,
300 75dc539e Christos Stavrakakis
                                               etime=self.event_time,
301 75dc539e Christos Stavrakakis
                                               nics=gnt_nics)
302 75dc539e Christos Stavrakakis
303 75dc539e Christos Stavrakakis
    def reconcile_unsynced_disks(self, server_id, db_server, gnt_server):
304 75dc539e Christos Stavrakakis
        pass
305 75dc539e Christos Stavrakakis
306 63f9eb8e Christos Stavrakakis
    def reconcile_pending_task(self, server_id, db_server):
307 63f9eb8e Christos Stavrakakis
        job_id = db_server.task_job_id
308 63f9eb8e Christos Stavrakakis
        pending_task = False
309 63f9eb8e Christos Stavrakakis
        if job_id not in self.gnt_jobs:
310 63f9eb8e Christos Stavrakakis
            pending_task = True
311 63f9eb8e Christos Stavrakakis
        else:
312 63f9eb8e Christos Stavrakakis
            gnt_job_status = self.gnt_job[job_id]["status"]
313 63f9eb8e Christos Stavrakakis
            if gnt_job_status in GANETI_JOBS_FINALIZED:
314 63f9eb8e Christos Stavrakakis
                pending_task = True
315 63f9eb8e Christos Stavrakakis
316 63f9eb8e Christos Stavrakakis
        if pending_task:
317 63f9eb8e Christos Stavrakakis
            self.log.info("Found server '%s' with pending task: '%s'",
318 63f9eb8e Christos Stavrakakis
                          server_id, db_server.task)
319 63f9eb8e Christos Stavrakakis
            if self.options["fixed_pending_tasks"]:
320 63f9eb8e Christos Stavrakakis
                db_server.task = None
321 63f9eb8e Christos Stavrakakis
                db_server.task_job_id = None
322 63f9eb8e Christos Stavrakakis
                db_server.save()
323 63f9eb8e Christos Stavrakakis
                self.log.info("Cleared pending task for server '%s", server_id)
324 63f9eb8e Christos Stavrakakis
325 75dc539e Christos Stavrakakis
326 75dc539e Christos Stavrakakis
def format_db_nic(nic):
327 75dc539e Christos Stavrakakis
    return "Index: %s IP: %s Network: %s MAC: %s Firewall: %s" % (nic.index,
328 75dc539e Christos Stavrakakis
           nic.ipv4, nic.network_id, nic.mac, nic.firewall_profile)
329 75dc539e Christos Stavrakakis
330 75dc539e Christos Stavrakakis
331 75dc539e Christos Stavrakakis
def format_gnt_nic(nic):
332 75dc539e Christos Stavrakakis
    return "Index: %s IP: %s Network: %s MAC: %s Firewall: %s" %\
333 75dc539e Christos Stavrakakis
           (nic["index"], nic["ipv4"], nic["network"], nic["mac"],
334 75dc539e Christos Stavrakakis
            nic["firewall_profile"])
335 9fea53cc Vangelis Koukis
336 cc92b70f Christos Stavrakakis
337 0e9a423f Christos Stavrakakis
#
338 0e9a423f Christos Stavrakakis
# Networks
339 0e9a423f Christos Stavrakakis
#
340 3524241a Christos Stavrakakis
341 3524241a Christos Stavrakakis
342 0e9a423f Christos Stavrakakis
def get_networks_from_ganeti(backend):
343 44e2c577 Christos Stavrakakis
    prefix = settings.BACKEND_PREFIX_ID + 'net-'
344 0e9a423f Christos Stavrakakis
345 0e9a423f Christos Stavrakakis
    networks = {}
346 3524241a Christos Stavrakakis
    with pooled_rapi_client(backend) as c:
347 3524241a Christos Stavrakakis
        for net in c.GetNetworks(bulk=True):
348 3524241a Christos Stavrakakis
            if net['name'].startswith(prefix):
349 3524241a Christos Stavrakakis
                id = utils.id_from_network_name(net['name'])
350 3524241a Christos Stavrakakis
                networks[id] = net
351 0e9a423f Christos Stavrakakis
352 0e9a423f Christos Stavrakakis
    return networks
353 0e9a423f Christos Stavrakakis
354 0e9a423f Christos Stavrakakis
355 0e9a423f Christos Stavrakakis
def hanging_networks(backend, GNets):
356 0e9a423f Christos Stavrakakis
    """Get networks that are not connected to all Nodegroups.
357 0e9a423f Christos Stavrakakis

358 0e9a423f Christos Stavrakakis
    """
359 0e9a423f Christos Stavrakakis
    def get_network_groups(group_list):
360 0e9a423f Christos Stavrakakis
        groups = set()
361 0e9a423f Christos Stavrakakis
        for g in group_list:
362 0e9a423f Christos Stavrakakis
            g_name = g.split('(')[0]
363 0e9a423f Christos Stavrakakis
            groups.add(g_name)
364 0e9a423f Christos Stavrakakis
        return groups
365 0e9a423f Christos Stavrakakis
366 3524241a Christos Stavrakakis
    with pooled_rapi_client(backend) as c:
367 3524241a Christos Stavrakakis
        groups = set(c.GetGroups())
368 0e9a423f Christos Stavrakakis
369 0e9a423f Christos Stavrakakis
    hanging = {}
370 0e9a423f Christos Stavrakakis
    for id, info in GNets.items():
371 0e9a423f Christos Stavrakakis
        group_list = get_network_groups(info['group_list'])
372 0e9a423f Christos Stavrakakis
        if group_list != groups:
373 0e9a423f Christos Stavrakakis
            hanging[id] = groups - group_list
374 0e9a423f Christos Stavrakakis
    return hanging
375 0e9a423f Christos Stavrakakis
376 9fea53cc Vangelis Koukis
377 75dc539e Christos Stavrakakis
def get_online_backends():
378 75dc539e Christos Stavrakakis
    return Backend.objects.filter(offline=False)
379 75dc539e Christos Stavrakakis
380 75dc539e Christos Stavrakakis
381 75dc539e Christos Stavrakakis
def get_database_servers(backend):
382 75dc539e Christos Stavrakakis
    servers = backend.virtual_machines.select_related("nics", "flavor")\
383 75dc539e Christos Stavrakakis
                                      .filter(deleted=False)
384 75dc539e Christos Stavrakakis
    return dict([(s.id, s) for s in servers])
385 75dc539e Christos Stavrakakis
386 75dc539e Christos Stavrakakis
387 75dc539e Christos Stavrakakis
def get_ganeti_servers(backend):
388 75dc539e Christos Stavrakakis
    gnt_instances = backend_mod.get_instances(backend)
389 75dc539e Christos Stavrakakis
    # Filter out non-synnefo instances
390 75dc539e Christos Stavrakakis
    snf_backend_prefix = settings.BACKEND_PREFIX_ID
391 75dc539e Christos Stavrakakis
    gnt_instances = filter(lambda i: i["name"].startswith(snf_backend_prefix),
392 75dc539e Christos Stavrakakis
                           gnt_instances)
393 75dc539e Christos Stavrakakis
    gnt_instances = map(parse_gnt_instance, gnt_instances)
394 75dc539e Christos Stavrakakis
    return dict([(i["id"], i) for i in gnt_instances if i["id"] is not None])
395 75dc539e Christos Stavrakakis
396 75dc539e Christos Stavrakakis
397 75dc539e Christos Stavrakakis
def parse_gnt_instance(instance):
398 75dc539e Christos Stavrakakis
    try:
399 75dc539e Christos Stavrakakis
        instance_id = utils.id_from_instance_name(instance['name'])
400 75dc539e Christos Stavrakakis
    except Exception:
401 75dc539e Christos Stavrakakis
        logger.error("Ignoring instance with malformed name %s",
402 75dc539e Christos Stavrakakis
                     instance['name'])
403 75dc539e Christos Stavrakakis
        return (None, None)
404 75dc539e Christos Stavrakakis
405 75dc539e Christos Stavrakakis
    beparams = instance["beparams"]
406 75dc539e Christos Stavrakakis
407 75dc539e Christos Stavrakakis
    vcpus = beparams["vcpus"]
408 75dc539e Christos Stavrakakis
    ram = beparams["maxmem"]
409 75dc539e Christos Stavrakakis
    state = instance["oper_state"] and "STARTED" or "STOPPED"
410 75dc539e Christos Stavrakakis
411 75dc539e Christos Stavrakakis
    return {
412 75dc539e Christos Stavrakakis
        "id": instance_id,
413 75dc539e Christos Stavrakakis
        "state": state,  # FIX
414 75dc539e Christos Stavrakakis
        "updated": datetime.fromtimestamp(instance["mtime"]),
415 75dc539e Christos Stavrakakis
        "disks": disks_from_instance(instance),
416 75dc539e Christos Stavrakakis
        "nics": nics_from_instance(instance),
417 75dc539e Christos Stavrakakis
        "flavor": {"vcpus": vcpus,
418 75dc539e Christos Stavrakakis
                   "ram": ram},
419 75dc539e Christos Stavrakakis
        "tags": instance["tags"]
420 75dc539e Christos Stavrakakis
    }
421 75dc539e Christos Stavrakakis
422 75dc539e Christos Stavrakakis
423 75dc539e Christos Stavrakakis
def nics_from_instance(i):
424 75dc539e Christos Stavrakakis
    ips = zip(itertools.repeat('ip'), i['nic.ips'])
425 75dc539e Christos Stavrakakis
    macs = zip(itertools.repeat('mac'), i['nic.macs'])
426 75dc539e Christos Stavrakakis
    networks = zip(itertools.repeat('network'), i['nic.networks'])
427 75dc539e Christos Stavrakakis
    # modes = zip(itertools.repeat('mode'), i['nic.modes'])
428 75dc539e Christos Stavrakakis
    # links = zip(itertools.repeat('link'), i['nic.links'])
429 75dc539e Christos Stavrakakis
    # nics = zip(ips,macs,modes,networks,links)
430 75dc539e Christos Stavrakakis
    nics = zip(ips, macs, networks)
431 75dc539e Christos Stavrakakis
    nics = map(lambda x: dict(x), nics)
432 75dc539e Christos Stavrakakis
    #nics = dict(enumerate(nics))
433 75dc539e Christos Stavrakakis
    tags = i["tags"]
434 75dc539e Christos Stavrakakis
    for tag in tags:
435 75dc539e Christos Stavrakakis
        t = tag.split(":")
436 75dc539e Christos Stavrakakis
        if t[0:2] == ["synnefo", "network"]:
437 75dc539e Christos Stavrakakis
            if len(t) != 4:
438 75dc539e Christos Stavrakakis
                logger.error("Malformed synefo tag %s", tag)
439 75dc539e Christos Stavrakakis
                continue
440 75dc539e Christos Stavrakakis
            try:
441 75dc539e Christos Stavrakakis
                index = int(t[2])
442 75dc539e Christos Stavrakakis
                nics[index]['firewall'] = t[3]
443 75dc539e Christos Stavrakakis
            except ValueError:
444 75dc539e Christos Stavrakakis
                logger.error("Malformed synnefo tag %s", tag)
445 75dc539e Christos Stavrakakis
            except IndexError:
446 75dc539e Christos Stavrakakis
                logger.error("Found tag %s for non-existent NIC %d",
447 75dc539e Christos Stavrakakis
                             tag, index)
448 75dc539e Christos Stavrakakis
    return nics
449 9fea53cc Vangelis Koukis
450 9fea53cc Vangelis Koukis
451 63f9eb8e Christos Stavrakakis
def get_ganeti_jobs(backend):
452 63f9eb8e Christos Stavrakakis
    gnt_jobs = backend_mod.get_jobs(backend)
453 63f9eb8e Christos Stavrakakis
    return dict([(int(j["id"]), j) for j in gnt_jobs])
454 63f9eb8e Christos Stavrakakis
455 63f9eb8e Christos Stavrakakis
456 75dc539e Christos Stavrakakis
def disks_from_instance(i):
457 75dc539e Christos Stavrakakis
    return dict([(index, {"size": size})
458 75dc539e Christos Stavrakakis
                 for index, size in enumerate(i["disk.sizes"])])
459 89b2b908 Christos Stavrakakis
460 89b2b908 Christos Stavrakakis
461 89b2b908 Christos Stavrakakis
class NetworkReconciler(object):
462 89b2b908 Christos Stavrakakis
    def __init__(self, logger, fix=False, conflicting_ips=False):
463 89b2b908 Christos Stavrakakis
        self.log = logger
464 89b2b908 Christos Stavrakakis
        self.conflicting_ips = conflicting_ips
465 89b2b908 Christos Stavrakakis
        self.fix = fix
466 89b2b908 Christos Stavrakakis
467 89b2b908 Christos Stavrakakis
    @transaction.commit_on_success
468 89b2b908 Christos Stavrakakis
    def reconcile_networks(self):
469 89b2b908 Christos Stavrakakis
        # Get models from DB
470 89b2b908 Christos Stavrakakis
        backends = Backend.objects.exclude(offline=True)
471 89b2b908 Christos Stavrakakis
        networks = Network.objects.filter(deleted=False)
472 89b2b908 Christos Stavrakakis
473 89b2b908 Christos Stavrakakis
        self.event_time = datetime.now()
474 89b2b908 Christos Stavrakakis
475 89b2b908 Christos Stavrakakis
        # Get info from all ganeti backends
476 89b2b908 Christos Stavrakakis
        ganeti_networks = {}
477 89b2b908 Christos Stavrakakis
        ganeti_hanging_networks = {}
478 89b2b908 Christos Stavrakakis
        for b in backends:
479 89b2b908 Christos Stavrakakis
            g_nets = get_networks_from_ganeti(b)
480 89b2b908 Christos Stavrakakis
            ganeti_networks[b] = g_nets
481 89b2b908 Christos Stavrakakis
            g_hanging_nets = hanging_networks(b, g_nets)
482 89b2b908 Christos Stavrakakis
            ganeti_hanging_networks[b] = g_hanging_nets
483 89b2b908 Christos Stavrakakis
484 89b2b908 Christos Stavrakakis
        # Perform reconciliation for each network
485 89b2b908 Christos Stavrakakis
        for network in networks:
486 89b2b908 Christos Stavrakakis
            ip_available_maps = []
487 89b2b908 Christos Stavrakakis
            ip_reserved_maps = []
488 89b2b908 Christos Stavrakakis
            for bend in backends:
489 89b2b908 Christos Stavrakakis
                bnet = get_backend_network(network, bend)
490 89b2b908 Christos Stavrakakis
                gnet = ganeti_networks[bend].get(network.id)
491 89b2b908 Christos Stavrakakis
                if not bnet:
492 89b2b908 Christos Stavrakakis
                    if network.floating_ip_pool:
493 89b2b908 Christos Stavrakakis
                        # Network is a floating IP pool and does not exist in
494 89b2b908 Christos Stavrakakis
                        # backend. We need to create it
495 89b2b908 Christos Stavrakakis
                        bnet = self.reconcile_parted_network(network, bend)
496 89b2b908 Christos Stavrakakis
                    elif not gnet:
497 89b2b908 Christos Stavrakakis
                        # Network does not exist either in Ganeti nor in BD.
498 89b2b908 Christos Stavrakakis
                        continue
499 89b2b908 Christos Stavrakakis
                    else:
500 89b2b908 Christos Stavrakakis
                        # Network exists in Ganeti and not in DB.
501 89b2b908 Christos Stavrakakis
                        if network.action != "DESTROY" and not network.public:
502 89b2b908 Christos Stavrakakis
                            bnet = self.reconcile_parted_network(network, bend)
503 89b2b908 Christos Stavrakakis
                        else:
504 89b2b908 Christos Stavrakakis
                            continue
505 89b2b908 Christos Stavrakakis
506 89b2b908 Christos Stavrakakis
                if not gnet:
507 89b2b908 Christos Stavrakakis
                    # Network does not exist in Ganeti. If the network action
508 89b2b908 Christos Stavrakakis
                    # is DESTROY, we have to mark as deleted in DB, else we
509 89b2b908 Christos Stavrakakis
                    # have to create it in Ganeti.
510 89b2b908 Christos Stavrakakis
                    if network.action == "DESTROY":
511 89b2b908 Christos Stavrakakis
                        if bnet.operstate != "DELETED":
512 89b2b908 Christos Stavrakakis
                            self.reconcile_stale_network(bnet)
513 89b2b908 Christos Stavrakakis
                    else:
514 89b2b908 Christos Stavrakakis
                        self.reconcile_missing_network(network, bend)
515 89b2b908 Christos Stavrakakis
                    # Skip rest reconciliation!
516 89b2b908 Christos Stavrakakis
                    continue
517 89b2b908 Christos Stavrakakis
518 89b2b908 Christos Stavrakakis
                try:
519 89b2b908 Christos Stavrakakis
                    hanging_groups = ganeti_hanging_networks[bend][network.id]
520 89b2b908 Christos Stavrakakis
                except KeyError:
521 89b2b908 Christos Stavrakakis
                    # Network is connected to all nodegroups
522 89b2b908 Christos Stavrakakis
                    hanging_groups = []
523 89b2b908 Christos Stavrakakis
524 89b2b908 Christos Stavrakakis
                if hanging_groups:
525 89b2b908 Christos Stavrakakis
                    # CASE-3: Ganeti networks not connected to all nodegroups
526 89b2b908 Christos Stavrakakis
                    self.reconcile_hanging_groups(network, bend,
527 89b2b908 Christos Stavrakakis
                                                  hanging_groups)
528 89b2b908 Christos Stavrakakis
                    continue
529 89b2b908 Christos Stavrakakis
530 89b2b908 Christos Stavrakakis
                if bnet.operstate != 'ACTIVE':
531 89b2b908 Christos Stavrakakis
                    # CASE-4: Unsynced network state. At this point the network
532 89b2b908 Christos Stavrakakis
                    # exists and is connected to all nodes so is must be
533 89b2b908 Christos Stavrakakis
                    # active!
534 89b2b908 Christos Stavrakakis
                    self.reconcile_unsynced_network(network, bend, bnet)
535 89b2b908 Christos Stavrakakis
536 89b2b908 Christos Stavrakakis
                # Get ganeti IP Pools
537 89b2b908 Christos Stavrakakis
                available_map, reserved_map = get_network_pool(gnet)
538 89b2b908 Christos Stavrakakis
                ip_available_maps.append(available_map)
539 89b2b908 Christos Stavrakakis
                ip_reserved_maps.append(reserved_map)
540 89b2b908 Christos Stavrakakis
541 89b2b908 Christos Stavrakakis
            if ip_available_maps or ip_reserved_maps:
542 89b2b908 Christos Stavrakakis
                # CASE-5: Unsynced IP Pools
543 89b2b908 Christos Stavrakakis
                self.reconcile_ip_pools(network, ip_available_maps,
544 89b2b908 Christos Stavrakakis
                                        ip_reserved_maps)
545 89b2b908 Christos Stavrakakis
546 89b2b908 Christos Stavrakakis
            if self.conflicting_ips:
547 89b2b908 Christos Stavrakakis
                self.detect_conflicting_ips()
548 89b2b908 Christos Stavrakakis
549 89b2b908 Christos Stavrakakis
        # CASE-6: Orphan networks
550 89b2b908 Christos Stavrakakis
        self.reconcile_orphan_networks(networks, ganeti_networks)
551 89b2b908 Christos Stavrakakis
552 89b2b908 Christos Stavrakakis
    def reconcile_parted_network(self, network, backend):
553 89b2b908 Christos Stavrakakis
        self.log.info("D: Missing DB entry for network %s in backend %s",
554 89b2b908 Christos Stavrakakis
                      network, backend)
555 89b2b908 Christos Stavrakakis
        if self.fix:
556 89b2b908 Christos Stavrakakis
            network.create_backend_network(backend)
557 89b2b908 Christos Stavrakakis
            self.log.info("F: Created DB entry")
558 89b2b908 Christos Stavrakakis
            bnet = get_backend_network(network, backend)
559 89b2b908 Christos Stavrakakis
            return bnet
560 89b2b908 Christos Stavrakakis
561 89b2b908 Christos Stavrakakis
    def reconcile_stale_network(self, backend_network):
562 89b2b908 Christos Stavrakakis
        self.log.info("D: Stale DB entry for network %s in backend %s",
563 89b2b908 Christos Stavrakakis
                      backend_network.network, backend_network.backend)
564 89b2b908 Christos Stavrakakis
        if self.fix:
565 89b2b908 Christos Stavrakakis
            backend_mod.process_network_status(
566 89b2b908 Christos Stavrakakis
                backend_network, self.event_time, 0,
567 89b2b908 Christos Stavrakakis
                "OP_NETWORK_REMOVE",
568 89b2b908 Christos Stavrakakis
                "success",
569 89b2b908 Christos Stavrakakis
                "Reconciliation simulated event")
570 89b2b908 Christos Stavrakakis
            self.log.info("F: Reconciled event: OP_NETWORK_REMOVE")
571 89b2b908 Christos Stavrakakis
572 89b2b908 Christos Stavrakakis
    def reconcile_missing_network(self, network, backend):
573 89b2b908 Christos Stavrakakis
        self.log.info("D: Missing Ganeti network %s in backend %s",
574 89b2b908 Christos Stavrakakis
                      network, backend)
575 89b2b908 Christos Stavrakakis
        if self.fix:
576 89b2b908 Christos Stavrakakis
            backend_mod.create_network(network, backend)
577 89b2b908 Christos Stavrakakis
            self.log.info("F: Issued OP_NETWORK_CONNECT")
578 89b2b908 Christos Stavrakakis
579 89b2b908 Christos Stavrakakis
    def reconcile_hanging_groups(self, network, backend, hanging_groups):
580 89b2b908 Christos Stavrakakis
        self.log.info('D: Network %s in backend %s is not connected to '
581 89b2b908 Christos Stavrakakis
                      'the following groups:', network, backend)
582 89b2b908 Christos Stavrakakis
        self.log.info('-  ' + '\n-  '.join(hanging_groups))
583 89b2b908 Christos Stavrakakis
        if self.fix:
584 89b2b908 Christos Stavrakakis
            for group in hanging_groups:
585 89b2b908 Christos Stavrakakis
                self.log.info('F: Connecting network %s to nodegroup %s',
586 89b2b908 Christos Stavrakakis
                              network, group)
587 89b2b908 Christos Stavrakakis
                backend_mod.connect_network(network, backend, depends=[],
588 89b2b908 Christos Stavrakakis
                                            group=group)
589 89b2b908 Christos Stavrakakis
590 89b2b908 Christos Stavrakakis
    def reconcile_unsynced_network(self, network, backend, backend_network):
591 89b2b908 Christos Stavrakakis
        self.log.info("D: Unsynced network %s in backend %s", network, backend)
592 89b2b908 Christos Stavrakakis
        if self.fix:
593 89b2b908 Christos Stavrakakis
            self.log.info("F: Issuing OP_NETWORK_CONNECT")
594 89b2b908 Christos Stavrakakis
            backend_mod.process_network_status(
595 89b2b908 Christos Stavrakakis
                backend_network, self.event_time, 0,
596 89b2b908 Christos Stavrakakis
                "OP_NETWORK_CONNECT",
597 89b2b908 Christos Stavrakakis
                "success",
598 89b2b908 Christos Stavrakakis
                "Reconciliation simulated eventd")
599 89b2b908 Christos Stavrakakis
600 89b2b908 Christos Stavrakakis
    def reconcile_ip_pools(self, network, available_maps, reserved_maps):
601 89b2b908 Christos Stavrakakis
        available_map = reduce(lambda x, y: x & y, available_maps)
602 89b2b908 Christos Stavrakakis
        reserved_map = reduce(lambda x, y: x & y, reserved_maps)
603 89b2b908 Christos Stavrakakis
604 89b2b908 Christos Stavrakakis
        pool = network.get_pool()
605 89b2b908 Christos Stavrakakis
        # Temporary release unused floating IPs
606 89b2b908 Christos Stavrakakis
        temp_pool = network.get_pool()
607 89b2b908 Christos Stavrakakis
        used_ips = network.nics.values_list("ipv4", flat=True)
608 89b2b908 Christos Stavrakakis
        unused_static_ips = network.floating_ips.exclude(ipv4__in=used_ips)
609 89b2b908 Christos Stavrakakis
        map(lambda ip: temp_pool.put(ip.ipv4), unused_static_ips)
610 89b2b908 Christos Stavrakakis
        if temp_pool.available != available_map:
611 89b2b908 Christos Stavrakakis
            self.log.info("D: Unsynced available map of network %s:\n"
612 89b2b908 Christos Stavrakakis
                          "\tDB: %r\n\tGB: %r", network,
613 89b2b908 Christos Stavrakakis
                          temp_pool.available.to01(),
614 89b2b908 Christos Stavrakakis
                          available_map.to01())
615 89b2b908 Christos Stavrakakis
            if self.fix:
616 89b2b908 Christos Stavrakakis
                pool.available = available_map
617 89b2b908 Christos Stavrakakis
                # Release unsued floating IPs, as they are not included in the
618 89b2b908 Christos Stavrakakis
                # available map
619 89b2b908 Christos Stavrakakis
                map(lambda ip: pool.reserve(ip.ipv4), unused_static_ips)
620 89b2b908 Christos Stavrakakis
                pool.save()
621 89b2b908 Christos Stavrakakis
        if pool.reserved != reserved_map:
622 89b2b908 Christos Stavrakakis
            self.log.info("D: Unsynced reserved map of network %s:\n"
623 89b2b908 Christos Stavrakakis
                          "\tDB: %r\n\tGB: %r", network, pool.reserved.to01(),
624 89b2b908 Christos Stavrakakis
                          reserved_map.to01())
625 89b2b908 Christos Stavrakakis
            if self.fix:
626 89b2b908 Christos Stavrakakis
                pool.reserved = reserved_map
627 89b2b908 Christos Stavrakakis
                pool.save()
628 89b2b908 Christos Stavrakakis
629 89b2b908 Christos Stavrakakis
    def detect_conflicting_ips(self, network):
630 89b2b908 Christos Stavrakakis
        """Detect NIC's that have the same IP in the same network."""
631 89b2b908 Christos Stavrakakis
        machine_ips = network.nics.all().values_list('ipv4', 'machine')
632 89b2b908 Christos Stavrakakis
        ips = map(lambda x: x[0], machine_ips)
633 89b2b908 Christos Stavrakakis
        distinct_ips = set(ips)
634 89b2b908 Christos Stavrakakis
        if len(distinct_ips) < len(ips):
635 89b2b908 Christos Stavrakakis
            for i in distinct_ips:
636 89b2b908 Christos Stavrakakis
                ips.remove(i)
637 89b2b908 Christos Stavrakakis
            for i in ips:
638 89b2b908 Christos Stavrakakis
                machines = [utils.id_to_instance_name(x[1])
639 89b2b908 Christos Stavrakakis
                            for x in machine_ips if x[0] == i]
640 89b2b908 Christos Stavrakakis
                self.log.info('D: Conflicting IP:%s Machines: %s',
641 89b2b908 Christos Stavrakakis
                              i, ', '.join(machines))
642 89b2b908 Christos Stavrakakis
643 89b2b908 Christos Stavrakakis
    def reconcile_orphan_networks(self, db_networks, ganeti_networks):
644 89b2b908 Christos Stavrakakis
        # Detect Orphan Networks in Ganeti
645 89b2b908 Christos Stavrakakis
        db_network_ids = set([net.id for net in db_networks])
646 89b2b908 Christos Stavrakakis
        for back_end, ganeti_networks in ganeti_networks.items():
647 89b2b908 Christos Stavrakakis
            ganeti_network_ids = set(ganeti_networks.keys())
648 89b2b908 Christos Stavrakakis
            orphans = ganeti_network_ids - db_network_ids
649 89b2b908 Christos Stavrakakis
650 89b2b908 Christos Stavrakakis
            if len(orphans) > 0:
651 89b2b908 Christos Stavrakakis
                self.log.info('D: Orphan Networks in backend %s:',
652 89b2b908 Christos Stavrakakis
                              back_end.clustername)
653 89b2b908 Christos Stavrakakis
                self.log.info('-  ' + '\n-  '.join([str(o) for o in orphans]))
654 89b2b908 Christos Stavrakakis
                if self.fix:
655 89b2b908 Christos Stavrakakis
                    for net_id in orphans:
656 89b2b908 Christos Stavrakakis
                        self.log.info('Disconnecting and deleting network %d',
657 89b2b908 Christos Stavrakakis
                                      net_id)
658 89b2b908 Christos Stavrakakis
                        try:
659 89b2b908 Christos Stavrakakis
                            network = Network.objects.get(id=net_id)
660 89b2b908 Christos Stavrakakis
                            backend_mod.delete_network(network,
661 89b2b908 Christos Stavrakakis
                                                       backend=back_end)
662 89b2b908 Christos Stavrakakis
                        except Network.DoesNotExist:
663 89b2b908 Christos Stavrakakis
                            self.log.info("Not entry for network %s in DB !!",
664 89b2b908 Christos Stavrakakis
                                          net_id)
665 89b2b908 Christos Stavrakakis
666 89b2b908 Christos Stavrakakis
667 89b2b908 Christos Stavrakakis
def get_backend_network(network, backend):
668 89b2b908 Christos Stavrakakis
    try:
669 89b2b908 Christos Stavrakakis
        return BackendNetwork.objects.get(network=network, backend=backend)
670 89b2b908 Christos Stavrakakis
    except BackendNetwork.DoesNotExist:
671 89b2b908 Christos Stavrakakis
        return None
672 89b2b908 Christos Stavrakakis
673 89b2b908 Christos Stavrakakis
674 89b2b908 Christos Stavrakakis
def get_network_pool(gnet):
675 89b2b908 Christos Stavrakakis
    """Return available and reserved IP maps.
676 89b2b908 Christos Stavrakakis

677 89b2b908 Christos Stavrakakis
    Extract the available and reserved IP map from the info return from Ganeti
678 89b2b908 Christos Stavrakakis
    for a network.
679 89b2b908 Christos Stavrakakis

680 89b2b908 Christos Stavrakakis
    """
681 89b2b908 Christos Stavrakakis
    converter = IPPool(Foo(gnet['network']))
682 89b2b908 Christos Stavrakakis
    a_map = bitarray_from_map(gnet['map'])
683 89b2b908 Christos Stavrakakis
    a_map.invert()
684 89b2b908 Christos Stavrakakis
    reserved = gnet['external_reservations']
685 89b2b908 Christos Stavrakakis
    r_map = a_map.copy()
686 89b2b908 Christos Stavrakakis
    r_map.setall(True)
687 89b2b908 Christos Stavrakakis
    if reserved:
688 89b2b908 Christos Stavrakakis
        for address in reserved.split(','):
689 89b2b908 Christos Stavrakakis
            index = converter.value_to_index(address.strip())
690 89b2b908 Christos Stavrakakis
            a_map[index] = True
691 89b2b908 Christos Stavrakakis
            r_map[index] = False
692 89b2b908 Christos Stavrakakis
    return a_map, r_map
693 89b2b908 Christos Stavrakakis
694 89b2b908 Christos Stavrakakis
695 89b2b908 Christos Stavrakakis
def bitarray_from_map(bitmap):
696 89b2b908 Christos Stavrakakis
    return bitarray.bitarray(bitmap.replace("X", "1").replace(".", "0"))
697 89b2b908 Christos Stavrakakis
698 89b2b908 Christos Stavrakakis
699 89b2b908 Christos Stavrakakis
class Foo():
700 89b2b908 Christos Stavrakakis
    def __init__(self, subnet):
701 89b2b908 Christos Stavrakakis
        self.available_map = ''
702 89b2b908 Christos Stavrakakis
        self.reserved_map = ''
703 89b2b908 Christos Stavrakakis
        self.size = 0
704 89b2b908 Christos Stavrakakis
        self.network = Foo.Foo1(subnet)
705 89b2b908 Christos Stavrakakis
706 89b2b908 Christos Stavrakakis
    class Foo1():
707 89b2b908 Christos Stavrakakis
        def __init__(self, subnet):
708 89b2b908 Christos Stavrakakis
            self.subnet = subnet
709 89b2b908 Christos Stavrakakis
            self.gateway = None