Statistics
| Branch: | Tag: | Revision:

root / snf-cyclades-app / synnefo / logic / reconciliation.py @ 0d069390

History | View | Annotate | Download (30.4 kB)

1 9fea53cc Vangelis Koukis
# -*- coding: utf-8 -*-
2 9fea53cc Vangelis Koukis
#
3 75dc539e Christos Stavrakakis
# Copyright 2011-2013 GRNET S.A. All rights reserved.
4 9fea53cc Vangelis Koukis
#
5 9fea53cc Vangelis Koukis
# Redistribution and use in source and binary forms, with or
6 9fea53cc Vangelis Koukis
# without modification, are permitted provided that the following
7 9fea53cc Vangelis Koukis
# conditions are met:
8 9fea53cc Vangelis Koukis
#
9 9fea53cc Vangelis Koukis
#   1. Redistributions of source code must retain the above
10 9fea53cc Vangelis Koukis
#      copyright notice, this list of conditions and the following
11 9fea53cc Vangelis Koukis
#      disclaimer.
12 9fea53cc Vangelis Koukis
#
13 9fea53cc Vangelis Koukis
#   2. Redistributions in binary form must reproduce the above
14 9fea53cc Vangelis Koukis
#      copyright notice, this list of conditions and the following
15 9fea53cc Vangelis Koukis
#      disclaimer in the documentation and/or other materials
16 9fea53cc Vangelis Koukis
#      provided with the distribution.
17 9fea53cc Vangelis Koukis
#
18 9fea53cc Vangelis Koukis
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19 9fea53cc Vangelis Koukis
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 9fea53cc Vangelis Koukis
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 9fea53cc Vangelis Koukis
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22 9fea53cc Vangelis Koukis
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 9fea53cc Vangelis Koukis
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 9fea53cc Vangelis Koukis
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25 9fea53cc Vangelis Koukis
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 9fea53cc Vangelis Koukis
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 9fea53cc Vangelis Koukis
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 9fea53cc Vangelis Koukis
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 9fea53cc Vangelis Koukis
# POSSIBILITY OF SUCH DAMAGE.
30 9fea53cc Vangelis Koukis
#
31 9fea53cc Vangelis Koukis
# The views and conclusions contained in the software and
32 9fea53cc Vangelis Koukis
# documentation are those of the authors and should not be
33 9fea53cc Vangelis Koukis
# interpreted as representing official policies, either expressed
34 9fea53cc Vangelis Koukis
# or implied, of GRNET S.A.
35 9fea53cc Vangelis Koukis
#
36 9fea53cc Vangelis Koukis
"""Business logic for reconciliation
37 9fea53cc Vangelis Koukis

38 9fea53cc Vangelis Koukis
Reconcile the contents of the DB with the actual state of the
39 9fea53cc Vangelis Koukis
Ganeti backend.
40 9fea53cc Vangelis Koukis

41 9fea53cc Vangelis Koukis
Let D be the set of VMs in the DB, G the set of VMs in Ganeti.
42 9fea53cc Vangelis Koukis
RULES:
43 9fea53cc Vangelis Koukis
    R1. Stale servers in DB:
44 9fea53cc Vangelis Koukis
            For any v in D but not in G:
45 9fea53cc Vangelis Koukis
            Set deleted=True.
46 9fea53cc Vangelis Koukis
    R2. Orphan instances in Ganet:
47 9fea53cc Vangelis Koukis
            For any v in G with deleted=True in D:
48 9fea53cc Vangelis Koukis
            Issue OP_INSTANCE_DESTROY.
49 9fea53cc Vangelis Koukis
    R3. Unsynced operstate:
50 9fea53cc Vangelis Koukis
            For any v whose operating state differs between G and V:
51 9fea53cc Vangelis Koukis
            Set the operating state in D based on the state in G.
52 9fea53cc Vangelis Koukis
In the code, D, G are Python dicts mapping instance ids to operating state.
53 9fea53cc Vangelis Koukis
For D, the operating state is chosen from VirtualMachine.OPER_STATES.
54 9fea53cc Vangelis Koukis
For G, the operating state is True if the machine is up, False otherwise.
55 9fea53cc Vangelis Koukis

56 9fea53cc Vangelis Koukis
"""
57 9fea53cc Vangelis Koukis
58 9fea53cc Vangelis Koukis
59 0c09b1c0 Christos Stavrakakis
from django.conf import settings
60 4161cb41 Christos Stavrakakis
61 75dc539e Christos Stavrakakis
import logging
62 75dc539e Christos Stavrakakis
import itertools
63 89b2b908 Christos Stavrakakis
import bitarray
64 0d069390 Christos Stavrakakis
from datetime import datetime
65 4161cb41 Christos Stavrakakis
66 75dc539e Christos Stavrakakis
from django.db import transaction
67 75dc539e Christos Stavrakakis
from synnefo.db.models import (Backend, VirtualMachine, Flavor,
68 89b2b908 Christos Stavrakakis
                               pooled_rapi_client, Network,
69 0ccb6461 Christos Stavrakakis
                               BackendNetwork, BridgePoolTable,
70 0ccb6461 Christos Stavrakakis
                               MacPrefixPoolTable)
71 0ccb6461 Christos Stavrakakis
from synnefo.db import pools
72 75dc539e Christos Stavrakakis
from synnefo.logic import utils, backend as backend_mod
73 9fea53cc Vangelis Koukis
74 75dc539e Christos Stavrakakis
logger = logging.getLogger()
75 75dc539e Christos Stavrakakis
logging.basicConfig()
76 9e98ba3c Giorgos Verigakis
77 e63050ca Christos Stavrakakis
try:
78 e63050ca Christos Stavrakakis
    CHECK_INTERVAL = settings.RECONCILIATION_CHECK_INTERVAL
79 e63050ca Christos Stavrakakis
except AttributeError:
80 e63050ca Christos Stavrakakis
    CHECK_INTERVAL = 60
81 e63050ca Christos Stavrakakis
82 63f9eb8e Christos Stavrakakis
GANETI_JOB_ERROR = "error"
83 63f9eb8e Christos Stavrakakis
GANETI_JOBS_PENDING = ["queued", "waiting", "running", "canceling"]
84 63f9eb8e Christos Stavrakakis
GANETI_JOBS_FINALIZED = ["success", "error", "canceled"]
85 63f9eb8e Christos Stavrakakis
86 e63050ca Christos Stavrakakis
87 75dc539e Christos Stavrakakis
class BackendReconciler(object):
88 75dc539e Christos Stavrakakis
    def __init__(self, backend, logger, options=None):
89 75dc539e Christos Stavrakakis
        self.backend = backend
90 75dc539e Christos Stavrakakis
        self.log = logger
91 75dc539e Christos Stavrakakis
        self.client = backend.get_client()
92 75dc539e Christos Stavrakakis
        if options is None:
93 75dc539e Christos Stavrakakis
            self.options = {}
94 4161cb41 Christos Stavrakakis
        else:
95 75dc539e Christos Stavrakakis
            self.options = options
96 75dc539e Christos Stavrakakis
97 75dc539e Christos Stavrakakis
    def close(self):
98 75dc539e Christos Stavrakakis
        self.backend.put_client(self.client)
99 75dc539e Christos Stavrakakis
100 75dc539e Christos Stavrakakis
    @transaction.commit_on_success
101 75dc539e Christos Stavrakakis
    def reconcile(self):
102 75dc539e Christos Stavrakakis
        log = self.log
103 75dc539e Christos Stavrakakis
        backend = self.backend
104 75dc539e Christos Stavrakakis
        log.debug("Reconciling backend %s", backend)
105 75dc539e Christos Stavrakakis
106 75dc539e Christos Stavrakakis
        self.db_servers = get_database_servers(backend)
107 75dc539e Christos Stavrakakis
        self.db_servers_keys = set(self.db_servers.keys())
108 75dc539e Christos Stavrakakis
        log.debug("Got servers info from database.")
109 75dc539e Christos Stavrakakis
110 75dc539e Christos Stavrakakis
        self.gnt_servers = get_ganeti_servers(backend)
111 75dc539e Christos Stavrakakis
        self.gnt_servers_keys = set(self.gnt_servers.keys())
112 75dc539e Christos Stavrakakis
        log.debug("Got servers info from Ganeti backend.")
113 75dc539e Christos Stavrakakis
114 63f9eb8e Christos Stavrakakis
        self.gnt_jobs = get_ganeti_jobs(backend)
115 63f9eb8e Christos Stavrakakis
        log.debug("Got jobs from Ganeti backend")
116 63f9eb8e Christos Stavrakakis
117 75dc539e Christos Stavrakakis
        self.event_time = datetime.now()
118 75dc539e Christos Stavrakakis
119 75dc539e Christos Stavrakakis
        self.stale_servers = self.reconcile_stale_servers()
120 75dc539e Christos Stavrakakis
        self.orphan_servers = self.reconcile_orphan_servers()
121 75dc539e Christos Stavrakakis
        self.unsynced_servers = self.reconcile_unsynced_servers()
122 75dc539e Christos Stavrakakis
        self.close()
123 75dc539e Christos Stavrakakis
124 75dc539e Christos Stavrakakis
    def get_build_status(self, db_server):
125 63f9eb8e Christos Stavrakakis
        job_id = db_server.backendjobid
126 63f9eb8e Christos Stavrakakis
        if job_id in self.gnt_jobs:
127 63f9eb8e Christos Stavrakakis
            gnt_job_status = self.gnt_jobs[job_id]["status"]
128 63f9eb8e Christos Stavrakakis
            if gnt_job_status == GANETI_JOB_ERROR:
129 63f9eb8e Christos Stavrakakis
                return "ERROR"
130 63f9eb8e Christos Stavrakakis
            elif gnt_job_status not in GANETI_JOBS_FINALIZED:
131 75dc539e Christos Stavrakakis
                return "RUNNING"
132 75dc539e Christos Stavrakakis
            else:
133 63f9eb8e Christos Stavrakakis
                return "FINALIZED"
134 75dc539e Christos Stavrakakis
        else:
135 63f9eb8e Christos Stavrakakis
            return "ERROR"
136 75dc539e Christos Stavrakakis
137 75dc539e Christos Stavrakakis
    def reconcile_stale_servers(self):
138 75dc539e Christos Stavrakakis
        # Detect stale servers
139 75dc539e Christos Stavrakakis
        stale = []
140 75dc539e Christos Stavrakakis
        stale_keys = self.db_servers_keys - self.gnt_servers_keys
141 75dc539e Christos Stavrakakis
        for server_id in stale_keys:
142 75dc539e Christos Stavrakakis
            db_server = self.db_servers[server_id]
143 75dc539e Christos Stavrakakis
            if db_server.operstate == "BUILD":
144 75dc539e Christos Stavrakakis
                build_status = self.get_build_status(db_server)
145 75dc539e Christos Stavrakakis
                if build_status == "ERROR":
146 75dc539e Christos Stavrakakis
                    # Special handling of BUILD eerrors
147 75dc539e Christos Stavrakakis
                    self.reconcile_building_server(db_server)
148 75dc539e Christos Stavrakakis
                elif build_status != "RUNNING":
149 75dc539e Christos Stavrakakis
                    stale.append(server_id)
150 1464a17b Christos Stavrakakis
            elif (db_server.operstate == "ERROR" and
151 1464a17b Christos Stavrakakis
                  db_server.action != "DESTROY"):
152 1464a17b Christos Stavrakakis
                # Servers at building ERROR are stale only if the user has
153 1464a17b Christos Stavrakakis
                # asked to destroy them.
154 1464a17b Christos Stavrakakis
                pass
155 75dc539e Christos Stavrakakis
            else:
156 75dc539e Christos Stavrakakis
                stale.append(server_id)
157 75dc539e Christos Stavrakakis
158 75dc539e Christos Stavrakakis
        # Report them
159 75dc539e Christos Stavrakakis
        if stale:
160 75dc539e Christos Stavrakakis
            self.log.info("Found stale servers %s at backend %s",
161 75dc539e Christos Stavrakakis
                          ", ".join(map(str, stale)), self.backend)
162 75dc539e Christos Stavrakakis
        else:
163 75dc539e Christos Stavrakakis
            self.log.debug("No stale servers at backend %s", self.backend)
164 75dc539e Christos Stavrakakis
165 75dc539e Christos Stavrakakis
        # Fix them
166 75dc539e Christos Stavrakakis
        if stale and self.options["fix_stale"]:
167 75dc539e Christos Stavrakakis
            for server_id in stale:
168 75dc539e Christos Stavrakakis
                db_server = self.db_servers[server_id]
169 75dc539e Christos Stavrakakis
                backend_mod.process_op_status(
170 75dc539e Christos Stavrakakis
                    vm=db_server,
171 75dc539e Christos Stavrakakis
                    etime=self.event_time,
172 75dc539e Christos Stavrakakis
                    jobid=-0,
173 75dc539e Christos Stavrakakis
                    opcode='OP_INSTANCE_REMOVE', status='success',
174 75dc539e Christos Stavrakakis
                    logmsg='Reconciliation: simulated Ganeti event')
175 75dc539e Christos Stavrakakis
            self.log.debug("Simulated Ganeti removal for stale servers.")
176 75dc539e Christos Stavrakakis
177 75dc539e Christos Stavrakakis
    def reconcile_orphan_servers(self):
178 75dc539e Christos Stavrakakis
        orphans = self.gnt_servers_keys - self.db_servers_keys
179 75dc539e Christos Stavrakakis
        if orphans:
180 75dc539e Christos Stavrakakis
            self.log.info("Found orphan servers %s at backend %s",
181 75dc539e Christos Stavrakakis
                          ", ".join(map(str, orphans)), self.backend)
182 75dc539e Christos Stavrakakis
        else:
183 75dc539e Christos Stavrakakis
            self.log.debug("No orphan servers at backend %s", self.backend)
184 75dc539e Christos Stavrakakis
185 75dc539e Christos Stavrakakis
        if orphans and self.options["fix_orphans"]:
186 75dc539e Christos Stavrakakis
            for server_id in orphans:
187 75dc539e Christos Stavrakakis
                server_name = utils.id_to_instance_name(server_id)
188 75dc539e Christos Stavrakakis
                self.client.DeleteInstance(server_name)
189 75dc539e Christos Stavrakakis
            self.log.debug("Issued OP_INSTANCE_REMOVE for orphan servers.")
190 75dc539e Christos Stavrakakis
191 75dc539e Christos Stavrakakis
    def reconcile_unsynced_servers(self):
192 75dc539e Christos Stavrakakis
        #log = self.log
193 75dc539e Christos Stavrakakis
        for server_id in self.db_servers_keys & self.gnt_servers_keys:
194 75dc539e Christos Stavrakakis
            db_server = self.db_servers[server_id]
195 75dc539e Christos Stavrakakis
            gnt_server = self.gnt_servers[server_id]
196 75dc539e Christos Stavrakakis
            if db_server.operstate == "BUILD":
197 75dc539e Christos Stavrakakis
                build_status = self.get_build_status(db_server)
198 75dc539e Christos Stavrakakis
                if build_status == "RUNNING":
199 75dc539e Christos Stavrakakis
                    # Do not reconcile building VMs
200 75dc539e Christos Stavrakakis
                    continue
201 75dc539e Christos Stavrakakis
                elif build_status == "ERROR":
202 75dc539e Christos Stavrakakis
                    # Special handling of build errors
203 75dc539e Christos Stavrakakis
                    self.reconcile_building_server(db_server)
204 75dc539e Christos Stavrakakis
                    continue
205 75dc539e Christos Stavrakakis
206 75dc539e Christos Stavrakakis
            self.reconcile_unsynced_operstate(server_id, db_server,
207 75dc539e Christos Stavrakakis
                                              gnt_server)
208 75dc539e Christos Stavrakakis
            self.reconcile_unsynced_flavor(server_id, db_server,
209 75dc539e Christos Stavrakakis
                                           gnt_server)
210 75dc539e Christos Stavrakakis
            self.reconcile_unsynced_nics(server_id, db_server, gnt_server)
211 75dc539e Christos Stavrakakis
            self.reconcile_unsynced_disks(server_id, db_server, gnt_server)
212 63f9eb8e Christos Stavrakakis
            if db_server.task is not None:
213 63f9eb8e Christos Stavrakakis
                self.reconcile_pending_task(server_id, db_server)
214 75dc539e Christos Stavrakakis
215 75dc539e Christos Stavrakakis
    def reconcile_building_server(self, db_server):
216 75dc539e Christos Stavrakakis
        self.log.info("Server '%s' is BUILD in DB, but 'ERROR' in Ganeti.",
217 75dc539e Christos Stavrakakis
                      db_server.id)
218 75dc539e Christos Stavrakakis
        if self.options["fix_unsynced"]:
219 75dc539e Christos Stavrakakis
            fix_opcode = "OP_INSTANCE_CREATE"
220 75dc539e Christos Stavrakakis
            backend_mod.process_op_status(
221 75dc539e Christos Stavrakakis
                vm=db_server,
222 75dc539e Christos Stavrakakis
                etime=self.event_time,
223 75dc539e Christos Stavrakakis
                jobid=-0,
224 75dc539e Christos Stavrakakis
                opcode=fix_opcode, status='error',
225 75dc539e Christos Stavrakakis
                logmsg='Reconciliation: simulated Ganeti event')
226 75dc539e Christos Stavrakakis
            self.log.debug("Simulated Ganeti error build event for"
227 75dc539e Christos Stavrakakis
                           " server '%s'", db_server.id)
228 75dc539e Christos Stavrakakis
229 75dc539e Christos Stavrakakis
    def reconcile_unsynced_operstate(self, server_id, db_server, gnt_server):
230 75dc539e Christos Stavrakakis
        if db_server.operstate != gnt_server["state"]:
231 75dc539e Christos Stavrakakis
            self.log.info("Server '%s' is '%s' in DB and '%s' in Ganeti.",
232 75dc539e Christos Stavrakakis
                          server_id, db_server.operstate, gnt_server["state"])
233 75dc539e Christos Stavrakakis
            if self.options["fix_unsynced"]:
234 5d3e597a Christos Stavrakakis
                # If server is in building state, you will have first to
235 5d3e597a Christos Stavrakakis
                # reconcile it's creation, to avoid wrong quotas
236 5d3e597a Christos Stavrakakis
                if db_server.operstate == "BUILD":
237 5d3e597a Christos Stavrakakis
                    backend_mod.process_op_status(
238 5d3e597a Christos Stavrakakis
                        vm=db_server, etime=self.event_time, jobid=-0,
239 5d3e597a Christos Stavrakakis
                        opcode="OP_INSTANCE_CREATE", status='success',
240 5d3e597a Christos Stavrakakis
                        logmsg='Reconciliation: simulated Ganeti event')
241 5d3e597a Christos Stavrakakis
                fix_opcode = "OP_INSTANCE_STARTUP"\
242 5d3e597a Christos Stavrakakis
                    if gnt_server["state"] == "STARTED"\
243 75dc539e Christos Stavrakakis
                    else "OP_INSTANCE_SHUTDOWN"
244 75dc539e Christos Stavrakakis
                backend_mod.process_op_status(
245 5d3e597a Christos Stavrakakis
                    vm=db_server, etime=self.event_time, jobid=-0,
246 75dc539e Christos Stavrakakis
                    opcode=fix_opcode, status='success',
247 75dc539e Christos Stavrakakis
                    logmsg='Reconciliation: simulated Ganeti event')
248 75dc539e Christos Stavrakakis
                self.log.debug("Simulated Ganeti state event for server '%s'",
249 75dc539e Christos Stavrakakis
                               server_id)
250 75dc539e Christos Stavrakakis
251 75dc539e Christos Stavrakakis
    def reconcile_unsynced_flavor(self, server_id, db_server, gnt_server):
252 75dc539e Christos Stavrakakis
        db_flavor = db_server.flavor
253 75dc539e Christos Stavrakakis
        gnt_flavor = gnt_server["flavor"]
254 75dc539e Christos Stavrakakis
        if (db_flavor.ram != gnt_flavor["ram"] or
255 5d3e597a Christos Stavrakakis
           db_flavor.cpu != gnt_flavor["vcpus"]):
256 a67419d8 Christos Stavrakakis
            try:
257 a67419d8 Christos Stavrakakis
                gnt_flavor = Flavor.objects.get(
258 75dc539e Christos Stavrakakis
                    ram=gnt_flavor["ram"],
259 75dc539e Christos Stavrakakis
                    cpu=gnt_flavor["vcpus"],
260 75dc539e Christos Stavrakakis
                    disk=db_flavor.disk,
261 75dc539e Christos Stavrakakis
                    disk_template=db_flavor.disk_template)
262 a67419d8 Christos Stavrakakis
            except Flavor.DoesNotExist:
263 75dc539e Christos Stavrakakis
                self.log.warning("Server '%s' has unknown flavor.", server_id)
264 75dc539e Christos Stavrakakis
                return
265 75dc539e Christos Stavrakakis
266 4111c53e Christos Stavrakakis
            self.log.info("Server '%s' has flavor '%s' in DB and '%s' in"
267 75dc539e Christos Stavrakakis
                          " Ganeti", server_id, db_flavor, gnt_flavor)
268 75dc539e Christos Stavrakakis
            if self.options["fix_unsynced_flavors"]:
269 75dc539e Christos Stavrakakis
                old_state = db_server.operstate
270 75dc539e Christos Stavrakakis
                opcode = "OP_INSTANCE_SET_PARAMS"
271 75dc539e Christos Stavrakakis
                beparams = {"vcpus": gnt_flavor.cpu,
272 75dc539e Christos Stavrakakis
                            "minmem": gnt_flavor.ram,
273 75dc539e Christos Stavrakakis
                            "maxmem": gnt_flavor.ram}
274 75dc539e Christos Stavrakakis
                backend_mod.process_op_status(
275 75dc539e Christos Stavrakakis
                    vm=db_server, etime=self.event_time, jobid=-0,
276 75dc539e Christos Stavrakakis
                    opcode=opcode, status='success',
277 e6fbada1 Christos Stavrakakis
                    job_fields={"beparams": beparams},
278 75dc539e Christos Stavrakakis
                    logmsg='Reconciliation: simulated Ganeti event')
279 75dc539e Christos Stavrakakis
                # process_op_status with beparams will set the vmstate to
280 75dc539e Christos Stavrakakis
                # shutdown. Fix this be returning it to old state
281 75dc539e Christos Stavrakakis
                vm = VirtualMachine.objects.get(pk=server_id)
282 75dc539e Christos Stavrakakis
                vm.operstate = old_state
283 75dc539e Christos Stavrakakis
                vm.save()
284 75dc539e Christos Stavrakakis
                self.log.debug("Simulated Ganeti flavor event for server '%s'",
285 75dc539e Christos Stavrakakis
                               server_id)
286 75dc539e Christos Stavrakakis
287 75dc539e Christos Stavrakakis
    def reconcile_unsynced_nics(self, server_id, db_server, gnt_server):
288 d7ff7f5a Christos Stavrakakis
        building_time = (self.event_time -
289 0d069390 Christos Stavrakakis
                         backend_mod.BUILDING_NIC_TIMEOUT)
290 d7ff7f5a Christos Stavrakakis
        db_nics = db_server.nics.exclude(state="BUILDING",
291 d7ff7f5a Christos Stavrakakis
                                         created__lte=building_time) \
292 0d069390 Christos Stavrakakis
                                .order_by("id")
293 75dc539e Christos Stavrakakis
        gnt_nics = gnt_server["nics"]
294 75dc539e Christos Stavrakakis
        gnt_nics_parsed = backend_mod.process_ganeti_nics(gnt_nics)
295 a1baa42b Christos Stavrakakis
        nics_changed = len(db_nics) != len(gnt_nics)
296 a1baa42b Christos Stavrakakis
        for db_nic, gnt_nic in zip(db_nics, sorted(gnt_nics_parsed.items())):
297 a1baa42b Christos Stavrakakis
            gnt_nic_id, gnt_nic = gnt_nic
298 a1baa42b Christos Stavrakakis
            if (db_nic.id == gnt_nic_id) and\
299 a1baa42b Christos Stavrakakis
               backend_mod.nics_are_equal(db_nic, gnt_nic):
300 a1baa42b Christos Stavrakakis
                continue
301 a1baa42b Christos Stavrakakis
            else:
302 a1baa42b Christos Stavrakakis
                nics_changed = True
303 a1baa42b Christos Stavrakakis
                break
304 a1baa42b Christos Stavrakakis
        if nics_changed:
305 a1baa42b Christos Stavrakakis
            msg = "Found unsynced NICs for server '%s'.\n"\
306 a1baa42b Christos Stavrakakis
                  "\tDB:\n\t\t%s\n\tGaneti:\n\t\t%s"
307 a1baa42b Christos Stavrakakis
            db_nics_str = "\n\t\t".join(map(format_db_nic, db_nics))
308 a1baa42b Christos Stavrakakis
            gnt_nics_str = "\n\t\t".join(map(format_gnt_nic,
309 0d069390 Christos Stavrakakis
                                         sorted(gnt_nics_parsed.items())))
310 75dc539e Christos Stavrakakis
            self.log.info(msg, server_id, db_nics_str, gnt_nics_str)
311 75dc539e Christos Stavrakakis
            if self.options["fix_unsynced_nics"]:
312 75dc539e Christos Stavrakakis
                backend_mod.process_net_status(vm=db_server,
313 75dc539e Christos Stavrakakis
                                               etime=self.event_time,
314 75dc539e Christos Stavrakakis
                                               nics=gnt_nics)
315 75dc539e Christos Stavrakakis
316 75dc539e Christos Stavrakakis
    def reconcile_unsynced_disks(self, server_id, db_server, gnt_server):
317 75dc539e Christos Stavrakakis
        pass
318 75dc539e Christos Stavrakakis
319 63f9eb8e Christos Stavrakakis
    def reconcile_pending_task(self, server_id, db_server):
320 63f9eb8e Christos Stavrakakis
        job_id = db_server.task_job_id
321 63f9eb8e Christos Stavrakakis
        pending_task = False
322 63f9eb8e Christos Stavrakakis
        if job_id not in self.gnt_jobs:
323 63f9eb8e Christos Stavrakakis
            pending_task = True
324 63f9eb8e Christos Stavrakakis
        else:
325 ce55f724 Christos Stavrakakis
            gnt_job_status = self.gnt_jobs[job_id]["status"]
326 63f9eb8e Christos Stavrakakis
            if gnt_job_status in GANETI_JOBS_FINALIZED:
327 63f9eb8e Christos Stavrakakis
                pending_task = True
328 63f9eb8e Christos Stavrakakis
329 63f9eb8e Christos Stavrakakis
        if pending_task:
330 63f9eb8e Christos Stavrakakis
            self.log.info("Found server '%s' with pending task: '%s'",
331 63f9eb8e Christos Stavrakakis
                          server_id, db_server.task)
332 160c81a1 Christos Stavrakakis
            if self.options["fix_pending_tasks"]:
333 63f9eb8e Christos Stavrakakis
                db_server.task = None
334 63f9eb8e Christos Stavrakakis
                db_server.task_job_id = None
335 63f9eb8e Christos Stavrakakis
                db_server.save()
336 63f9eb8e Christos Stavrakakis
                self.log.info("Cleared pending task for server '%s", server_id)
337 63f9eb8e Christos Stavrakakis
338 75dc539e Christos Stavrakakis
339 bfd04b01 Christos Stavrakakis
NIC_MSG = ": %s\t".join(["ID", "State", "IP", "Network", "MAC", "Index",
340 bfd04b01 Christos Stavrakakis
                         "Firewall"]) + ": %s"
341 a1baa42b Christos Stavrakakis
342 a1baa42b Christos Stavrakakis
343 75dc539e Christos Stavrakakis
def format_db_nic(nic):
344 a1baa42b Christos Stavrakakis
    return NIC_MSG % (nic.id, nic.state, nic.ipv4, nic.network_id, nic.mac,
345 bfd04b01 Christos Stavrakakis
                      nic.index, nic.firewall_profile)
346 75dc539e Christos Stavrakakis
347 75dc539e Christos Stavrakakis
348 75dc539e Christos Stavrakakis
def format_gnt_nic(nic):
349 a1baa42b Christos Stavrakakis
    nic_name, nic = nic
350 0d069390 Christos Stavrakakis
    return NIC_MSG % (nic_name, nic["state"], nic["ipv4"], nic["network"].id,
351 bfd04b01 Christos Stavrakakis
                      nic["mac"], nic["index"], nic["firewall_profile"])
352 9fea53cc Vangelis Koukis
353 cc92b70f Christos Stavrakakis
354 0e9a423f Christos Stavrakakis
#
355 0e9a423f Christos Stavrakakis
# Networks
356 0e9a423f Christos Stavrakakis
#
357 3524241a Christos Stavrakakis
358 3524241a Christos Stavrakakis
359 0e9a423f Christos Stavrakakis
def get_networks_from_ganeti(backend):
360 44e2c577 Christos Stavrakakis
    prefix = settings.BACKEND_PREFIX_ID + 'net-'
361 0e9a423f Christos Stavrakakis
362 0e9a423f Christos Stavrakakis
    networks = {}
363 3524241a Christos Stavrakakis
    with pooled_rapi_client(backend) as c:
364 3524241a Christos Stavrakakis
        for net in c.GetNetworks(bulk=True):
365 3524241a Christos Stavrakakis
            if net['name'].startswith(prefix):
366 3524241a Christos Stavrakakis
                id = utils.id_from_network_name(net['name'])
367 3524241a Christos Stavrakakis
                networks[id] = net
368 0e9a423f Christos Stavrakakis
369 0e9a423f Christos Stavrakakis
    return networks
370 0e9a423f Christos Stavrakakis
371 0e9a423f Christos Stavrakakis
372 0e9a423f Christos Stavrakakis
def hanging_networks(backend, GNets):
373 0e9a423f Christos Stavrakakis
    """Get networks that are not connected to all Nodegroups.
374 0e9a423f Christos Stavrakakis

375 0e9a423f Christos Stavrakakis
    """
376 0e9a423f Christos Stavrakakis
    def get_network_groups(group_list):
377 0e9a423f Christos Stavrakakis
        groups = set()
378 a9bb2a1a Christos Stavrakakis
        for (name, mode, link) in group_list:
379 a9bb2a1a Christos Stavrakakis
            groups.add(name)
380 0e9a423f Christos Stavrakakis
        return groups
381 0e9a423f Christos Stavrakakis
382 3524241a Christos Stavrakakis
    with pooled_rapi_client(backend) as c:
383 3524241a Christos Stavrakakis
        groups = set(c.GetGroups())
384 0e9a423f Christos Stavrakakis
385 0e9a423f Christos Stavrakakis
    hanging = {}
386 0e9a423f Christos Stavrakakis
    for id, info in GNets.items():
387 0e9a423f Christos Stavrakakis
        group_list = get_network_groups(info['group_list'])
388 0e9a423f Christos Stavrakakis
        if group_list != groups:
389 0e9a423f Christos Stavrakakis
            hanging[id] = groups - group_list
390 0e9a423f Christos Stavrakakis
    return hanging
391 0e9a423f Christos Stavrakakis
392 9fea53cc Vangelis Koukis
393 75dc539e Christos Stavrakakis
def get_online_backends():
394 75dc539e Christos Stavrakakis
    return Backend.objects.filter(offline=False)
395 75dc539e Christos Stavrakakis
396 75dc539e Christos Stavrakakis
397 75dc539e Christos Stavrakakis
def get_database_servers(backend):
398 75dc539e Christos Stavrakakis
    servers = backend.virtual_machines.select_related("nics", "flavor")\
399 75dc539e Christos Stavrakakis
                                      .filter(deleted=False)
400 75dc539e Christos Stavrakakis
    return dict([(s.id, s) for s in servers])
401 75dc539e Christos Stavrakakis
402 75dc539e Christos Stavrakakis
403 75dc539e Christos Stavrakakis
def get_ganeti_servers(backend):
404 75dc539e Christos Stavrakakis
    gnt_instances = backend_mod.get_instances(backend)
405 75dc539e Christos Stavrakakis
    # Filter out non-synnefo instances
406 75dc539e Christos Stavrakakis
    snf_backend_prefix = settings.BACKEND_PREFIX_ID
407 75dc539e Christos Stavrakakis
    gnt_instances = filter(lambda i: i["name"].startswith(snf_backend_prefix),
408 75dc539e Christos Stavrakakis
                           gnt_instances)
409 75dc539e Christos Stavrakakis
    gnt_instances = map(parse_gnt_instance, gnt_instances)
410 75dc539e Christos Stavrakakis
    return dict([(i["id"], i) for i in gnt_instances if i["id"] is not None])
411 75dc539e Christos Stavrakakis
412 75dc539e Christos Stavrakakis
413 75dc539e Christos Stavrakakis
def parse_gnt_instance(instance):
414 75dc539e Christos Stavrakakis
    try:
415 75dc539e Christos Stavrakakis
        instance_id = utils.id_from_instance_name(instance['name'])
416 75dc539e Christos Stavrakakis
    except Exception:
417 75dc539e Christos Stavrakakis
        logger.error("Ignoring instance with malformed name %s",
418 75dc539e Christos Stavrakakis
                     instance['name'])
419 75dc539e Christos Stavrakakis
        return (None, None)
420 75dc539e Christos Stavrakakis
421 75dc539e Christos Stavrakakis
    beparams = instance["beparams"]
422 75dc539e Christos Stavrakakis
423 75dc539e Christos Stavrakakis
    vcpus = beparams["vcpus"]
424 75dc539e Christos Stavrakakis
    ram = beparams["maxmem"]
425 75dc539e Christos Stavrakakis
    state = instance["oper_state"] and "STARTED" or "STOPPED"
426 75dc539e Christos Stavrakakis
427 75dc539e Christos Stavrakakis
    return {
428 75dc539e Christos Stavrakakis
        "id": instance_id,
429 75dc539e Christos Stavrakakis
        "state": state,  # FIX
430 75dc539e Christos Stavrakakis
        "updated": datetime.fromtimestamp(instance["mtime"]),
431 75dc539e Christos Stavrakakis
        "disks": disks_from_instance(instance),
432 75dc539e Christos Stavrakakis
        "nics": nics_from_instance(instance),
433 75dc539e Christos Stavrakakis
        "flavor": {"vcpus": vcpus,
434 75dc539e Christos Stavrakakis
                   "ram": ram},
435 75dc539e Christos Stavrakakis
        "tags": instance["tags"]
436 75dc539e Christos Stavrakakis
    }
437 75dc539e Christos Stavrakakis
438 75dc539e Christos Stavrakakis
439 75dc539e Christos Stavrakakis
def nics_from_instance(i):
440 75dc539e Christos Stavrakakis
    ips = zip(itertools.repeat('ip'), i['nic.ips'])
441 a1baa42b Christos Stavrakakis
    names = zip(itertools.repeat('name'), i['nic.names'])
442 75dc539e Christos Stavrakakis
    macs = zip(itertools.repeat('mac'), i['nic.macs'])
443 a1baa42b Christos Stavrakakis
    networks = zip(itertools.repeat('network'), i['nic.networks.names'])
444 75dc539e Christos Stavrakakis
    # modes = zip(itertools.repeat('mode'), i['nic.modes'])
445 75dc539e Christos Stavrakakis
    # links = zip(itertools.repeat('link'), i['nic.links'])
446 75dc539e Christos Stavrakakis
    # nics = zip(ips,macs,modes,networks,links)
447 a1baa42b Christos Stavrakakis
    nics = zip(ips, names, macs, networks)
448 75dc539e Christos Stavrakakis
    nics = map(lambda x: dict(x), nics)
449 75dc539e Christos Stavrakakis
    #nics = dict(enumerate(nics))
450 75dc539e Christos Stavrakakis
    tags = i["tags"]
451 75dc539e Christos Stavrakakis
    for tag in tags:
452 75dc539e Christos Stavrakakis
        t = tag.split(":")
453 75dc539e Christos Stavrakakis
        if t[0:2] == ["synnefo", "network"]:
454 75dc539e Christos Stavrakakis
            if len(t) != 4:
455 75dc539e Christos Stavrakakis
                logger.error("Malformed synefo tag %s", tag)
456 75dc539e Christos Stavrakakis
                continue
457 75dc539e Christos Stavrakakis
            try:
458 75dc539e Christos Stavrakakis
                index = int(t[2])
459 75dc539e Christos Stavrakakis
                nics[index]['firewall'] = t[3]
460 75dc539e Christos Stavrakakis
            except ValueError:
461 75dc539e Christos Stavrakakis
                logger.error("Malformed synnefo tag %s", tag)
462 75dc539e Christos Stavrakakis
            except IndexError:
463 75dc539e Christos Stavrakakis
                logger.error("Found tag %s for non-existent NIC %d",
464 75dc539e Christos Stavrakakis
                             tag, index)
465 75dc539e Christos Stavrakakis
    return nics
466 9fea53cc Vangelis Koukis
467 9fea53cc Vangelis Koukis
468 63f9eb8e Christos Stavrakakis
def get_ganeti_jobs(backend):
469 63f9eb8e Christos Stavrakakis
    gnt_jobs = backend_mod.get_jobs(backend)
470 63f9eb8e Christos Stavrakakis
    return dict([(int(j["id"]), j) for j in gnt_jobs])
471 63f9eb8e Christos Stavrakakis
472 63f9eb8e Christos Stavrakakis
473 75dc539e Christos Stavrakakis
def disks_from_instance(i):
474 75dc539e Christos Stavrakakis
    return dict([(index, {"size": size})
475 75dc539e Christos Stavrakakis
                 for index, size in enumerate(i["disk.sizes"])])
476 89b2b908 Christos Stavrakakis
477 89b2b908 Christos Stavrakakis
478 89b2b908 Christos Stavrakakis
class NetworkReconciler(object):
479 0ccb6461 Christos Stavrakakis
    def __init__(self, logger, fix=False):
480 89b2b908 Christos Stavrakakis
        self.log = logger
481 89b2b908 Christos Stavrakakis
        self.fix = fix
482 89b2b908 Christos Stavrakakis
483 89b2b908 Christos Stavrakakis
    @transaction.commit_on_success
484 89b2b908 Christos Stavrakakis
    def reconcile_networks(self):
485 89b2b908 Christos Stavrakakis
        # Get models from DB
486 0ccb6461 Christos Stavrakakis
        self.backends = Backend.objects.exclude(offline=True)
487 0ccb6461 Christos Stavrakakis
        self.networks = Network.objects.filter(deleted=False)
488 89b2b908 Christos Stavrakakis
489 89b2b908 Christos Stavrakakis
        self.event_time = datetime.now()
490 89b2b908 Christos Stavrakakis
491 89b2b908 Christos Stavrakakis
        # Get info from all ganeti backends
492 0ccb6461 Christos Stavrakakis
        self.ganeti_networks = {}
493 0ccb6461 Christos Stavrakakis
        self.ganeti_hanging_networks = {}
494 0ccb6461 Christos Stavrakakis
        for b in self.backends:
495 89b2b908 Christos Stavrakakis
            g_nets = get_networks_from_ganeti(b)
496 0ccb6461 Christos Stavrakakis
            self.ganeti_networks[b] = g_nets
497 89b2b908 Christos Stavrakakis
            g_hanging_nets = hanging_networks(b, g_nets)
498 0ccb6461 Christos Stavrakakis
            self.ganeti_hanging_networks[b] = g_hanging_nets
499 89b2b908 Christos Stavrakakis
500 0ccb6461 Christos Stavrakakis
        self._reconcile_orphan_networks()
501 89b2b908 Christos Stavrakakis
502 0ccb6461 Christos Stavrakakis
        for network in self.networks:
503 0ccb6461 Christos Stavrakakis
            self._reconcile_network(network)
504 89b2b908 Christos Stavrakakis
505 0ccb6461 Christos Stavrakakis
    @transaction.commit_on_success
506 0ccb6461 Christos Stavrakakis
    def _reconcile_network(self, network):
507 0ccb6461 Christos Stavrakakis
        """Reconcile a network with corresponging Ganeti networks.
508 0ccb6461 Christos Stavrakakis

509 0ccb6461 Christos Stavrakakis
        Reconcile a Network and the associated BackendNetworks with the
510 0ccb6461 Christos Stavrakakis
        corresponding Ganeti networks in all Ganeti backends.
511 0ccb6461 Christos Stavrakakis

512 0ccb6461 Christos Stavrakakis
        """
513 0ccb6461 Christos Stavrakakis
        network_ip_pool = network.get_pool()  # X-Lock on IP Pool
514 0ccb6461 Christos Stavrakakis
        for bend in self.backends:
515 0ccb6461 Christos Stavrakakis
            bnet = get_backend_network(network, bend)
516 0ccb6461 Christos Stavrakakis
            gnet = self.ganeti_networks[bend].get(network.id)
517 0ccb6461 Christos Stavrakakis
            if not bnet:
518 0ccb6461 Christos Stavrakakis
                if network.floating_ip_pool:
519 0ccb6461 Christos Stavrakakis
                    # Network is a floating IP pool and does not exist in
520 0ccb6461 Christos Stavrakakis
                    # backend. We need to create it
521 0ccb6461 Christos Stavrakakis
                    bnet = self.reconcile_parted_network(network, bend)
522 0ccb6461 Christos Stavrakakis
                elif not gnet:
523 0ccb6461 Christos Stavrakakis
                    # Network does not exist either in Ganeti nor in BD.
524 0ccb6461 Christos Stavrakakis
                    continue
525 0ccb6461 Christos Stavrakakis
                else:
526 0ccb6461 Christos Stavrakakis
                    # Network exists in Ganeti and not in DB.
527 0ccb6461 Christos Stavrakakis
                    if network.action != "DESTROY" and not network.public:
528 0ccb6461 Christos Stavrakakis
                        bnet = self.reconcile_parted_network(network, bend)
529 0ccb6461 Christos Stavrakakis
                    else:
530 0ccb6461 Christos Stavrakakis
                        continue
531 89b2b908 Christos Stavrakakis
532 0ccb6461 Christos Stavrakakis
            if not gnet:
533 0ccb6461 Christos Stavrakakis
                # Network does not exist in Ganeti. If the network action
534 0ccb6461 Christos Stavrakakis
                # is DESTROY, we have to mark as deleted in DB, else we
535 0ccb6461 Christos Stavrakakis
                # have to create it in Ganeti.
536 0ccb6461 Christos Stavrakakis
                if network.action == "DESTROY":
537 0ccb6461 Christos Stavrakakis
                    if bnet.operstate != "DELETED":
538 0ccb6461 Christos Stavrakakis
                        self.reconcile_stale_network(bnet)
539 0ccb6461 Christos Stavrakakis
                else:
540 0ccb6461 Christos Stavrakakis
                    self.reconcile_missing_network(network, bend)
541 0ccb6461 Christos Stavrakakis
                # Skip rest reconciliation!
542 0ccb6461 Christos Stavrakakis
                continue
543 89b2b908 Christos Stavrakakis
544 0ccb6461 Christos Stavrakakis
            try:
545 0ccb6461 Christos Stavrakakis
                hanging_groups = self.ganeti_hanging_networks[bend][network.id]
546 0ccb6461 Christos Stavrakakis
            except KeyError:
547 0ccb6461 Christos Stavrakakis
                # Network is connected to all nodegroups
548 0ccb6461 Christos Stavrakakis
                hanging_groups = []
549 0ccb6461 Christos Stavrakakis
550 0ccb6461 Christos Stavrakakis
            if hanging_groups:
551 0ccb6461 Christos Stavrakakis
                # CASE-3: Ganeti networks not connected to all nodegroups
552 0ccb6461 Christos Stavrakakis
                self.reconcile_hanging_groups(network, bend,
553 0ccb6461 Christos Stavrakakis
                                              hanging_groups)
554 0ccb6461 Christos Stavrakakis
                continue
555 89b2b908 Christos Stavrakakis
556 0ccb6461 Christos Stavrakakis
            if bnet.operstate != 'ACTIVE':
557 0ccb6461 Christos Stavrakakis
                # CASE-4: Unsynced network state. At this point the network
558 0ccb6461 Christos Stavrakakis
                # exists and is connected to all nodes so is must be
559 0ccb6461 Christos Stavrakakis
                # active!
560 0ccb6461 Christos Stavrakakis
                self.reconcile_unsynced_network(network, bend, bnet)
561 0ccb6461 Christos Stavrakakis
562 0ccb6461 Christos Stavrakakis
            # Check that externally reserved IPs of the network in Ganeti are
563 0ccb6461 Christos Stavrakakis
            # also externally reserved to the IP pool
564 0ccb6461 Christos Stavrakakis
            externally_reserved = gnet['external_reservations']
565 98a01362 Christos Stavrakakis
            if externally_reserved:
566 98a01362 Christos Stavrakakis
                for ip in externally_reserved.split(","):
567 98a01362 Christos Stavrakakis
                    ip = ip.strip()
568 98a01362 Christos Stavrakakis
                    if not network_ip_pool.is_reserved(ip):
569 98a01362 Christos Stavrakakis
                        msg = ("D: IP '%s' is reserved for network '%s' in"
570 98a01362 Christos Stavrakakis
                               " backend '%s' but not in DB.")
571 98a01362 Christos Stavrakakis
                        self.log.info(msg, ip, network, bend)
572 98a01362 Christos Stavrakakis
                        if self.fix:
573 98a01362 Christos Stavrakakis
                            network_ip_pool.reserve(ip, external=True)
574 98a01362 Christos Stavrakakis
                            network_ip_pool.save()
575 98a01362 Christos Stavrakakis
                            self.log.info("F: Reserved IP '%s'", ip)
576 89b2b908 Christos Stavrakakis
577 89b2b908 Christos Stavrakakis
    def reconcile_parted_network(self, network, backend):
578 89b2b908 Christos Stavrakakis
        self.log.info("D: Missing DB entry for network %s in backend %s",
579 89b2b908 Christos Stavrakakis
                      network, backend)
580 89b2b908 Christos Stavrakakis
        if self.fix:
581 89b2b908 Christos Stavrakakis
            network.create_backend_network(backend)
582 89b2b908 Christos Stavrakakis
            self.log.info("F: Created DB entry")
583 89b2b908 Christos Stavrakakis
            bnet = get_backend_network(network, backend)
584 89b2b908 Christos Stavrakakis
            return bnet
585 89b2b908 Christos Stavrakakis
586 89b2b908 Christos Stavrakakis
    def reconcile_stale_network(self, backend_network):
587 89b2b908 Christos Stavrakakis
        self.log.info("D: Stale DB entry for network %s in backend %s",
588 89b2b908 Christos Stavrakakis
                      backend_network.network, backend_network.backend)
589 89b2b908 Christos Stavrakakis
        if self.fix:
590 89b2b908 Christos Stavrakakis
            backend_mod.process_network_status(
591 89b2b908 Christos Stavrakakis
                backend_network, self.event_time, 0,
592 89b2b908 Christos Stavrakakis
                "OP_NETWORK_REMOVE",
593 89b2b908 Christos Stavrakakis
                "success",
594 89b2b908 Christos Stavrakakis
                "Reconciliation simulated event")
595 89b2b908 Christos Stavrakakis
            self.log.info("F: Reconciled event: OP_NETWORK_REMOVE")
596 89b2b908 Christos Stavrakakis
597 89b2b908 Christos Stavrakakis
    def reconcile_missing_network(self, network, backend):
598 89b2b908 Christos Stavrakakis
        self.log.info("D: Missing Ganeti network %s in backend %s",
599 89b2b908 Christos Stavrakakis
                      network, backend)
600 89b2b908 Christos Stavrakakis
        if self.fix:
601 89b2b908 Christos Stavrakakis
            backend_mod.create_network(network, backend)
602 89b2b908 Christos Stavrakakis
            self.log.info("F: Issued OP_NETWORK_CONNECT")
603 89b2b908 Christos Stavrakakis
604 89b2b908 Christos Stavrakakis
    def reconcile_hanging_groups(self, network, backend, hanging_groups):
605 89b2b908 Christos Stavrakakis
        self.log.info('D: Network %s in backend %s is not connected to '
606 89b2b908 Christos Stavrakakis
                      'the following groups:', network, backend)
607 89b2b908 Christos Stavrakakis
        self.log.info('-  ' + '\n-  '.join(hanging_groups))
608 89b2b908 Christos Stavrakakis
        if self.fix:
609 89b2b908 Christos Stavrakakis
            for group in hanging_groups:
610 89b2b908 Christos Stavrakakis
                self.log.info('F: Connecting network %s to nodegroup %s',
611 89b2b908 Christos Stavrakakis
                              network, group)
612 89b2b908 Christos Stavrakakis
                backend_mod.connect_network(network, backend, depends=[],
613 89b2b908 Christos Stavrakakis
                                            group=group)
614 89b2b908 Christos Stavrakakis
615 89b2b908 Christos Stavrakakis
    def reconcile_unsynced_network(self, network, backend, backend_network):
616 89b2b908 Christos Stavrakakis
        self.log.info("D: Unsynced network %s in backend %s", network, backend)
617 89b2b908 Christos Stavrakakis
        if self.fix:
618 89b2b908 Christos Stavrakakis
            self.log.info("F: Issuing OP_NETWORK_CONNECT")
619 89b2b908 Christos Stavrakakis
            backend_mod.process_network_status(
620 89b2b908 Christos Stavrakakis
                backend_network, self.event_time, 0,
621 89b2b908 Christos Stavrakakis
                "OP_NETWORK_CONNECT",
622 89b2b908 Christos Stavrakakis
                "success",
623 89b2b908 Christos Stavrakakis
                "Reconciliation simulated eventd")
624 89b2b908 Christos Stavrakakis
625 0ccb6461 Christos Stavrakakis
    def _reconcile_orphan_networks(self):
626 0ccb6461 Christos Stavrakakis
        db_networks = self.networks
627 0ccb6461 Christos Stavrakakis
        ganeti_networks = self.ganeti_networks
628 89b2b908 Christos Stavrakakis
        # Detect Orphan Networks in Ganeti
629 89b2b908 Christos Stavrakakis
        db_network_ids = set([net.id for net in db_networks])
630 89b2b908 Christos Stavrakakis
        for back_end, ganeti_networks in ganeti_networks.items():
631 89b2b908 Christos Stavrakakis
            ganeti_network_ids = set(ganeti_networks.keys())
632 89b2b908 Christos Stavrakakis
            orphans = ganeti_network_ids - db_network_ids
633 89b2b908 Christos Stavrakakis
634 89b2b908 Christos Stavrakakis
            if len(orphans) > 0:
635 89b2b908 Christos Stavrakakis
                self.log.info('D: Orphan Networks in backend %s:',
636 89b2b908 Christos Stavrakakis
                              back_end.clustername)
637 89b2b908 Christos Stavrakakis
                self.log.info('-  ' + '\n-  '.join([str(o) for o in orphans]))
638 89b2b908 Christos Stavrakakis
                if self.fix:
639 89b2b908 Christos Stavrakakis
                    for net_id in orphans:
640 89b2b908 Christos Stavrakakis
                        self.log.info('Disconnecting and deleting network %d',
641 89b2b908 Christos Stavrakakis
                                      net_id)
642 89b2b908 Christos Stavrakakis
                        try:
643 89b2b908 Christos Stavrakakis
                            network = Network.objects.get(id=net_id)
644 89b2b908 Christos Stavrakakis
                            backend_mod.delete_network(network,
645 89b2b908 Christos Stavrakakis
                                                       backend=back_end)
646 89b2b908 Christos Stavrakakis
                        except Network.DoesNotExist:
647 89b2b908 Christos Stavrakakis
                            self.log.info("Not entry for network %s in DB !!",
648 89b2b908 Christos Stavrakakis
                                          net_id)
649 89b2b908 Christos Stavrakakis
650 89b2b908 Christos Stavrakakis
651 89b2b908 Christos Stavrakakis
def get_backend_network(network, backend):
652 89b2b908 Christos Stavrakakis
    try:
653 89b2b908 Christos Stavrakakis
        return BackendNetwork.objects.get(network=network, backend=backend)
654 89b2b908 Christos Stavrakakis
    except BackendNetwork.DoesNotExist:
655 89b2b908 Christos Stavrakakis
        return None
656 89b2b908 Christos Stavrakakis
657 89b2b908 Christos Stavrakakis
658 0ccb6461 Christos Stavrakakis
class PoolReconciler(object):
659 0ccb6461 Christos Stavrakakis
    def __init__(self, logger, fix=False):
660 0ccb6461 Christos Stavrakakis
        self.log = logger
661 0ccb6461 Christos Stavrakakis
        self.fix = fix
662 0ccb6461 Christos Stavrakakis
663 0ccb6461 Christos Stavrakakis
    def reconcile(self):
664 0ccb6461 Christos Stavrakakis
        self.reconcile_bridges()
665 0ccb6461 Christos Stavrakakis
        self.reconcile_mac_prefixes()
666 0ccb6461 Christos Stavrakakis
        for network in Network.objects.filter(deleted=False):
667 0ccb6461 Christos Stavrakakis
            self.reconcile_ip_pool(network)
668 0ccb6461 Christos Stavrakakis
669 0ccb6461 Christos Stavrakakis
    @transaction.commit_on_success
670 0ccb6461 Christos Stavrakakis
    def reconcile_bridges(self):
671 0ccb6461 Christos Stavrakakis
        networks = Network.objects.filter(deleted=False,
672 0ccb6461 Christos Stavrakakis
                                          flavor="PHYSICAL_VLAN")
673 0ccb6461 Christos Stavrakakis
        check_unique_values(objects=networks, field='link', logger=self.log)
674 0ccb6461 Christos Stavrakakis
        try:
675 0ccb6461 Christos Stavrakakis
            pool = BridgePoolTable.get_pool()
676 0ccb6461 Christos Stavrakakis
        except pools.EmptyPool:
677 0ccb6461 Christos Stavrakakis
            self.log.info("There is no available pool for bridges.")
678 0ccb6461 Christos Stavrakakis
            return
679 0ccb6461 Christos Stavrakakis
680 0ccb6461 Christos Stavrakakis
        used_bridges = set(networks.values_list('link', flat=True))
681 0ccb6461 Christos Stavrakakis
        check_pool_consistent(pool=pool, pool_class=pools.BridgePool,
682 0ccb6461 Christos Stavrakakis
                              used_values=used_bridges, fix=self.fix,
683 0ccb6461 Christos Stavrakakis
                              logger=self.log)
684 89b2b908 Christos Stavrakakis
685 0ccb6461 Christos Stavrakakis
    @transaction.commit_on_success
686 0ccb6461 Christos Stavrakakis
    def reconcile_mac_prefixes(self):
687 0ccb6461 Christos Stavrakakis
        networks = Network.objects.filter(deleted=False, flavor="MAC_FILTERED")
688 0ccb6461 Christos Stavrakakis
        check_unique_values(objects=networks, field='mac_prefix',
689 0ccb6461 Christos Stavrakakis
                            logger=self.log)
690 0ccb6461 Christos Stavrakakis
        try:
691 0ccb6461 Christos Stavrakakis
            pool = MacPrefixPoolTable.get_pool()
692 0ccb6461 Christos Stavrakakis
        except pools.EmptyPool:
693 0ccb6461 Christos Stavrakakis
            self.log.info("There is no available pool for MAC prefixes.")
694 0ccb6461 Christos Stavrakakis
            return
695 0ccb6461 Christos Stavrakakis
696 0ccb6461 Christos Stavrakakis
        used_mac_prefixes = set(networks.values_list('mac_prefix', flat=True))
697 0ccb6461 Christos Stavrakakis
        check_pool_consistent(pool=pool, pool_class=pools.MacPrefixPool,
698 0ccb6461 Christos Stavrakakis
                              used_values=used_mac_prefixes, fix=self.fix,
699 0ccb6461 Christos Stavrakakis
                              logger=self.log)
700 89b2b908 Christos Stavrakakis
701 0ccb6461 Christos Stavrakakis
    @transaction.commit_on_success
702 0ccb6461 Christos Stavrakakis
    def reconcile_ip_pool(self, network):
703 0ccb6461 Christos Stavrakakis
        # Check that all NICs have unique IPv4 address
704 0ccb6461 Christos Stavrakakis
        nics = network.nics.filter(ipv4__isnull=False)
705 0ccb6461 Christos Stavrakakis
        check_unique_values(objects=nics, field='ipv4', logger=self.log)
706 0ccb6461 Christos Stavrakakis
707 0ccb6461 Christos Stavrakakis
        # Check that all Floating IPs have unique IPv4 address
708 0ccb6461 Christos Stavrakakis
        floating_ips = network.floating_ips.filter(deleted=False)
709 0ccb6461 Christos Stavrakakis
        check_unique_values(objects=floating_ips, field='ipv4',
710 0ccb6461 Christos Stavrakakis
                            logger=self.log)
711 0ccb6461 Christos Stavrakakis
712 0ccb6461 Christos Stavrakakis
        # First get(lock) the IP pool of the network to prevent new NICs
713 0ccb6461 Christos Stavrakakis
        # from being created.
714 0ccb6461 Christos Stavrakakis
        network_ip_pool = network.get_pool()
715 0ccb6461 Christos Stavrakakis
        used_ips = set(list(nics.values_list("ipv4", flat=True)) +
716 0ccb6461 Christos Stavrakakis
                       list(floating_ips.values_list("ipv4", flat=True)))
717 0ccb6461 Christos Stavrakakis
718 0ccb6461 Christos Stavrakakis
        check_pool_consistent(pool=network_ip_pool,
719 0ccb6461 Christos Stavrakakis
                              pool_class=pools.IPPool,
720 0ccb6461 Christos Stavrakakis
                              used_values=used_ips,
721 0ccb6461 Christos Stavrakakis
                              fix=self.fix, logger=self.log)
722 0ccb6461 Christos Stavrakakis
723 0ccb6461 Christos Stavrakakis
724 0ccb6461 Christos Stavrakakis
def check_unique_values(objects, field, logger):
725 0ccb6461 Christos Stavrakakis
    used_values = list(objects.values_list(field, flat=True))
726 0ccb6461 Christos Stavrakakis
    if len(used_values) != len(set(used_values)):
727 0ccb6461 Christos Stavrakakis
        duplicate_values = [v for v in used_values if used_values.count(v) > 1]
728 0ccb6461 Christos Stavrakakis
        for value in duplicate_values:
729 0ccb6461 Christos Stavrakakis
            filter_args = {field: value}
730 0ccb6461 Christos Stavrakakis
            using_objects = objects.filter(**filter_args)
731 0ccb6461 Christos Stavrakakis
            msg = "Value '%s' is used as %s for more than one objects: %s"
732 0ccb6461 Christos Stavrakakis
            logger.error(msg, value, field, ",".join(map(str, using_objects)))
733 0ccb6461 Christos Stavrakakis
        return False
734 0ccb6461 Christos Stavrakakis
    logger.debug("Values for field '%s' are unique.", field)
735 0ccb6461 Christos Stavrakakis
    return True
736 0ccb6461 Christos Stavrakakis
737 0ccb6461 Christos Stavrakakis
738 0ccb6461 Christos Stavrakakis
def check_pool_consistent(pool, pool_class, used_values, fix, logger):
739 0ccb6461 Christos Stavrakakis
    dummy_pool = create_empty_pool(pool, pool_class)
740 0ccb6461 Christos Stavrakakis
    [dummy_pool.reserve(value) for value in used_values]
741 0ccb6461 Christos Stavrakakis
    if dummy_pool.available != pool.available:
742 0ccb6461 Christos Stavrakakis
        msg = "'%s' is not consistent!\nPool: %s\nUsed: %s"
743 0ccb6461 Christos Stavrakakis
        pool_diff = dummy_pool.available ^ pool.available
744 0ccb6461 Christos Stavrakakis
        for index in pool_diff.itersearch(bitarray.bitarray("1")):
745 0ccb6461 Christos Stavrakakis
            value = pool.index_to_value(int(index))
746 0ccb6461 Christos Stavrakakis
            msg = "%s is incosistent! Value '%s' is %s but should be %s."
747 0ccb6461 Christos Stavrakakis
            value1 = pool.is_available(value) and "available" or "unavailable"
748 0ccb6461 Christos Stavrakakis
            value2 = dummy_pool.is_available(value) and "available"\
749 0ccb6461 Christos Stavrakakis
                or "unavailable"
750 0ccb6461 Christos Stavrakakis
            logger.error(msg, pool, value, value1, value2)
751 0ccb6461 Christos Stavrakakis
        if fix:
752 0ccb6461 Christos Stavrakakis
            pool.available = dummy_pool.available
753 0ccb6461 Christos Stavrakakis
            pool.save()
754 0ccb6461 Christos Stavrakakis
            logger.info("Fixed available map of pool '%s'", pool)
755 0ccb6461 Christos Stavrakakis
756 0ccb6461 Christos Stavrakakis
757 0ccb6461 Christos Stavrakakis
def create_empty_pool(pool, pool_class):
758 0ccb6461 Christos Stavrakakis
    pool_row = pool.pool_table
759 0ccb6461 Christos Stavrakakis
    pool_row.available_map = ""
760 0ccb6461 Christos Stavrakakis
    pool_row.reserved_map = ""
761 0ccb6461 Christos Stavrakakis
    return pool_class(pool_row)