root / snf-cyclades-app / synnefo / logic / reconciliation.py @ 90858bda
History | View | Annotate | Download (27.5 kB)
1 | 9fea53cc | Vangelis Koukis | # -*- coding: utf-8 -*-
|
---|---|---|---|
2 | 9fea53cc | Vangelis Koukis | #
|
3 | 75dc539e | Christos Stavrakakis | # Copyright 2011-2013 GRNET S.A. All rights reserved.
|
4 | 9fea53cc | Vangelis Koukis | #
|
5 | 9fea53cc | Vangelis Koukis | # Redistribution and use in source and binary forms, with or
|
6 | 9fea53cc | Vangelis Koukis | # without modification, are permitted provided that the following
|
7 | 9fea53cc | Vangelis Koukis | # conditions are met:
|
8 | 9fea53cc | Vangelis Koukis | #
|
9 | 9fea53cc | Vangelis Koukis | # 1. Redistributions of source code must retain the above
|
10 | 9fea53cc | Vangelis Koukis | # copyright notice, this list of conditions and the following
|
11 | 9fea53cc | Vangelis Koukis | # disclaimer.
|
12 | 9fea53cc | Vangelis Koukis | #
|
13 | 9fea53cc | Vangelis Koukis | # 2. Redistributions in binary form must reproduce the above
|
14 | 9fea53cc | Vangelis Koukis | # copyright notice, this list of conditions and the following
|
15 | 9fea53cc | Vangelis Koukis | # disclaimer in the documentation and/or other materials
|
16 | 9fea53cc | Vangelis Koukis | # provided with the distribution.
|
17 | 9fea53cc | Vangelis Koukis | #
|
18 | 9fea53cc | Vangelis Koukis | # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
|
19 | 9fea53cc | Vangelis Koukis | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
20 | 9fea53cc | Vangelis Koukis | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
21 | 9fea53cc | Vangelis Koukis | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
|
22 | 9fea53cc | Vangelis Koukis | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
23 | 9fea53cc | Vangelis Koukis | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
24 | 9fea53cc | Vangelis Koukis | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
25 | 9fea53cc | Vangelis Koukis | # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
26 | 9fea53cc | Vangelis Koukis | # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
27 | 9fea53cc | Vangelis Koukis | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
28 | 9fea53cc | Vangelis Koukis | # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
29 | 9fea53cc | Vangelis Koukis | # POSSIBILITY OF SUCH DAMAGE.
|
30 | 9fea53cc | Vangelis Koukis | #
|
31 | 9fea53cc | Vangelis Koukis | # The views and conclusions contained in the software and
|
32 | 9fea53cc | Vangelis Koukis | # documentation are those of the authors and should not be
|
33 | 9fea53cc | Vangelis Koukis | # interpreted as representing official policies, either expressed
|
34 | 9fea53cc | Vangelis Koukis | # or implied, of GRNET S.A.
|
35 | 9fea53cc | Vangelis Koukis | #
|
36 | 9fea53cc | Vangelis Koukis | """Business logic for reconciliation
|
37 | 9fea53cc | Vangelis Koukis |
|
38 | 9fea53cc | Vangelis Koukis | Reconcile the contents of the DB with the actual state of the
|
39 | 9fea53cc | Vangelis Koukis | Ganeti backend.
|
40 | 9fea53cc | Vangelis Koukis |
|
41 | 9fea53cc | Vangelis Koukis | Let D be the set of VMs in the DB, G the set of VMs in Ganeti.
|
42 | 9fea53cc | Vangelis Koukis | RULES:
|
43 | 9fea53cc | Vangelis Koukis | R1. Stale servers in DB:
|
44 | 9fea53cc | Vangelis Koukis | For any v in D but not in G:
|
45 | 9fea53cc | Vangelis Koukis | Set deleted=True.
|
46 | 9fea53cc | Vangelis Koukis | R2. Orphan instances in Ganet:
|
47 | 9fea53cc | Vangelis Koukis | For any v in G with deleted=True in D:
|
48 | 9fea53cc | Vangelis Koukis | Issue OP_INSTANCE_DESTROY.
|
49 | 9fea53cc | Vangelis Koukis | R3. Unsynced operstate:
|
50 | 9fea53cc | Vangelis Koukis | For any v whose operating state differs between G and V:
|
51 | 9fea53cc | Vangelis Koukis | Set the operating state in D based on the state in G.
|
52 | 9fea53cc | Vangelis Koukis | In the code, D, G are Python dicts mapping instance ids to operating state.
|
53 | 9fea53cc | Vangelis Koukis | For D, the operating state is chosen from VirtualMachine.OPER_STATES.
|
54 | 9fea53cc | Vangelis Koukis | For G, the operating state is True if the machine is up, False otherwise.
|
55 | 9fea53cc | Vangelis Koukis |
|
56 | 9fea53cc | Vangelis Koukis | """
|
57 | 9fea53cc | Vangelis Koukis | |
58 | 9fea53cc | Vangelis Koukis | |
59 | 0c09b1c0 | Christos Stavrakakis | from django.conf import settings |
60 | 4161cb41 | Christos Stavrakakis | |
61 | 75dc539e | Christos Stavrakakis | import logging |
62 | 75dc539e | Christos Stavrakakis | import itertools |
63 | 89b2b908 | Christos Stavrakakis | import bitarray |
64 | 63f9eb8e | Christos Stavrakakis | from datetime import datetime |
65 | 4161cb41 | Christos Stavrakakis | |
66 | 75dc539e | Christos Stavrakakis | from django.db import transaction |
67 | 75dc539e | Christos Stavrakakis | from synnefo.db.models import (Backend, VirtualMachine, Flavor, |
68 | 89b2b908 | Christos Stavrakakis | pooled_rapi_client, Network, |
69 | 89b2b908 | Christos Stavrakakis | BackendNetwork) |
70 | 89b2b908 | Christos Stavrakakis | from synnefo.db.pools import IPPool |
71 | 75dc539e | Christos Stavrakakis | from synnefo.logic import utils, backend as backend_mod |
72 | 9fea53cc | Vangelis Koukis | |
73 | 75dc539e | Christos Stavrakakis | logger = logging.getLogger() |
74 | 75dc539e | Christos Stavrakakis | logging.basicConfig() |
75 | 9e98ba3c | Giorgos Verigakis | |
76 | e63050ca | Christos Stavrakakis | try:
|
77 | e63050ca | Christos Stavrakakis | CHECK_INTERVAL = settings.RECONCILIATION_CHECK_INTERVAL |
78 | e63050ca | Christos Stavrakakis | except AttributeError: |
79 | e63050ca | Christos Stavrakakis | CHECK_INTERVAL = 60
|
80 | e63050ca | Christos Stavrakakis | |
81 | 63f9eb8e | Christos Stavrakakis | GANETI_JOB_ERROR = "error"
|
82 | 63f9eb8e | Christos Stavrakakis | GANETI_JOBS_PENDING = ["queued", "waiting", "running", "canceling"] |
83 | 63f9eb8e | Christos Stavrakakis | GANETI_JOBS_FINALIZED = ["success", "error", "canceled"] |
84 | 63f9eb8e | Christos Stavrakakis | |
85 | e63050ca | Christos Stavrakakis | |
86 | 75dc539e | Christos Stavrakakis | class BackendReconciler(object): |
87 | 75dc539e | Christos Stavrakakis | def __init__(self, backend, logger, options=None): |
88 | 75dc539e | Christos Stavrakakis | self.backend = backend
|
89 | 75dc539e | Christos Stavrakakis | self.log = logger
|
90 | 75dc539e | Christos Stavrakakis | self.client = backend.get_client()
|
91 | 75dc539e | Christos Stavrakakis | if options is None: |
92 | 75dc539e | Christos Stavrakakis | self.options = {}
|
93 | 4161cb41 | Christos Stavrakakis | else:
|
94 | 75dc539e | Christos Stavrakakis | self.options = options
|
95 | 75dc539e | Christos Stavrakakis | |
96 | 75dc539e | Christos Stavrakakis | def close(self): |
97 | 75dc539e | Christos Stavrakakis | self.backend.put_client(self.client) |
98 | 75dc539e | Christos Stavrakakis | |
99 | 75dc539e | Christos Stavrakakis | @transaction.commit_on_success
|
100 | 75dc539e | Christos Stavrakakis | def reconcile(self): |
101 | 75dc539e | Christos Stavrakakis | log = self.log
|
102 | 75dc539e | Christos Stavrakakis | backend = self.backend
|
103 | 75dc539e | Christos Stavrakakis | log.debug("Reconciling backend %s", backend)
|
104 | 75dc539e | Christos Stavrakakis | |
105 | 75dc539e | Christos Stavrakakis | self.db_servers = get_database_servers(backend)
|
106 | 75dc539e | Christos Stavrakakis | self.db_servers_keys = set(self.db_servers.keys()) |
107 | 75dc539e | Christos Stavrakakis | log.debug("Got servers info from database.")
|
108 | 75dc539e | Christos Stavrakakis | |
109 | 75dc539e | Christos Stavrakakis | self.gnt_servers = get_ganeti_servers(backend)
|
110 | 75dc539e | Christos Stavrakakis | self.gnt_servers_keys = set(self.gnt_servers.keys()) |
111 | 75dc539e | Christos Stavrakakis | log.debug("Got servers info from Ganeti backend.")
|
112 | 75dc539e | Christos Stavrakakis | |
113 | 63f9eb8e | Christos Stavrakakis | self.gnt_jobs = get_ganeti_jobs(backend)
|
114 | 63f9eb8e | Christos Stavrakakis | log.debug("Got jobs from Ganeti backend")
|
115 | 63f9eb8e | Christos Stavrakakis | |
116 | 75dc539e | Christos Stavrakakis | self.event_time = datetime.now()
|
117 | 75dc539e | Christos Stavrakakis | |
118 | 75dc539e | Christos Stavrakakis | self.stale_servers = self.reconcile_stale_servers() |
119 | 75dc539e | Christos Stavrakakis | self.orphan_servers = self.reconcile_orphan_servers() |
120 | 75dc539e | Christos Stavrakakis | self.unsynced_servers = self.reconcile_unsynced_servers() |
121 | 75dc539e | Christos Stavrakakis | self.close()
|
122 | 75dc539e | Christos Stavrakakis | |
123 | 75dc539e | Christos Stavrakakis | def get_build_status(self, db_server): |
124 | 63f9eb8e | Christos Stavrakakis | job_id = db_server.backendjobid |
125 | 63f9eb8e | Christos Stavrakakis | if job_id in self.gnt_jobs: |
126 | 63f9eb8e | Christos Stavrakakis | gnt_job_status = self.gnt_jobs[job_id]["status"] |
127 | 63f9eb8e | Christos Stavrakakis | if gnt_job_status == GANETI_JOB_ERROR:
|
128 | 63f9eb8e | Christos Stavrakakis | return "ERROR" |
129 | 63f9eb8e | Christos Stavrakakis | elif gnt_job_status not in GANETI_JOBS_FINALIZED: |
130 | 75dc539e | Christos Stavrakakis | return "RUNNING" |
131 | 75dc539e | Christos Stavrakakis | else:
|
132 | 63f9eb8e | Christos Stavrakakis | return "FINALIZED" |
133 | 75dc539e | Christos Stavrakakis | else:
|
134 | 63f9eb8e | Christos Stavrakakis | return "ERROR" |
135 | 75dc539e | Christos Stavrakakis | |
136 | 75dc539e | Christos Stavrakakis | def reconcile_stale_servers(self): |
137 | 75dc539e | Christos Stavrakakis | # Detect stale servers
|
138 | 75dc539e | Christos Stavrakakis | stale = [] |
139 | 75dc539e | Christos Stavrakakis | stale_keys = self.db_servers_keys - self.gnt_servers_keys |
140 | 75dc539e | Christos Stavrakakis | for server_id in stale_keys: |
141 | 75dc539e | Christos Stavrakakis | db_server = self.db_servers[server_id]
|
142 | 75dc539e | Christos Stavrakakis | if db_server.operstate == "BUILD": |
143 | 75dc539e | Christos Stavrakakis | build_status = self.get_build_status(db_server)
|
144 | 75dc539e | Christos Stavrakakis | if build_status == "ERROR": |
145 | 75dc539e | Christos Stavrakakis | # Special handling of BUILD eerrors
|
146 | 75dc539e | Christos Stavrakakis | self.reconcile_building_server(db_server)
|
147 | 75dc539e | Christos Stavrakakis | elif build_status != "RUNNING": |
148 | 75dc539e | Christos Stavrakakis | stale.append(server_id) |
149 | 75dc539e | Christos Stavrakakis | else:
|
150 | 75dc539e | Christos Stavrakakis | stale.append(server_id) |
151 | 75dc539e | Christos Stavrakakis | |
152 | 75dc539e | Christos Stavrakakis | # Report them
|
153 | 75dc539e | Christos Stavrakakis | if stale:
|
154 | 75dc539e | Christos Stavrakakis | self.log.info("Found stale servers %s at backend %s", |
155 | 75dc539e | Christos Stavrakakis | ", ".join(map(str, stale)), self.backend) |
156 | 75dc539e | Christos Stavrakakis | else:
|
157 | 75dc539e | Christos Stavrakakis | self.log.debug("No stale servers at backend %s", self.backend) |
158 | 75dc539e | Christos Stavrakakis | |
159 | 75dc539e | Christos Stavrakakis | # Fix them
|
160 | 75dc539e | Christos Stavrakakis | if stale and self.options["fix_stale"]: |
161 | 75dc539e | Christos Stavrakakis | for server_id in stale: |
162 | 75dc539e | Christos Stavrakakis | db_server = self.db_servers[server_id]
|
163 | 75dc539e | Christos Stavrakakis | backend_mod.process_op_status( |
164 | 75dc539e | Christos Stavrakakis | vm=db_server, |
165 | 75dc539e | Christos Stavrakakis | etime=self.event_time,
|
166 | 75dc539e | Christos Stavrakakis | jobid=-0,
|
167 | 75dc539e | Christos Stavrakakis | opcode='OP_INSTANCE_REMOVE', status='success', |
168 | 75dc539e | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
169 | 75dc539e | Christos Stavrakakis | self.log.debug("Simulated Ganeti removal for stale servers.") |
170 | 75dc539e | Christos Stavrakakis | |
171 | 75dc539e | Christos Stavrakakis | def reconcile_orphan_servers(self): |
172 | 75dc539e | Christos Stavrakakis | orphans = self.gnt_servers_keys - self.db_servers_keys |
173 | 75dc539e | Christos Stavrakakis | if orphans:
|
174 | 75dc539e | Christos Stavrakakis | self.log.info("Found orphan servers %s at backend %s", |
175 | 75dc539e | Christos Stavrakakis | ", ".join(map(str, orphans)), self.backend) |
176 | 75dc539e | Christos Stavrakakis | else:
|
177 | 75dc539e | Christos Stavrakakis | self.log.debug("No orphan servers at backend %s", self.backend) |
178 | 75dc539e | Christos Stavrakakis | |
179 | 75dc539e | Christos Stavrakakis | if orphans and self.options["fix_orphans"]: |
180 | 75dc539e | Christos Stavrakakis | for server_id in orphans: |
181 | 75dc539e | Christos Stavrakakis | server_name = utils.id_to_instance_name(server_id) |
182 | 75dc539e | Christos Stavrakakis | self.client.DeleteInstance(server_name)
|
183 | 75dc539e | Christos Stavrakakis | self.log.debug("Issued OP_INSTANCE_REMOVE for orphan servers.") |
184 | 75dc539e | Christos Stavrakakis | |
185 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_servers(self): |
186 | 75dc539e | Christos Stavrakakis | #log = self.log
|
187 | 75dc539e | Christos Stavrakakis | for server_id in self.db_servers_keys & self.gnt_servers_keys: |
188 | 75dc539e | Christos Stavrakakis | db_server = self.db_servers[server_id]
|
189 | 75dc539e | Christos Stavrakakis | gnt_server = self.gnt_servers[server_id]
|
190 | 75dc539e | Christos Stavrakakis | if db_server.operstate == "BUILD": |
191 | 75dc539e | Christos Stavrakakis | build_status = self.get_build_status(db_server)
|
192 | 75dc539e | Christos Stavrakakis | if build_status == "RUNNING": |
193 | 75dc539e | Christos Stavrakakis | # Do not reconcile building VMs
|
194 | 75dc539e | Christos Stavrakakis | continue
|
195 | 75dc539e | Christos Stavrakakis | elif build_status == "ERROR": |
196 | 75dc539e | Christos Stavrakakis | # Special handling of build errors
|
197 | 75dc539e | Christos Stavrakakis | self.reconcile_building_server(db_server)
|
198 | 75dc539e | Christos Stavrakakis | continue
|
199 | 75dc539e | Christos Stavrakakis | |
200 | 75dc539e | Christos Stavrakakis | self.reconcile_unsynced_operstate(server_id, db_server,
|
201 | 75dc539e | Christos Stavrakakis | gnt_server) |
202 | 75dc539e | Christos Stavrakakis | self.reconcile_unsynced_flavor(server_id, db_server,
|
203 | 75dc539e | Christos Stavrakakis | gnt_server) |
204 | 75dc539e | Christos Stavrakakis | self.reconcile_unsynced_nics(server_id, db_server, gnt_server)
|
205 | 75dc539e | Christos Stavrakakis | self.reconcile_unsynced_disks(server_id, db_server, gnt_server)
|
206 | 63f9eb8e | Christos Stavrakakis | if db_server.task is not None: |
207 | 63f9eb8e | Christos Stavrakakis | self.reconcile_pending_task(server_id, db_server)
|
208 | 75dc539e | Christos Stavrakakis | |
209 | 75dc539e | Christos Stavrakakis | def reconcile_building_server(self, db_server): |
210 | 75dc539e | Christos Stavrakakis | self.log.info("Server '%s' is BUILD in DB, but 'ERROR' in Ganeti.", |
211 | 75dc539e | Christos Stavrakakis | db_server.id) |
212 | 75dc539e | Christos Stavrakakis | if self.options["fix_unsynced"]: |
213 | 75dc539e | Christos Stavrakakis | fix_opcode = "OP_INSTANCE_CREATE"
|
214 | 75dc539e | Christos Stavrakakis | backend_mod.process_op_status( |
215 | 75dc539e | Christos Stavrakakis | vm=db_server, |
216 | 75dc539e | Christos Stavrakakis | etime=self.event_time,
|
217 | 75dc539e | Christos Stavrakakis | jobid=-0,
|
218 | 75dc539e | Christos Stavrakakis | opcode=fix_opcode, status='error',
|
219 | 75dc539e | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
220 | 75dc539e | Christos Stavrakakis | self.log.debug("Simulated Ganeti error build event for" |
221 | 75dc539e | Christos Stavrakakis | " server '%s'", db_server.id)
|
222 | 75dc539e | Christos Stavrakakis | |
223 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_operstate(self, server_id, db_server, gnt_server): |
224 | 75dc539e | Christos Stavrakakis | if db_server.operstate != gnt_server["state"]: |
225 | 75dc539e | Christos Stavrakakis | self.log.info("Server '%s' is '%s' in DB and '%s' in Ganeti.", |
226 | 75dc539e | Christos Stavrakakis | server_id, db_server.operstate, gnt_server["state"])
|
227 | 75dc539e | Christos Stavrakakis | if self.options["fix_unsynced"]: |
228 | 5d3e597a | Christos Stavrakakis | # If server is in building state, you will have first to
|
229 | 5d3e597a | Christos Stavrakakis | # reconcile it's creation, to avoid wrong quotas
|
230 | 5d3e597a | Christos Stavrakakis | if db_server.operstate == "BUILD": |
231 | 5d3e597a | Christos Stavrakakis | backend_mod.process_op_status( |
232 | 5d3e597a | Christos Stavrakakis | vm=db_server, etime=self.event_time, jobid=-0, |
233 | 5d3e597a | Christos Stavrakakis | opcode="OP_INSTANCE_CREATE", status='success', |
234 | 5d3e597a | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
235 | 5d3e597a | Christos Stavrakakis | fix_opcode = "OP_INSTANCE_STARTUP"\
|
236 | 5d3e597a | Christos Stavrakakis | if gnt_server["state"] == "STARTED"\ |
237 | 75dc539e | Christos Stavrakakis | else "OP_INSTANCE_SHUTDOWN" |
238 | 75dc539e | Christos Stavrakakis | backend_mod.process_op_status( |
239 | 5d3e597a | Christos Stavrakakis | vm=db_server, etime=self.event_time, jobid=-0, |
240 | 75dc539e | Christos Stavrakakis | opcode=fix_opcode, status='success',
|
241 | 75dc539e | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
242 | 75dc539e | Christos Stavrakakis | self.log.debug("Simulated Ganeti state event for server '%s'", |
243 | 75dc539e | Christos Stavrakakis | server_id) |
244 | 75dc539e | Christos Stavrakakis | |
245 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_flavor(self, server_id, db_server, gnt_server): |
246 | 75dc539e | Christos Stavrakakis | db_flavor = db_server.flavor |
247 | 75dc539e | Christos Stavrakakis | gnt_flavor = gnt_server["flavor"]
|
248 | 75dc539e | Christos Stavrakakis | if (db_flavor.ram != gnt_flavor["ram"] or |
249 | 5d3e597a | Christos Stavrakakis | db_flavor.cpu != gnt_flavor["vcpus"]):
|
250 | a67419d8 | Christos Stavrakakis | try:
|
251 | a67419d8 | Christos Stavrakakis | gnt_flavor = Flavor.objects.get( |
252 | 75dc539e | Christos Stavrakakis | ram=gnt_flavor["ram"],
|
253 | 75dc539e | Christos Stavrakakis | cpu=gnt_flavor["vcpus"],
|
254 | 75dc539e | Christos Stavrakakis | disk=db_flavor.disk, |
255 | 75dc539e | Christos Stavrakakis | disk_template=db_flavor.disk_template) |
256 | a67419d8 | Christos Stavrakakis | except Flavor.DoesNotExist:
|
257 | 75dc539e | Christos Stavrakakis | self.log.warning("Server '%s' has unknown flavor.", server_id) |
258 | 75dc539e | Christos Stavrakakis | return
|
259 | 75dc539e | Christos Stavrakakis | |
260 | 4111c53e | Christos Stavrakakis | self.log.info("Server '%s' has flavor '%s' in DB and '%s' in" |
261 | 75dc539e | Christos Stavrakakis | " Ganeti", server_id, db_flavor, gnt_flavor)
|
262 | 75dc539e | Christos Stavrakakis | if self.options["fix_unsynced_flavors"]: |
263 | 75dc539e | Christos Stavrakakis | old_state = db_server.operstate |
264 | 75dc539e | Christos Stavrakakis | opcode = "OP_INSTANCE_SET_PARAMS"
|
265 | 75dc539e | Christos Stavrakakis | beparams = {"vcpus": gnt_flavor.cpu,
|
266 | 75dc539e | Christos Stavrakakis | "minmem": gnt_flavor.ram,
|
267 | 75dc539e | Christos Stavrakakis | "maxmem": gnt_flavor.ram}
|
268 | 75dc539e | Christos Stavrakakis | backend_mod.process_op_status( |
269 | 75dc539e | Christos Stavrakakis | vm=db_server, etime=self.event_time, jobid=-0, |
270 | 75dc539e | Christos Stavrakakis | opcode=opcode, status='success',
|
271 | 75dc539e | Christos Stavrakakis | beparams=beparams, |
272 | 75dc539e | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
273 | 75dc539e | Christos Stavrakakis | # process_op_status with beparams will set the vmstate to
|
274 | 75dc539e | Christos Stavrakakis | # shutdown. Fix this be returning it to old state
|
275 | 75dc539e | Christos Stavrakakis | vm = VirtualMachine.objects.get(pk=server_id) |
276 | 75dc539e | Christos Stavrakakis | vm.operstate = old_state |
277 | 75dc539e | Christos Stavrakakis | vm.save() |
278 | 75dc539e | Christos Stavrakakis | self.log.debug("Simulated Ganeti flavor event for server '%s'", |
279 | 75dc539e | Christos Stavrakakis | server_id) |
280 | 75dc539e | Christos Stavrakakis | |
281 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_nics(self, server_id, db_server, gnt_server): |
282 | 75dc539e | Christos Stavrakakis | db_nics = db_server.nics.order_by("index")
|
283 | 75dc539e | Christos Stavrakakis | gnt_nics = gnt_server["nics"]
|
284 | 75dc539e | Christos Stavrakakis | gnt_nics_parsed = backend_mod.process_ganeti_nics(gnt_nics) |
285 | 75dc539e | Christos Stavrakakis | if backend_mod.nics_changed(db_nics, gnt_nics_parsed):
|
286 | 75dc539e | Christos Stavrakakis | msg = "Found unsynced NICs for server '%s'.\n\t"\
|
287 | 75dc539e | Christos Stavrakakis | "DB: %s\n\tGaneti: %s"
|
288 | 75dc539e | Christos Stavrakakis | db_nics_str = ", ".join(map(format_db_nic, db_nics)) |
289 | 75dc539e | Christos Stavrakakis | gnt_nics_str = ", ".join(map(format_gnt_nic, gnt_nics_parsed)) |
290 | 75dc539e | Christos Stavrakakis | self.log.info(msg, server_id, db_nics_str, gnt_nics_str)
|
291 | 75dc539e | Christos Stavrakakis | if self.options["fix_unsynced_nics"]: |
292 | 75dc539e | Christos Stavrakakis | backend_mod.process_net_status(vm=db_server, |
293 | 75dc539e | Christos Stavrakakis | etime=self.event_time,
|
294 | 75dc539e | Christos Stavrakakis | nics=gnt_nics) |
295 | 75dc539e | Christos Stavrakakis | |
296 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_disks(self, server_id, db_server, gnt_server): |
297 | 75dc539e | Christos Stavrakakis | pass
|
298 | 75dc539e | Christos Stavrakakis | |
299 | 63f9eb8e | Christos Stavrakakis | def reconcile_pending_task(self, server_id, db_server): |
300 | 63f9eb8e | Christos Stavrakakis | job_id = db_server.task_job_id |
301 | 63f9eb8e | Christos Stavrakakis | pending_task = False
|
302 | 63f9eb8e | Christos Stavrakakis | if job_id not in self.gnt_jobs: |
303 | 63f9eb8e | Christos Stavrakakis | pending_task = True
|
304 | 63f9eb8e | Christos Stavrakakis | else:
|
305 | ce55f724 | Christos Stavrakakis | gnt_job_status = self.gnt_jobs[job_id]["status"] |
306 | 63f9eb8e | Christos Stavrakakis | if gnt_job_status in GANETI_JOBS_FINALIZED: |
307 | 63f9eb8e | Christos Stavrakakis | pending_task = True
|
308 | 63f9eb8e | Christos Stavrakakis | |
309 | 63f9eb8e | Christos Stavrakakis | if pending_task:
|
310 | 63f9eb8e | Christos Stavrakakis | self.log.info("Found server '%s' with pending task: '%s'", |
311 | 63f9eb8e | Christos Stavrakakis | server_id, db_server.task) |
312 | 160c81a1 | Christos Stavrakakis | if self.options["fix_pending_tasks"]: |
313 | 63f9eb8e | Christos Stavrakakis | db_server.task = None
|
314 | 63f9eb8e | Christos Stavrakakis | db_server.task_job_id = None
|
315 | 63f9eb8e | Christos Stavrakakis | db_server.save() |
316 | 63f9eb8e | Christos Stavrakakis | self.log.info("Cleared pending task for server '%s", server_id) |
317 | 63f9eb8e | Christos Stavrakakis | |
318 | 75dc539e | Christos Stavrakakis | |
319 | 75dc539e | Christos Stavrakakis | def format_db_nic(nic): |
320 | 4111c53e | Christos Stavrakakis | return "Index: %s, IP: %s Network: %s MAC: %s Firewall: %s" % (nic.index, |
321 | 75dc539e | Christos Stavrakakis | nic.ipv4, nic.network_id, nic.mac, nic.firewall_profile) |
322 | 75dc539e | Christos Stavrakakis | |
323 | 75dc539e | Christos Stavrakakis | |
324 | 75dc539e | Christos Stavrakakis | def format_gnt_nic(nic): |
325 | 75dc539e | Christos Stavrakakis | return "Index: %s IP: %s Network: %s MAC: %s Firewall: %s" %\ |
326 | 75dc539e | Christos Stavrakakis | (nic["index"], nic["ipv4"], nic["network"], nic["mac"], |
327 | 75dc539e | Christos Stavrakakis | nic["firewall_profile"])
|
328 | 9fea53cc | Vangelis Koukis | |
329 | cc92b70f | Christos Stavrakakis | |
330 | 0e9a423f | Christos Stavrakakis | #
|
331 | 0e9a423f | Christos Stavrakakis | # Networks
|
332 | 0e9a423f | Christos Stavrakakis | #
|
333 | 3524241a | Christos Stavrakakis | |
334 | 3524241a | Christos Stavrakakis | |
335 | 0e9a423f | Christos Stavrakakis | def get_networks_from_ganeti(backend): |
336 | 44e2c577 | Christos Stavrakakis | prefix = settings.BACKEND_PREFIX_ID + 'net-'
|
337 | 0e9a423f | Christos Stavrakakis | |
338 | 0e9a423f | Christos Stavrakakis | networks = {} |
339 | 3524241a | Christos Stavrakakis | with pooled_rapi_client(backend) as c: |
340 | 3524241a | Christos Stavrakakis | for net in c.GetNetworks(bulk=True): |
341 | 3524241a | Christos Stavrakakis | if net['name'].startswith(prefix): |
342 | 3524241a | Christos Stavrakakis | id = utils.id_from_network_name(net['name'])
|
343 | 3524241a | Christos Stavrakakis | networks[id] = net
|
344 | 0e9a423f | Christos Stavrakakis | |
345 | 0e9a423f | Christos Stavrakakis | return networks
|
346 | 0e9a423f | Christos Stavrakakis | |
347 | 0e9a423f | Christos Stavrakakis | |
348 | 0e9a423f | Christos Stavrakakis | def hanging_networks(backend, GNets): |
349 | 0e9a423f | Christos Stavrakakis | """Get networks that are not connected to all Nodegroups.
|
350 | 0e9a423f | Christos Stavrakakis |
|
351 | 0e9a423f | Christos Stavrakakis | """
|
352 | 0e9a423f | Christos Stavrakakis | def get_network_groups(group_list): |
353 | 0e9a423f | Christos Stavrakakis | groups = set()
|
354 | 0e9a423f | Christos Stavrakakis | for g in group_list: |
355 | 0e9a423f | Christos Stavrakakis | g_name = g.split('(')[0] |
356 | 0e9a423f | Christos Stavrakakis | groups.add(g_name) |
357 | 0e9a423f | Christos Stavrakakis | return groups
|
358 | 0e9a423f | Christos Stavrakakis | |
359 | 3524241a | Christos Stavrakakis | with pooled_rapi_client(backend) as c: |
360 | 3524241a | Christos Stavrakakis | groups = set(c.GetGroups())
|
361 | 0e9a423f | Christos Stavrakakis | |
362 | 0e9a423f | Christos Stavrakakis | hanging = {} |
363 | 0e9a423f | Christos Stavrakakis | for id, info in GNets.items(): |
364 | 0e9a423f | Christos Stavrakakis | group_list = get_network_groups(info['group_list'])
|
365 | 0e9a423f | Christos Stavrakakis | if group_list != groups:
|
366 | 0e9a423f | Christos Stavrakakis | hanging[id] = groups - group_list
|
367 | 0e9a423f | Christos Stavrakakis | return hanging
|
368 | 0e9a423f | Christos Stavrakakis | |
369 | 9fea53cc | Vangelis Koukis | |
370 | 75dc539e | Christos Stavrakakis | def get_online_backends(): |
371 | 75dc539e | Christos Stavrakakis | return Backend.objects.filter(offline=False) |
372 | 75dc539e | Christos Stavrakakis | |
373 | 75dc539e | Christos Stavrakakis | |
374 | 75dc539e | Christos Stavrakakis | def get_database_servers(backend): |
375 | 75dc539e | Christos Stavrakakis | servers = backend.virtual_machines.select_related("nics", "flavor")\ |
376 | 75dc539e | Christos Stavrakakis | .filter(deleted=False)
|
377 | 75dc539e | Christos Stavrakakis | return dict([(s.id, s) for s in servers]) |
378 | 75dc539e | Christos Stavrakakis | |
379 | 75dc539e | Christos Stavrakakis | |
380 | 75dc539e | Christos Stavrakakis | def get_ganeti_servers(backend): |
381 | 75dc539e | Christos Stavrakakis | gnt_instances = backend_mod.get_instances(backend) |
382 | 75dc539e | Christos Stavrakakis | # Filter out non-synnefo instances
|
383 | 75dc539e | Christos Stavrakakis | snf_backend_prefix = settings.BACKEND_PREFIX_ID |
384 | 75dc539e | Christos Stavrakakis | gnt_instances = filter(lambda i: i["name"].startswith(snf_backend_prefix), |
385 | 75dc539e | Christos Stavrakakis | gnt_instances) |
386 | 75dc539e | Christos Stavrakakis | gnt_instances = map(parse_gnt_instance, gnt_instances)
|
387 | 75dc539e | Christos Stavrakakis | return dict([(i["id"], i) for i in gnt_instances if i["id"] is not None]) |
388 | 75dc539e | Christos Stavrakakis | |
389 | 75dc539e | Christos Stavrakakis | |
390 | 75dc539e | Christos Stavrakakis | def parse_gnt_instance(instance): |
391 | 75dc539e | Christos Stavrakakis | try:
|
392 | 75dc539e | Christos Stavrakakis | instance_id = utils.id_from_instance_name(instance['name'])
|
393 | 75dc539e | Christos Stavrakakis | except Exception: |
394 | 75dc539e | Christos Stavrakakis | logger.error("Ignoring instance with malformed name %s",
|
395 | 75dc539e | Christos Stavrakakis | instance['name'])
|
396 | 75dc539e | Christos Stavrakakis | return (None, None) |
397 | 75dc539e | Christos Stavrakakis | |
398 | 75dc539e | Christos Stavrakakis | beparams = instance["beparams"]
|
399 | 75dc539e | Christos Stavrakakis | |
400 | 75dc539e | Christos Stavrakakis | vcpus = beparams["vcpus"]
|
401 | 75dc539e | Christos Stavrakakis | ram = beparams["maxmem"]
|
402 | 75dc539e | Christos Stavrakakis | state = instance["oper_state"] and "STARTED" or "STOPPED" |
403 | 75dc539e | Christos Stavrakakis | |
404 | 75dc539e | Christos Stavrakakis | return {
|
405 | 75dc539e | Christos Stavrakakis | "id": instance_id,
|
406 | 75dc539e | Christos Stavrakakis | "state": state, # FIX |
407 | 75dc539e | Christos Stavrakakis | "updated": datetime.fromtimestamp(instance["mtime"]), |
408 | 75dc539e | Christos Stavrakakis | "disks": disks_from_instance(instance),
|
409 | 75dc539e | Christos Stavrakakis | "nics": nics_from_instance(instance),
|
410 | 75dc539e | Christos Stavrakakis | "flavor": {"vcpus": vcpus, |
411 | 75dc539e | Christos Stavrakakis | "ram": ram},
|
412 | 75dc539e | Christos Stavrakakis | "tags": instance["tags"] |
413 | 75dc539e | Christos Stavrakakis | } |
414 | 75dc539e | Christos Stavrakakis | |
415 | 75dc539e | Christos Stavrakakis | |
416 | 75dc539e | Christos Stavrakakis | def nics_from_instance(i): |
417 | 75dc539e | Christos Stavrakakis | ips = zip(itertools.repeat('ip'), i['nic.ips']) |
418 | 75dc539e | Christos Stavrakakis | macs = zip(itertools.repeat('mac'), i['nic.macs']) |
419 | 75dc539e | Christos Stavrakakis | networks = zip(itertools.repeat('network'), i['nic.networks']) |
420 | 75dc539e | Christos Stavrakakis | # modes = zip(itertools.repeat('mode'), i['nic.modes'])
|
421 | 75dc539e | Christos Stavrakakis | # links = zip(itertools.repeat('link'), i['nic.links'])
|
422 | 75dc539e | Christos Stavrakakis | # nics = zip(ips,macs,modes,networks,links)
|
423 | 75dc539e | Christos Stavrakakis | nics = zip(ips, macs, networks)
|
424 | 75dc539e | Christos Stavrakakis | nics = map(lambda x: dict(x), nics) |
425 | 75dc539e | Christos Stavrakakis | #nics = dict(enumerate(nics))
|
426 | 75dc539e | Christos Stavrakakis | tags = i["tags"]
|
427 | 75dc539e | Christos Stavrakakis | for tag in tags: |
428 | 75dc539e | Christos Stavrakakis | t = tag.split(":")
|
429 | 75dc539e | Christos Stavrakakis | if t[0:2] == ["synnefo", "network"]: |
430 | 75dc539e | Christos Stavrakakis | if len(t) != 4: |
431 | 75dc539e | Christos Stavrakakis | logger.error("Malformed synefo tag %s", tag)
|
432 | 75dc539e | Christos Stavrakakis | continue
|
433 | 75dc539e | Christos Stavrakakis | try:
|
434 | 75dc539e | Christos Stavrakakis | index = int(t[2]) |
435 | 75dc539e | Christos Stavrakakis | nics[index]['firewall'] = t[3] |
436 | 75dc539e | Christos Stavrakakis | except ValueError: |
437 | 75dc539e | Christos Stavrakakis | logger.error("Malformed synnefo tag %s", tag)
|
438 | 75dc539e | Christos Stavrakakis | except IndexError: |
439 | 75dc539e | Christos Stavrakakis | logger.error("Found tag %s for non-existent NIC %d",
|
440 | 75dc539e | Christos Stavrakakis | tag, index) |
441 | 75dc539e | Christos Stavrakakis | return nics
|
442 | 9fea53cc | Vangelis Koukis | |
443 | 9fea53cc | Vangelis Koukis | |
444 | 63f9eb8e | Christos Stavrakakis | def get_ganeti_jobs(backend): |
445 | 63f9eb8e | Christos Stavrakakis | gnt_jobs = backend_mod.get_jobs(backend) |
446 | 63f9eb8e | Christos Stavrakakis | return dict([(int(j["id"]), j) for j in gnt_jobs]) |
447 | 63f9eb8e | Christos Stavrakakis | |
448 | 63f9eb8e | Christos Stavrakakis | |
449 | 75dc539e | Christos Stavrakakis | def disks_from_instance(i): |
450 | 75dc539e | Christos Stavrakakis | return dict([(index, {"size": size}) |
451 | 75dc539e | Christos Stavrakakis | for index, size in enumerate(i["disk.sizes"])]) |
452 | 89b2b908 | Christos Stavrakakis | |
453 | 89b2b908 | Christos Stavrakakis | |
454 | 89b2b908 | Christos Stavrakakis | class NetworkReconciler(object): |
455 | 89b2b908 | Christos Stavrakakis | def __init__(self, logger, fix=False, conflicting_ips=False): |
456 | 89b2b908 | Christos Stavrakakis | self.log = logger
|
457 | 89b2b908 | Christos Stavrakakis | self.conflicting_ips = conflicting_ips
|
458 | 89b2b908 | Christos Stavrakakis | self.fix = fix
|
459 | 89b2b908 | Christos Stavrakakis | |
460 | 89b2b908 | Christos Stavrakakis | @transaction.commit_on_success
|
461 | 89b2b908 | Christos Stavrakakis | def reconcile_networks(self): |
462 | 89b2b908 | Christos Stavrakakis | # Get models from DB
|
463 | 89b2b908 | Christos Stavrakakis | backends = Backend.objects.exclude(offline=True)
|
464 | 89b2b908 | Christos Stavrakakis | networks = Network.objects.filter(deleted=False)
|
465 | 89b2b908 | Christos Stavrakakis | |
466 | 89b2b908 | Christos Stavrakakis | self.event_time = datetime.now()
|
467 | 89b2b908 | Christos Stavrakakis | |
468 | 89b2b908 | Christos Stavrakakis | # Get info from all ganeti backends
|
469 | 89b2b908 | Christos Stavrakakis | ganeti_networks = {} |
470 | 89b2b908 | Christos Stavrakakis | ganeti_hanging_networks = {} |
471 | 89b2b908 | Christos Stavrakakis | for b in backends: |
472 | 89b2b908 | Christos Stavrakakis | g_nets = get_networks_from_ganeti(b) |
473 | 89b2b908 | Christos Stavrakakis | ganeti_networks[b] = g_nets |
474 | 89b2b908 | Christos Stavrakakis | g_hanging_nets = hanging_networks(b, g_nets) |
475 | 89b2b908 | Christos Stavrakakis | ganeti_hanging_networks[b] = g_hanging_nets |
476 | 89b2b908 | Christos Stavrakakis | |
477 | 89b2b908 | Christos Stavrakakis | # Perform reconciliation for each network
|
478 | 89b2b908 | Christos Stavrakakis | for network in networks: |
479 | 89b2b908 | Christos Stavrakakis | ip_available_maps = [] |
480 | 89b2b908 | Christos Stavrakakis | ip_reserved_maps = [] |
481 | 89b2b908 | Christos Stavrakakis | for bend in backends: |
482 | 89b2b908 | Christos Stavrakakis | bnet = get_backend_network(network, bend) |
483 | 89b2b908 | Christos Stavrakakis | gnet = ganeti_networks[bend].get(network.id) |
484 | 89b2b908 | Christos Stavrakakis | if not bnet: |
485 | 89b2b908 | Christos Stavrakakis | if network.floating_ip_pool:
|
486 | 89b2b908 | Christos Stavrakakis | # Network is a floating IP pool and does not exist in
|
487 | 89b2b908 | Christos Stavrakakis | # backend. We need to create it
|
488 | 89b2b908 | Christos Stavrakakis | bnet = self.reconcile_parted_network(network, bend)
|
489 | 89b2b908 | Christos Stavrakakis | elif not gnet: |
490 | 89b2b908 | Christos Stavrakakis | # Network does not exist either in Ganeti nor in BD.
|
491 | 89b2b908 | Christos Stavrakakis | continue
|
492 | 89b2b908 | Christos Stavrakakis | else:
|
493 | 89b2b908 | Christos Stavrakakis | # Network exists in Ganeti and not in DB.
|
494 | 89b2b908 | Christos Stavrakakis | if network.action != "DESTROY" and not network.public: |
495 | 89b2b908 | Christos Stavrakakis | bnet = self.reconcile_parted_network(network, bend)
|
496 | 89b2b908 | Christos Stavrakakis | else:
|
497 | 89b2b908 | Christos Stavrakakis | continue
|
498 | 89b2b908 | Christos Stavrakakis | |
499 | 89b2b908 | Christos Stavrakakis | if not gnet: |
500 | 89b2b908 | Christos Stavrakakis | # Network does not exist in Ganeti. If the network action
|
501 | 89b2b908 | Christos Stavrakakis | # is DESTROY, we have to mark as deleted in DB, else we
|
502 | 89b2b908 | Christos Stavrakakis | # have to create it in Ganeti.
|
503 | 89b2b908 | Christos Stavrakakis | if network.action == "DESTROY": |
504 | 89b2b908 | Christos Stavrakakis | if bnet.operstate != "DELETED": |
505 | 89b2b908 | Christos Stavrakakis | self.reconcile_stale_network(bnet)
|
506 | 89b2b908 | Christos Stavrakakis | else:
|
507 | 89b2b908 | Christos Stavrakakis | self.reconcile_missing_network(network, bend)
|
508 | 89b2b908 | Christos Stavrakakis | # Skip rest reconciliation!
|
509 | 89b2b908 | Christos Stavrakakis | continue
|
510 | 89b2b908 | Christos Stavrakakis | |
511 | 89b2b908 | Christos Stavrakakis | try:
|
512 | 89b2b908 | Christos Stavrakakis | hanging_groups = ganeti_hanging_networks[bend][network.id] |
513 | 89b2b908 | Christos Stavrakakis | except KeyError: |
514 | 89b2b908 | Christos Stavrakakis | # Network is connected to all nodegroups
|
515 | 89b2b908 | Christos Stavrakakis | hanging_groups = [] |
516 | 89b2b908 | Christos Stavrakakis | |
517 | 89b2b908 | Christos Stavrakakis | if hanging_groups:
|
518 | 89b2b908 | Christos Stavrakakis | # CASE-3: Ganeti networks not connected to all nodegroups
|
519 | 89b2b908 | Christos Stavrakakis | self.reconcile_hanging_groups(network, bend,
|
520 | 89b2b908 | Christos Stavrakakis | hanging_groups) |
521 | 89b2b908 | Christos Stavrakakis | continue
|
522 | 89b2b908 | Christos Stavrakakis | |
523 | 89b2b908 | Christos Stavrakakis | if bnet.operstate != 'ACTIVE': |
524 | 89b2b908 | Christos Stavrakakis | # CASE-4: Unsynced network state. At this point the network
|
525 | 89b2b908 | Christos Stavrakakis | # exists and is connected to all nodes so is must be
|
526 | 89b2b908 | Christos Stavrakakis | # active!
|
527 | 89b2b908 | Christos Stavrakakis | self.reconcile_unsynced_network(network, bend, bnet)
|
528 | 89b2b908 | Christos Stavrakakis | |
529 | 89b2b908 | Christos Stavrakakis | # Get ganeti IP Pools
|
530 | 89b2b908 | Christos Stavrakakis | available_map, reserved_map = get_network_pool(gnet) |
531 | 89b2b908 | Christos Stavrakakis | ip_available_maps.append(available_map) |
532 | 89b2b908 | Christos Stavrakakis | ip_reserved_maps.append(reserved_map) |
533 | 89b2b908 | Christos Stavrakakis | |
534 | 89b2b908 | Christos Stavrakakis | if ip_available_maps or ip_reserved_maps: |
535 | 89b2b908 | Christos Stavrakakis | # CASE-5: Unsynced IP Pools
|
536 | 89b2b908 | Christos Stavrakakis | self.reconcile_ip_pools(network, ip_available_maps,
|
537 | 89b2b908 | Christos Stavrakakis | ip_reserved_maps) |
538 | 89b2b908 | Christos Stavrakakis | |
539 | 89b2b908 | Christos Stavrakakis | if self.conflicting_ips: |
540 | 89b2b908 | Christos Stavrakakis | self.detect_conflicting_ips()
|
541 | 89b2b908 | Christos Stavrakakis | |
542 | 89b2b908 | Christos Stavrakakis | # CASE-6: Orphan networks
|
543 | 89b2b908 | Christos Stavrakakis | self.reconcile_orphan_networks(networks, ganeti_networks)
|
544 | 89b2b908 | Christos Stavrakakis | |
545 | 89b2b908 | Christos Stavrakakis | def reconcile_parted_network(self, network, backend): |
546 | 89b2b908 | Christos Stavrakakis | self.log.info("D: Missing DB entry for network %s in backend %s", |
547 | 89b2b908 | Christos Stavrakakis | network, backend) |
548 | 89b2b908 | Christos Stavrakakis | if self.fix: |
549 | 89b2b908 | Christos Stavrakakis | network.create_backend_network(backend) |
550 | 89b2b908 | Christos Stavrakakis | self.log.info("F: Created DB entry") |
551 | 89b2b908 | Christos Stavrakakis | bnet = get_backend_network(network, backend) |
552 | 89b2b908 | Christos Stavrakakis | return bnet
|
553 | 89b2b908 | Christos Stavrakakis | |
554 | 89b2b908 | Christos Stavrakakis | def reconcile_stale_network(self, backend_network): |
555 | 89b2b908 | Christos Stavrakakis | self.log.info("D: Stale DB entry for network %s in backend %s", |
556 | 89b2b908 | Christos Stavrakakis | backend_network.network, backend_network.backend) |
557 | 89b2b908 | Christos Stavrakakis | if self.fix: |
558 | 89b2b908 | Christos Stavrakakis | backend_mod.process_network_status( |
559 | 89b2b908 | Christos Stavrakakis | backend_network, self.event_time, 0, |
560 | 89b2b908 | Christos Stavrakakis | "OP_NETWORK_REMOVE",
|
561 | 89b2b908 | Christos Stavrakakis | "success",
|
562 | 89b2b908 | Christos Stavrakakis | "Reconciliation simulated event")
|
563 | 89b2b908 | Christos Stavrakakis | self.log.info("F: Reconciled event: OP_NETWORK_REMOVE") |
564 | 89b2b908 | Christos Stavrakakis | |
565 | 89b2b908 | Christos Stavrakakis | def reconcile_missing_network(self, network, backend): |
566 | 89b2b908 | Christos Stavrakakis | self.log.info("D: Missing Ganeti network %s in backend %s", |
567 | 89b2b908 | Christos Stavrakakis | network, backend) |
568 | 89b2b908 | Christos Stavrakakis | if self.fix: |
569 | 89b2b908 | Christos Stavrakakis | backend_mod.create_network(network, backend) |
570 | 89b2b908 | Christos Stavrakakis | self.log.info("F: Issued OP_NETWORK_CONNECT") |
571 | 89b2b908 | Christos Stavrakakis | |
572 | 89b2b908 | Christos Stavrakakis | def reconcile_hanging_groups(self, network, backend, hanging_groups): |
573 | 89b2b908 | Christos Stavrakakis | self.log.info('D: Network %s in backend %s is not connected to ' |
574 | 89b2b908 | Christos Stavrakakis | 'the following groups:', network, backend)
|
575 | 89b2b908 | Christos Stavrakakis | self.log.info('- ' + '\n- '.join(hanging_groups)) |
576 | 89b2b908 | Christos Stavrakakis | if self.fix: |
577 | 89b2b908 | Christos Stavrakakis | for group in hanging_groups: |
578 | 89b2b908 | Christos Stavrakakis | self.log.info('F: Connecting network %s to nodegroup %s', |
579 | 89b2b908 | Christos Stavrakakis | network, group) |
580 | 89b2b908 | Christos Stavrakakis | backend_mod.connect_network(network, backend, depends=[], |
581 | 89b2b908 | Christos Stavrakakis | group=group) |
582 | 89b2b908 | Christos Stavrakakis | |
583 | 89b2b908 | Christos Stavrakakis | def reconcile_unsynced_network(self, network, backend, backend_network): |
584 | 89b2b908 | Christos Stavrakakis | self.log.info("D: Unsynced network %s in backend %s", network, backend) |
585 | 89b2b908 | Christos Stavrakakis | if self.fix: |
586 | 89b2b908 | Christos Stavrakakis | self.log.info("F: Issuing OP_NETWORK_CONNECT") |
587 | 89b2b908 | Christos Stavrakakis | backend_mod.process_network_status( |
588 | 89b2b908 | Christos Stavrakakis | backend_network, self.event_time, 0, |
589 | 89b2b908 | Christos Stavrakakis | "OP_NETWORK_CONNECT",
|
590 | 89b2b908 | Christos Stavrakakis | "success",
|
591 | 89b2b908 | Christos Stavrakakis | "Reconciliation simulated eventd")
|
592 | 89b2b908 | Christos Stavrakakis | |
593 | 89b2b908 | Christos Stavrakakis | def reconcile_ip_pools(self, network, available_maps, reserved_maps): |
594 | 89b2b908 | Christos Stavrakakis | available_map = reduce(lambda x, y: x & y, available_maps) |
595 | 89b2b908 | Christos Stavrakakis | reserved_map = reduce(lambda x, y: x & y, reserved_maps) |
596 | 89b2b908 | Christos Stavrakakis | |
597 | 89b2b908 | Christos Stavrakakis | pool = network.get_pool() |
598 | 89b2b908 | Christos Stavrakakis | # Temporary release unused floating IPs
|
599 | 89b2b908 | Christos Stavrakakis | temp_pool = network.get_pool() |
600 | 89b2b908 | Christos Stavrakakis | used_ips = network.nics.values_list("ipv4", flat=True) |
601 | 89b2b908 | Christos Stavrakakis | unused_static_ips = network.floating_ips.exclude(ipv4__in=used_ips) |
602 | 89b2b908 | Christos Stavrakakis | map(lambda ip: temp_pool.put(ip.ipv4), unused_static_ips) |
603 | 89b2b908 | Christos Stavrakakis | if temp_pool.available != available_map:
|
604 | 89b2b908 | Christos Stavrakakis | self.log.info("D: Unsynced available map of network %s:\n" |
605 | 89b2b908 | Christos Stavrakakis | "\tDB: %r\n\tGB: %r", network,
|
606 | 89b2b908 | Christos Stavrakakis | temp_pool.available.to01(), |
607 | 89b2b908 | Christos Stavrakakis | available_map.to01()) |
608 | 89b2b908 | Christos Stavrakakis | if self.fix: |
609 | 89b2b908 | Christos Stavrakakis | pool.available = available_map |
610 | 89b2b908 | Christos Stavrakakis | # Release unsued floating IPs, as they are not included in the
|
611 | 89b2b908 | Christos Stavrakakis | # available map
|
612 | 89b2b908 | Christos Stavrakakis | map(lambda ip: pool.reserve(ip.ipv4), unused_static_ips) |
613 | 89b2b908 | Christos Stavrakakis | pool.save() |
614 | 89b2b908 | Christos Stavrakakis | if pool.reserved != reserved_map:
|
615 | 89b2b908 | Christos Stavrakakis | self.log.info("D: Unsynced reserved map of network %s:\n" |
616 | 89b2b908 | Christos Stavrakakis | "\tDB: %r\n\tGB: %r", network, pool.reserved.to01(),
|
617 | 89b2b908 | Christos Stavrakakis | reserved_map.to01()) |
618 | 89b2b908 | Christos Stavrakakis | if self.fix: |
619 | 89b2b908 | Christos Stavrakakis | pool.reserved = reserved_map |
620 | 89b2b908 | Christos Stavrakakis | pool.save() |
621 | 89b2b908 | Christos Stavrakakis | |
622 | 89b2b908 | Christos Stavrakakis | def detect_conflicting_ips(self, network): |
623 | 89b2b908 | Christos Stavrakakis | """Detect NIC's that have the same IP in the same network."""
|
624 | 89b2b908 | Christos Stavrakakis | machine_ips = network.nics.all().values_list('ipv4', 'machine') |
625 | 89b2b908 | Christos Stavrakakis | ips = map(lambda x: x[0], machine_ips) |
626 | 89b2b908 | Christos Stavrakakis | distinct_ips = set(ips)
|
627 | 89b2b908 | Christos Stavrakakis | if len(distinct_ips) < len(ips): |
628 | 89b2b908 | Christos Stavrakakis | for i in distinct_ips: |
629 | 89b2b908 | Christos Stavrakakis | ips.remove(i) |
630 | 89b2b908 | Christos Stavrakakis | for i in ips: |
631 | 89b2b908 | Christos Stavrakakis | machines = [utils.id_to_instance_name(x[1])
|
632 | 89b2b908 | Christos Stavrakakis | for x in machine_ips if x[0] == i] |
633 | 89b2b908 | Christos Stavrakakis | self.log.info('D: Conflicting IP:%s Machines: %s', |
634 | 89b2b908 | Christos Stavrakakis | i, ', '.join(machines))
|
635 | 89b2b908 | Christos Stavrakakis | |
636 | 89b2b908 | Christos Stavrakakis | def reconcile_orphan_networks(self, db_networks, ganeti_networks): |
637 | 89b2b908 | Christos Stavrakakis | # Detect Orphan Networks in Ganeti
|
638 | 89b2b908 | Christos Stavrakakis | db_network_ids = set([net.id for net in db_networks]) |
639 | 89b2b908 | Christos Stavrakakis | for back_end, ganeti_networks in ganeti_networks.items(): |
640 | 89b2b908 | Christos Stavrakakis | ganeti_network_ids = set(ganeti_networks.keys())
|
641 | 89b2b908 | Christos Stavrakakis | orphans = ganeti_network_ids - db_network_ids |
642 | 89b2b908 | Christos Stavrakakis | |
643 | 89b2b908 | Christos Stavrakakis | if len(orphans) > 0: |
644 | 89b2b908 | Christos Stavrakakis | self.log.info('D: Orphan Networks in backend %s:', |
645 | 89b2b908 | Christos Stavrakakis | back_end.clustername) |
646 | 89b2b908 | Christos Stavrakakis | self.log.info('- ' + '\n- '.join([str(o) for o in orphans])) |
647 | 89b2b908 | Christos Stavrakakis | if self.fix: |
648 | 89b2b908 | Christos Stavrakakis | for net_id in orphans: |
649 | 89b2b908 | Christos Stavrakakis | self.log.info('Disconnecting and deleting network %d', |
650 | 89b2b908 | Christos Stavrakakis | net_id) |
651 | 89b2b908 | Christos Stavrakakis | try:
|
652 | 89b2b908 | Christos Stavrakakis | network = Network.objects.get(id=net_id) |
653 | 89b2b908 | Christos Stavrakakis | backend_mod.delete_network(network, |
654 | 89b2b908 | Christos Stavrakakis | backend=back_end) |
655 | 89b2b908 | Christos Stavrakakis | except Network.DoesNotExist:
|
656 | 89b2b908 | Christos Stavrakakis | self.log.info("Not entry for network %s in DB !!", |
657 | 89b2b908 | Christos Stavrakakis | net_id) |
658 | 89b2b908 | Christos Stavrakakis | |
659 | 89b2b908 | Christos Stavrakakis | |
660 | 89b2b908 | Christos Stavrakakis | def get_backend_network(network, backend): |
661 | 89b2b908 | Christos Stavrakakis | try:
|
662 | 89b2b908 | Christos Stavrakakis | return BackendNetwork.objects.get(network=network, backend=backend)
|
663 | 89b2b908 | Christos Stavrakakis | except BackendNetwork.DoesNotExist:
|
664 | 89b2b908 | Christos Stavrakakis | return None |
665 | 89b2b908 | Christos Stavrakakis | |
666 | 89b2b908 | Christos Stavrakakis | |
667 | 89b2b908 | Christos Stavrakakis | def get_network_pool(gnet): |
668 | 89b2b908 | Christos Stavrakakis | """Return available and reserved IP maps.
|
669 | 89b2b908 | Christos Stavrakakis |
|
670 | 89b2b908 | Christos Stavrakakis | Extract the available and reserved IP map from the info return from Ganeti
|
671 | 89b2b908 | Christos Stavrakakis | for a network.
|
672 | 89b2b908 | Christos Stavrakakis |
|
673 | 89b2b908 | Christos Stavrakakis | """
|
674 | 89b2b908 | Christos Stavrakakis | converter = IPPool(Foo(gnet['network']))
|
675 | 89b2b908 | Christos Stavrakakis | a_map = bitarray_from_map(gnet['map'])
|
676 | 89b2b908 | Christos Stavrakakis | a_map.invert() |
677 | 89b2b908 | Christos Stavrakakis | reserved = gnet['external_reservations']
|
678 | 89b2b908 | Christos Stavrakakis | r_map = a_map.copy() |
679 | 89b2b908 | Christos Stavrakakis | r_map.setall(True)
|
680 | 89b2b908 | Christos Stavrakakis | if reserved:
|
681 | 89b2b908 | Christos Stavrakakis | for address in reserved.split(','): |
682 | 89b2b908 | Christos Stavrakakis | index = converter.value_to_index(address.strip()) |
683 | 89b2b908 | Christos Stavrakakis | a_map[index] = True
|
684 | 89b2b908 | Christos Stavrakakis | r_map[index] = False
|
685 | 89b2b908 | Christos Stavrakakis | return a_map, r_map
|
686 | 89b2b908 | Christos Stavrakakis | |
687 | 89b2b908 | Christos Stavrakakis | |
688 | 89b2b908 | Christos Stavrakakis | def bitarray_from_map(bitmap): |
689 | 89b2b908 | Christos Stavrakakis | return bitarray.bitarray(bitmap.replace("X", "1").replace(".", "0")) |
690 | 89b2b908 | Christos Stavrakakis | |
691 | 89b2b908 | Christos Stavrakakis | |
692 | 89b2b908 | Christos Stavrakakis | class Foo(): |
693 | 89b2b908 | Christos Stavrakakis | def __init__(self, subnet): |
694 | 89b2b908 | Christos Stavrakakis | self.available_map = '' |
695 | 89b2b908 | Christos Stavrakakis | self.reserved_map = '' |
696 | 89b2b908 | Christos Stavrakakis | self.size = 0 |
697 | 89b2b908 | Christos Stavrakakis | self.network = Foo.Foo1(subnet)
|
698 | 89b2b908 | Christos Stavrakakis | |
699 | 89b2b908 | Christos Stavrakakis | class Foo1(): |
700 | 89b2b908 | Christos Stavrakakis | def __init__(self, subnet): |
701 | 89b2b908 | Christos Stavrakakis | self.subnet = subnet
|
702 | 89b2b908 | Christos Stavrakakis | self.gateway = None |