root / snf-cyclades-app / synnefo / logic / reconciliation.py @ 7b438672
History | View | Annotate | Download (30.3 kB)
1 | 9fea53cc | Vangelis Koukis | # -*- coding: utf-8 -*-
|
---|---|---|---|
2 | 9fea53cc | Vangelis Koukis | #
|
3 | 75dc539e | Christos Stavrakakis | # Copyright 2011-2013 GRNET S.A. All rights reserved.
|
4 | 9fea53cc | Vangelis Koukis | #
|
5 | 9fea53cc | Vangelis Koukis | # Redistribution and use in source and binary forms, with or
|
6 | 9fea53cc | Vangelis Koukis | # without modification, are permitted provided that the following
|
7 | 9fea53cc | Vangelis Koukis | # conditions are met:
|
8 | 9fea53cc | Vangelis Koukis | #
|
9 | 9fea53cc | Vangelis Koukis | # 1. Redistributions of source code must retain the above
|
10 | 9fea53cc | Vangelis Koukis | # copyright notice, this list of conditions and the following
|
11 | 9fea53cc | Vangelis Koukis | # disclaimer.
|
12 | 9fea53cc | Vangelis Koukis | #
|
13 | 9fea53cc | Vangelis Koukis | # 2. Redistributions in binary form must reproduce the above
|
14 | 9fea53cc | Vangelis Koukis | # copyright notice, this list of conditions and the following
|
15 | 9fea53cc | Vangelis Koukis | # disclaimer in the documentation and/or other materials
|
16 | 9fea53cc | Vangelis Koukis | # provided with the distribution.
|
17 | 9fea53cc | Vangelis Koukis | #
|
18 | 9fea53cc | Vangelis Koukis | # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
|
19 | 9fea53cc | Vangelis Koukis | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
20 | 9fea53cc | Vangelis Koukis | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
21 | 9fea53cc | Vangelis Koukis | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
|
22 | 9fea53cc | Vangelis Koukis | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
23 | 9fea53cc | Vangelis Koukis | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
24 | 9fea53cc | Vangelis Koukis | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
25 | 9fea53cc | Vangelis Koukis | # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
26 | 9fea53cc | Vangelis Koukis | # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
27 | 9fea53cc | Vangelis Koukis | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
28 | 9fea53cc | Vangelis Koukis | # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
29 | 9fea53cc | Vangelis Koukis | # POSSIBILITY OF SUCH DAMAGE.
|
30 | 9fea53cc | Vangelis Koukis | #
|
31 | 9fea53cc | Vangelis Koukis | # The views and conclusions contained in the software and
|
32 | 9fea53cc | Vangelis Koukis | # documentation are those of the authors and should not be
|
33 | 9fea53cc | Vangelis Koukis | # interpreted as representing official policies, either expressed
|
34 | 9fea53cc | Vangelis Koukis | # or implied, of GRNET S.A.
|
35 | 9fea53cc | Vangelis Koukis | #
|
36 | 9fea53cc | Vangelis Koukis | """Business logic for reconciliation
|
37 | 9fea53cc | Vangelis Koukis |
|
38 | 9fea53cc | Vangelis Koukis | Reconcile the contents of the DB with the actual state of the
|
39 | 9fea53cc | Vangelis Koukis | Ganeti backend.
|
40 | 9fea53cc | Vangelis Koukis |
|
41 | 9fea53cc | Vangelis Koukis | Let D be the set of VMs in the DB, G the set of VMs in Ganeti.
|
42 | 9fea53cc | Vangelis Koukis | RULES:
|
43 | 9fea53cc | Vangelis Koukis | R1. Stale servers in DB:
|
44 | 9fea53cc | Vangelis Koukis | For any v in D but not in G:
|
45 | 9fea53cc | Vangelis Koukis | Set deleted=True.
|
46 | 9fea53cc | Vangelis Koukis | R2. Orphan instances in Ganet:
|
47 | 9fea53cc | Vangelis Koukis | For any v in G with deleted=True in D:
|
48 | 9fea53cc | Vangelis Koukis | Issue OP_INSTANCE_DESTROY.
|
49 | 9fea53cc | Vangelis Koukis | R3. Unsynced operstate:
|
50 | 9fea53cc | Vangelis Koukis | For any v whose operating state differs between G and V:
|
51 | 9fea53cc | Vangelis Koukis | Set the operating state in D based on the state in G.
|
52 | 9fea53cc | Vangelis Koukis | In the code, D, G are Python dicts mapping instance ids to operating state.
|
53 | 9fea53cc | Vangelis Koukis | For D, the operating state is chosen from VirtualMachine.OPER_STATES.
|
54 | 9fea53cc | Vangelis Koukis | For G, the operating state is True if the machine is up, False otherwise.
|
55 | 9fea53cc | Vangelis Koukis |
|
56 | 9fea53cc | Vangelis Koukis | """
|
57 | 9fea53cc | Vangelis Koukis | |
58 | 9fea53cc | Vangelis Koukis | |
59 | 0c09b1c0 | Christos Stavrakakis | from django.conf import settings |
60 | 4161cb41 | Christos Stavrakakis | |
61 | 75dc539e | Christos Stavrakakis | import logging |
62 | 75dc539e | Christos Stavrakakis | import itertools |
63 | 89b2b908 | Christos Stavrakakis | import bitarray |
64 | 1cb7846c | Christos Stavrakakis | from datetime import datetime, timedelta |
65 | 4161cb41 | Christos Stavrakakis | |
66 | 75dc539e | Christos Stavrakakis | from django.db import transaction |
67 | 75dc539e | Christos Stavrakakis | from synnefo.db.models import (Backend, VirtualMachine, Flavor, |
68 | 89b2b908 | Christos Stavrakakis | pooled_rapi_client, Network, |
69 | 0ccb6461 | Christos Stavrakakis | BackendNetwork, BridgePoolTable, |
70 | 0ccb6461 | Christos Stavrakakis | MacPrefixPoolTable) |
71 | 0ccb6461 | Christos Stavrakakis | from synnefo.db import pools |
72 | edbc1d5a | Christos Stavrakakis | from synnefo.logic import utils, rapi, backend as backend_mod |
73 | 9fea53cc | Vangelis Koukis | |
74 | 75dc539e | Christos Stavrakakis | logger = logging.getLogger() |
75 | 75dc539e | Christos Stavrakakis | logging.basicConfig() |
76 | 9e98ba3c | Giorgos Verigakis | |
77 | 1cb7846c | Christos Stavrakakis | BUILDING_NIC_TIMEOUT = timedelta(seconds=120)
|
78 | e63050ca | Christos Stavrakakis | |
79 | e63050ca | Christos Stavrakakis | |
80 | 75dc539e | Christos Stavrakakis | class BackendReconciler(object): |
81 | 75dc539e | Christos Stavrakakis | def __init__(self, backend, logger, options=None): |
82 | 75dc539e | Christos Stavrakakis | self.backend = backend
|
83 | 75dc539e | Christos Stavrakakis | self.log = logger
|
84 | 75dc539e | Christos Stavrakakis | self.client = backend.get_client()
|
85 | 75dc539e | Christos Stavrakakis | if options is None: |
86 | 75dc539e | Christos Stavrakakis | self.options = {}
|
87 | 4161cb41 | Christos Stavrakakis | else:
|
88 | 75dc539e | Christos Stavrakakis | self.options = options
|
89 | 75dc539e | Christos Stavrakakis | |
90 | 75dc539e | Christos Stavrakakis | def close(self): |
91 | 75dc539e | Christos Stavrakakis | self.backend.put_client(self.client) |
92 | 75dc539e | Christos Stavrakakis | |
93 | 75dc539e | Christos Stavrakakis | @transaction.commit_on_success
|
94 | 75dc539e | Christos Stavrakakis | def reconcile(self): |
95 | 75dc539e | Christos Stavrakakis | log = self.log
|
96 | 75dc539e | Christos Stavrakakis | backend = self.backend
|
97 | 75dc539e | Christos Stavrakakis | log.debug("Reconciling backend %s", backend)
|
98 | 75dc539e | Christos Stavrakakis | |
99 | 75dc539e | Christos Stavrakakis | self.db_servers = get_database_servers(backend)
|
100 | 75dc539e | Christos Stavrakakis | self.db_servers_keys = set(self.db_servers.keys()) |
101 | 75dc539e | Christos Stavrakakis | log.debug("Got servers info from database.")
|
102 | 75dc539e | Christos Stavrakakis | |
103 | 75dc539e | Christos Stavrakakis | self.gnt_servers = get_ganeti_servers(backend)
|
104 | 75dc539e | Christos Stavrakakis | self.gnt_servers_keys = set(self.gnt_servers.keys()) |
105 | 75dc539e | Christos Stavrakakis | log.debug("Got servers info from Ganeti backend.")
|
106 | 75dc539e | Christos Stavrakakis | |
107 | 63f9eb8e | Christos Stavrakakis | self.gnt_jobs = get_ganeti_jobs(backend)
|
108 | 63f9eb8e | Christos Stavrakakis | log.debug("Got jobs from Ganeti backend")
|
109 | 63f9eb8e | Christos Stavrakakis | |
110 | 75dc539e | Christos Stavrakakis | self.event_time = datetime.now()
|
111 | 75dc539e | Christos Stavrakakis | |
112 | 75dc539e | Christos Stavrakakis | self.stale_servers = self.reconcile_stale_servers() |
113 | 75dc539e | Christos Stavrakakis | self.orphan_servers = self.reconcile_orphan_servers() |
114 | 75dc539e | Christos Stavrakakis | self.unsynced_servers = self.reconcile_unsynced_servers() |
115 | 75dc539e | Christos Stavrakakis | self.close()
|
116 | 75dc539e | Christos Stavrakakis | |
117 | 75dc539e | Christos Stavrakakis | def get_build_status(self, db_server): |
118 | 63f9eb8e | Christos Stavrakakis | job_id = db_server.backendjobid |
119 | 63f9eb8e | Christos Stavrakakis | if job_id in self.gnt_jobs: |
120 | 63f9eb8e | Christos Stavrakakis | gnt_job_status = self.gnt_jobs[job_id]["status"] |
121 | edbc1d5a | Christos Stavrakakis | if gnt_job_status == rapi.JOB_STATUS_ERROR:
|
122 | 63f9eb8e | Christos Stavrakakis | return "ERROR" |
123 | edbc1d5a | Christos Stavrakakis | elif gnt_job_status not in rapi.JOB_STATUS_FINALIZED: |
124 | 75dc539e | Christos Stavrakakis | return "RUNNING" |
125 | 75dc539e | Christos Stavrakakis | else:
|
126 | 63f9eb8e | Christos Stavrakakis | return "FINALIZED" |
127 | 75dc539e | Christos Stavrakakis | else:
|
128 | 63f9eb8e | Christos Stavrakakis | return "ERROR" |
129 | 75dc539e | Christos Stavrakakis | |
130 | 75dc539e | Christos Stavrakakis | def reconcile_stale_servers(self): |
131 | 75dc539e | Christos Stavrakakis | # Detect stale servers
|
132 | 75dc539e | Christos Stavrakakis | stale = [] |
133 | 75dc539e | Christos Stavrakakis | stale_keys = self.db_servers_keys - self.gnt_servers_keys |
134 | 75dc539e | Christos Stavrakakis | for server_id in stale_keys: |
135 | 75dc539e | Christos Stavrakakis | db_server = self.db_servers[server_id]
|
136 | 75dc539e | Christos Stavrakakis | if db_server.operstate == "BUILD": |
137 | 75dc539e | Christos Stavrakakis | build_status = self.get_build_status(db_server)
|
138 | 75dc539e | Christos Stavrakakis | if build_status == "ERROR": |
139 | 75dc539e | Christos Stavrakakis | # Special handling of BUILD eerrors
|
140 | 75dc539e | Christos Stavrakakis | self.reconcile_building_server(db_server)
|
141 | 75dc539e | Christos Stavrakakis | elif build_status != "RUNNING": |
142 | 75dc539e | Christos Stavrakakis | stale.append(server_id) |
143 | 1464a17b | Christos Stavrakakis | elif (db_server.operstate == "ERROR" and |
144 | 1464a17b | Christos Stavrakakis | db_server.action != "DESTROY"):
|
145 | 1464a17b | Christos Stavrakakis | # Servers at building ERROR are stale only if the user has
|
146 | 1464a17b | Christos Stavrakakis | # asked to destroy them.
|
147 | 1464a17b | Christos Stavrakakis | pass
|
148 | 75dc539e | Christos Stavrakakis | else:
|
149 | 75dc539e | Christos Stavrakakis | stale.append(server_id) |
150 | 75dc539e | Christos Stavrakakis | |
151 | 75dc539e | Christos Stavrakakis | # Report them
|
152 | 75dc539e | Christos Stavrakakis | if stale:
|
153 | 75dc539e | Christos Stavrakakis | self.log.info("Found stale servers %s at backend %s", |
154 | 75dc539e | Christos Stavrakakis | ", ".join(map(str, stale)), self.backend) |
155 | 75dc539e | Christos Stavrakakis | else:
|
156 | 75dc539e | Christos Stavrakakis | self.log.debug("No stale servers at backend %s", self.backend) |
157 | 75dc539e | Christos Stavrakakis | |
158 | 75dc539e | Christos Stavrakakis | # Fix them
|
159 | 75dc539e | Christos Stavrakakis | if stale and self.options["fix_stale"]: |
160 | 75dc539e | Christos Stavrakakis | for server_id in stale: |
161 | 75dc539e | Christos Stavrakakis | db_server = self.db_servers[server_id]
|
162 | 75dc539e | Christos Stavrakakis | backend_mod.process_op_status( |
163 | 75dc539e | Christos Stavrakakis | vm=db_server, |
164 | 75dc539e | Christos Stavrakakis | etime=self.event_time,
|
165 | 75dc539e | Christos Stavrakakis | jobid=-0,
|
166 | 75dc539e | Christos Stavrakakis | opcode='OP_INSTANCE_REMOVE', status='success', |
167 | 75dc539e | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
168 | 75dc539e | Christos Stavrakakis | self.log.debug("Simulated Ganeti removal for stale servers.") |
169 | 75dc539e | Christos Stavrakakis | |
170 | 75dc539e | Christos Stavrakakis | def reconcile_orphan_servers(self): |
171 | 75dc539e | Christos Stavrakakis | orphans = self.gnt_servers_keys - self.db_servers_keys |
172 | 75dc539e | Christos Stavrakakis | if orphans:
|
173 | 75dc539e | Christos Stavrakakis | self.log.info("Found orphan servers %s at backend %s", |
174 | 75dc539e | Christos Stavrakakis | ", ".join(map(str, orphans)), self.backend) |
175 | 75dc539e | Christos Stavrakakis | else:
|
176 | 75dc539e | Christos Stavrakakis | self.log.debug("No orphan servers at backend %s", self.backend) |
177 | 75dc539e | Christos Stavrakakis | |
178 | 75dc539e | Christos Stavrakakis | if orphans and self.options["fix_orphans"]: |
179 | 75dc539e | Christos Stavrakakis | for server_id in orphans: |
180 | 75dc539e | Christos Stavrakakis | server_name = utils.id_to_instance_name(server_id) |
181 | 75dc539e | Christos Stavrakakis | self.client.DeleteInstance(server_name)
|
182 | 75dc539e | Christos Stavrakakis | self.log.debug("Issued OP_INSTANCE_REMOVE for orphan servers.") |
183 | 75dc539e | Christos Stavrakakis | |
184 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_servers(self): |
185 | 75dc539e | Christos Stavrakakis | #log = self.log
|
186 | 75dc539e | Christos Stavrakakis | for server_id in self.db_servers_keys & self.gnt_servers_keys: |
187 | 75dc539e | Christos Stavrakakis | db_server = self.db_servers[server_id]
|
188 | 75dc539e | Christos Stavrakakis | gnt_server = self.gnt_servers[server_id]
|
189 | 75dc539e | Christos Stavrakakis | if db_server.operstate == "BUILD": |
190 | 75dc539e | Christos Stavrakakis | build_status = self.get_build_status(db_server)
|
191 | 75dc539e | Christos Stavrakakis | if build_status == "RUNNING": |
192 | 75dc539e | Christos Stavrakakis | # Do not reconcile building VMs
|
193 | 75dc539e | Christos Stavrakakis | continue
|
194 | 75dc539e | Christos Stavrakakis | elif build_status == "ERROR": |
195 | 75dc539e | Christos Stavrakakis | # Special handling of build errors
|
196 | 75dc539e | Christos Stavrakakis | self.reconcile_building_server(db_server)
|
197 | 75dc539e | Christos Stavrakakis | continue
|
198 | 75dc539e | Christos Stavrakakis | |
199 | 75dc539e | Christos Stavrakakis | self.reconcile_unsynced_operstate(server_id, db_server,
|
200 | 75dc539e | Christos Stavrakakis | gnt_server) |
201 | 75dc539e | Christos Stavrakakis | self.reconcile_unsynced_flavor(server_id, db_server,
|
202 | 75dc539e | Christos Stavrakakis | gnt_server) |
203 | 75dc539e | Christos Stavrakakis | self.reconcile_unsynced_nics(server_id, db_server, gnt_server)
|
204 | 75dc539e | Christos Stavrakakis | self.reconcile_unsynced_disks(server_id, db_server, gnt_server)
|
205 | 63f9eb8e | Christos Stavrakakis | if db_server.task is not None: |
206 | 63f9eb8e | Christos Stavrakakis | self.reconcile_pending_task(server_id, db_server)
|
207 | 75dc539e | Christos Stavrakakis | |
208 | 75dc539e | Christos Stavrakakis | def reconcile_building_server(self, db_server): |
209 | 75dc539e | Christos Stavrakakis | self.log.info("Server '%s' is BUILD in DB, but 'ERROR' in Ganeti.", |
210 | 75dc539e | Christos Stavrakakis | db_server.id) |
211 | 75dc539e | Christos Stavrakakis | if self.options["fix_unsynced"]: |
212 | 75dc539e | Christos Stavrakakis | fix_opcode = "OP_INSTANCE_CREATE"
|
213 | 75dc539e | Christos Stavrakakis | backend_mod.process_op_status( |
214 | 75dc539e | Christos Stavrakakis | vm=db_server, |
215 | 75dc539e | Christos Stavrakakis | etime=self.event_time,
|
216 | 75dc539e | Christos Stavrakakis | jobid=-0,
|
217 | 75dc539e | Christos Stavrakakis | opcode=fix_opcode, status='error',
|
218 | 75dc539e | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
219 | 75dc539e | Christos Stavrakakis | self.log.debug("Simulated Ganeti error build event for" |
220 | 75dc539e | Christos Stavrakakis | " server '%s'", db_server.id)
|
221 | 75dc539e | Christos Stavrakakis | |
222 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_operstate(self, server_id, db_server, gnt_server): |
223 | 75dc539e | Christos Stavrakakis | if db_server.operstate != gnt_server["state"]: |
224 | 75dc539e | Christos Stavrakakis | self.log.info("Server '%s' is '%s' in DB and '%s' in Ganeti.", |
225 | 75dc539e | Christos Stavrakakis | server_id, db_server.operstate, gnt_server["state"])
|
226 | 75dc539e | Christos Stavrakakis | if self.options["fix_unsynced"]: |
227 | 5d3e597a | Christos Stavrakakis | # If server is in building state, you will have first to
|
228 | 5d3e597a | Christos Stavrakakis | # reconcile it's creation, to avoid wrong quotas
|
229 | 5d3e597a | Christos Stavrakakis | if db_server.operstate == "BUILD": |
230 | 5d3e597a | Christos Stavrakakis | backend_mod.process_op_status( |
231 | 5d3e597a | Christos Stavrakakis | vm=db_server, etime=self.event_time, jobid=-0, |
232 | 5d3e597a | Christos Stavrakakis | opcode="OP_INSTANCE_CREATE", status='success', |
233 | 5d3e597a | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
234 | 5d3e597a | Christos Stavrakakis | fix_opcode = "OP_INSTANCE_STARTUP"\
|
235 | 5d3e597a | Christos Stavrakakis | if gnt_server["state"] == "STARTED"\ |
236 | 75dc539e | Christos Stavrakakis | else "OP_INSTANCE_SHUTDOWN" |
237 | 75dc539e | Christos Stavrakakis | backend_mod.process_op_status( |
238 | 5d3e597a | Christos Stavrakakis | vm=db_server, etime=self.event_time, jobid=-0, |
239 | 75dc539e | Christos Stavrakakis | opcode=fix_opcode, status='success',
|
240 | 75dc539e | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
241 | 75dc539e | Christos Stavrakakis | self.log.debug("Simulated Ganeti state event for server '%s'", |
242 | 75dc539e | Christos Stavrakakis | server_id) |
243 | 75dc539e | Christos Stavrakakis | |
244 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_flavor(self, server_id, db_server, gnt_server): |
245 | 75dc539e | Christos Stavrakakis | db_flavor = db_server.flavor |
246 | 75dc539e | Christos Stavrakakis | gnt_flavor = gnt_server["flavor"]
|
247 | 75dc539e | Christos Stavrakakis | if (db_flavor.ram != gnt_flavor["ram"] or |
248 | 5d3e597a | Christos Stavrakakis | db_flavor.cpu != gnt_flavor["vcpus"]):
|
249 | a67419d8 | Christos Stavrakakis | try:
|
250 | a67419d8 | Christos Stavrakakis | gnt_flavor = Flavor.objects.get( |
251 | 75dc539e | Christos Stavrakakis | ram=gnt_flavor["ram"],
|
252 | 75dc539e | Christos Stavrakakis | cpu=gnt_flavor["vcpus"],
|
253 | 75dc539e | Christos Stavrakakis | disk=db_flavor.disk, |
254 | 75dc539e | Christos Stavrakakis | disk_template=db_flavor.disk_template) |
255 | a67419d8 | Christos Stavrakakis | except Flavor.DoesNotExist:
|
256 | 75dc539e | Christos Stavrakakis | self.log.warning("Server '%s' has unknown flavor.", server_id) |
257 | 75dc539e | Christos Stavrakakis | return
|
258 | 75dc539e | Christos Stavrakakis | |
259 | 4111c53e | Christos Stavrakakis | self.log.info("Server '%s' has flavor '%s' in DB and '%s' in" |
260 | 75dc539e | Christos Stavrakakis | " Ganeti", server_id, db_flavor, gnt_flavor)
|
261 | 75dc539e | Christos Stavrakakis | if self.options["fix_unsynced_flavors"]: |
262 | 75dc539e | Christos Stavrakakis | old_state = db_server.operstate |
263 | 75dc539e | Christos Stavrakakis | opcode = "OP_INSTANCE_SET_PARAMS"
|
264 | 75dc539e | Christos Stavrakakis | beparams = {"vcpus": gnt_flavor.cpu,
|
265 | 75dc539e | Christos Stavrakakis | "minmem": gnt_flavor.ram,
|
266 | 75dc539e | Christos Stavrakakis | "maxmem": gnt_flavor.ram}
|
267 | 75dc539e | Christos Stavrakakis | backend_mod.process_op_status( |
268 | 75dc539e | Christos Stavrakakis | vm=db_server, etime=self.event_time, jobid=-0, |
269 | 75dc539e | Christos Stavrakakis | opcode=opcode, status='success',
|
270 | e6fbada1 | Christos Stavrakakis | job_fields={"beparams": beparams},
|
271 | 75dc539e | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
272 | 75dc539e | Christos Stavrakakis | # process_op_status with beparams will set the vmstate to
|
273 | 75dc539e | Christos Stavrakakis | # shutdown. Fix this be returning it to old state
|
274 | 75dc539e | Christos Stavrakakis | vm = VirtualMachine.objects.get(pk=server_id) |
275 | 75dc539e | Christos Stavrakakis | vm.operstate = old_state |
276 | 75dc539e | Christos Stavrakakis | vm.save() |
277 | 75dc539e | Christos Stavrakakis | self.log.debug("Simulated Ganeti flavor event for server '%s'", |
278 | 75dc539e | Christos Stavrakakis | server_id) |
279 | 75dc539e | Christos Stavrakakis | |
280 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_nics(self, server_id, db_server, gnt_server): |
281 | 1cb7846c | Christos Stavrakakis | building_time = self.event_time - BUILDING_NIC_TIMEOUT
|
282 | 3a6be177 | Christos Stavrakakis | db_nics = db_server.nics.exclude(state="BUILD",
|
283 | d7ff7f5a | Christos Stavrakakis | created__lte=building_time) \ |
284 | 0d069390 | Christos Stavrakakis | .order_by("id")
|
285 | 75dc539e | Christos Stavrakakis | gnt_nics = gnt_server["nics"]
|
286 | 75dc539e | Christos Stavrakakis | gnt_nics_parsed = backend_mod.process_ganeti_nics(gnt_nics) |
287 | a1baa42b | Christos Stavrakakis | nics_changed = len(db_nics) != len(gnt_nics) |
288 | a1baa42b | Christos Stavrakakis | for db_nic, gnt_nic in zip(db_nics, sorted(gnt_nics_parsed.items())): |
289 | a1baa42b | Christos Stavrakakis | gnt_nic_id, gnt_nic = gnt_nic |
290 | a1baa42b | Christos Stavrakakis | if (db_nic.id == gnt_nic_id) and\ |
291 | a1baa42b | Christos Stavrakakis | backend_mod.nics_are_equal(db_nic, gnt_nic): |
292 | a1baa42b | Christos Stavrakakis | continue
|
293 | a1baa42b | Christos Stavrakakis | else:
|
294 | a1baa42b | Christos Stavrakakis | nics_changed = True
|
295 | a1baa42b | Christos Stavrakakis | break
|
296 | a1baa42b | Christos Stavrakakis | if nics_changed:
|
297 | a1baa42b | Christos Stavrakakis | msg = "Found unsynced NICs for server '%s'.\n"\
|
298 | a1baa42b | Christos Stavrakakis | "\tDB:\n\t\t%s\n\tGaneti:\n\t\t%s"
|
299 | a1baa42b | Christos Stavrakakis | db_nics_str = "\n\t\t".join(map(format_db_nic, db_nics)) |
300 | a1baa42b | Christos Stavrakakis | gnt_nics_str = "\n\t\t".join(map(format_gnt_nic, |
301 | 0d069390 | Christos Stavrakakis | sorted(gnt_nics_parsed.items())))
|
302 | 75dc539e | Christos Stavrakakis | self.log.info(msg, server_id, db_nics_str, gnt_nics_str)
|
303 | 75dc539e | Christos Stavrakakis | if self.options["fix_unsynced_nics"]: |
304 | 75dc539e | Christos Stavrakakis | backend_mod.process_net_status(vm=db_server, |
305 | 75dc539e | Christos Stavrakakis | etime=self.event_time,
|
306 | 75dc539e | Christos Stavrakakis | nics=gnt_nics) |
307 | 75dc539e | Christos Stavrakakis | |
308 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_disks(self, server_id, db_server, gnt_server): |
309 | 75dc539e | Christos Stavrakakis | pass
|
310 | 75dc539e | Christos Stavrakakis | |
311 | 63f9eb8e | Christos Stavrakakis | def reconcile_pending_task(self, server_id, db_server): |
312 | 63f9eb8e | Christos Stavrakakis | job_id = db_server.task_job_id |
313 | 63f9eb8e | Christos Stavrakakis | pending_task = False
|
314 | 63f9eb8e | Christos Stavrakakis | if job_id not in self.gnt_jobs: |
315 | 63f9eb8e | Christos Stavrakakis | pending_task = True
|
316 | 63f9eb8e | Christos Stavrakakis | else:
|
317 | ce55f724 | Christos Stavrakakis | gnt_job_status = self.gnt_jobs[job_id]["status"] |
318 | edbc1d5a | Christos Stavrakakis | if gnt_job_status in rapi.JOB_STATUS_FINALIZED: |
319 | 63f9eb8e | Christos Stavrakakis | pending_task = True
|
320 | 63f9eb8e | Christos Stavrakakis | |
321 | 63f9eb8e | Christos Stavrakakis | if pending_task:
|
322 | 63f9eb8e | Christos Stavrakakis | self.log.info("Found server '%s' with pending task: '%s'", |
323 | 63f9eb8e | Christos Stavrakakis | server_id, db_server.task) |
324 | 160c81a1 | Christos Stavrakakis | if self.options["fix_pending_tasks"]: |
325 | 63f9eb8e | Christos Stavrakakis | db_server.task = None
|
326 | 63f9eb8e | Christos Stavrakakis | db_server.task_job_id = None
|
327 | 63f9eb8e | Christos Stavrakakis | db_server.save() |
328 | 63f9eb8e | Christos Stavrakakis | self.log.info("Cleared pending task for server '%s", server_id) |
329 | 63f9eb8e | Christos Stavrakakis | |
330 | 75dc539e | Christos Stavrakakis | |
331 | bfd04b01 | Christos Stavrakakis | NIC_MSG = ": %s\t".join(["ID", "State", "IP", "Network", "MAC", "Index", |
332 | bfd04b01 | Christos Stavrakakis | "Firewall"]) + ": %s" |
333 | a1baa42b | Christos Stavrakakis | |
334 | a1baa42b | Christos Stavrakakis | |
335 | 75dc539e | Christos Stavrakakis | def format_db_nic(nic): |
336 | 8764d304 | Christos Stavrakakis | return NIC_MSG % (nic.id, nic.state, nic.ipv4_address, nic.network_id,
|
337 | 8764d304 | Christos Stavrakakis | nic.mac, nic.index, nic.firewall_profile) |
338 | 75dc539e | Christos Stavrakakis | |
339 | 75dc539e | Christos Stavrakakis | |
340 | 75dc539e | Christos Stavrakakis | def format_gnt_nic(nic): |
341 | a1baa42b | Christos Stavrakakis | nic_name, nic = nic |
342 | 8764d304 | Christos Stavrakakis | return NIC_MSG % (nic_name, nic["state"], nic["ipv4_address"], |
343 | 8764d304 | Christos Stavrakakis | nic["network"].id, nic["mac"], nic["index"], |
344 | 8764d304 | Christos Stavrakakis | nic["firewall_profile"])
|
345 | 9fea53cc | Vangelis Koukis | |
346 | cc92b70f | Christos Stavrakakis | |
347 | 0e9a423f | Christos Stavrakakis | #
|
348 | 0e9a423f | Christos Stavrakakis | # Networks
|
349 | 0e9a423f | Christos Stavrakakis | #
|
350 | 3524241a | Christos Stavrakakis | |
351 | 3524241a | Christos Stavrakakis | |
352 | 0e9a423f | Christos Stavrakakis | def get_networks_from_ganeti(backend): |
353 | 44e2c577 | Christos Stavrakakis | prefix = settings.BACKEND_PREFIX_ID + 'net-'
|
354 | 0e9a423f | Christos Stavrakakis | |
355 | 0e9a423f | Christos Stavrakakis | networks = {} |
356 | 3524241a | Christos Stavrakakis | with pooled_rapi_client(backend) as c: |
357 | 3524241a | Christos Stavrakakis | for net in c.GetNetworks(bulk=True): |
358 | 3524241a | Christos Stavrakakis | if net['name'].startswith(prefix): |
359 | 3524241a | Christos Stavrakakis | id = utils.id_from_network_name(net['name'])
|
360 | 3524241a | Christos Stavrakakis | networks[id] = net
|
361 | 0e9a423f | Christos Stavrakakis | |
362 | 0e9a423f | Christos Stavrakakis | return networks
|
363 | 0e9a423f | Christos Stavrakakis | |
364 | 0e9a423f | Christos Stavrakakis | |
365 | 0e9a423f | Christos Stavrakakis | def hanging_networks(backend, GNets): |
366 | 0e9a423f | Christos Stavrakakis | """Get networks that are not connected to all Nodegroups.
|
367 | 0e9a423f | Christos Stavrakakis |
|
368 | 0e9a423f | Christos Stavrakakis | """
|
369 | 0e9a423f | Christos Stavrakakis | def get_network_groups(group_list): |
370 | 0e9a423f | Christos Stavrakakis | groups = set()
|
371 | a9bb2a1a | Christos Stavrakakis | for (name, mode, link) in group_list: |
372 | a9bb2a1a | Christos Stavrakakis | groups.add(name) |
373 | 0e9a423f | Christos Stavrakakis | return groups
|
374 | 0e9a423f | Christos Stavrakakis | |
375 | 3524241a | Christos Stavrakakis | with pooled_rapi_client(backend) as c: |
376 | 3524241a | Christos Stavrakakis | groups = set(c.GetGroups())
|
377 | 0e9a423f | Christos Stavrakakis | |
378 | 0e9a423f | Christos Stavrakakis | hanging = {} |
379 | 0e9a423f | Christos Stavrakakis | for id, info in GNets.items(): |
380 | 0e9a423f | Christos Stavrakakis | group_list = get_network_groups(info['group_list'])
|
381 | 0e9a423f | Christos Stavrakakis | if group_list != groups:
|
382 | 0e9a423f | Christos Stavrakakis | hanging[id] = groups - group_list
|
383 | 0e9a423f | Christos Stavrakakis | return hanging
|
384 | 0e9a423f | Christos Stavrakakis | |
385 | 9fea53cc | Vangelis Koukis | |
386 | 75dc539e | Christos Stavrakakis | def get_online_backends(): |
387 | 75dc539e | Christos Stavrakakis | return Backend.objects.filter(offline=False) |
388 | 75dc539e | Christos Stavrakakis | |
389 | 75dc539e | Christos Stavrakakis | |
390 | 75dc539e | Christos Stavrakakis | def get_database_servers(backend): |
391 | 3278725f | Christos Stavrakakis | servers = backend.virtual_machines.select_related("flavor")\
|
392 | 3278725f | Christos Stavrakakis | .prefetch_related("nics__ips__subnet")\
|
393 | 75dc539e | Christos Stavrakakis | .filter(deleted=False)
|
394 | 75dc539e | Christos Stavrakakis | return dict([(s.id, s) for s in servers]) |
395 | 75dc539e | Christos Stavrakakis | |
396 | 75dc539e | Christos Stavrakakis | |
397 | 75dc539e | Christos Stavrakakis | def get_ganeti_servers(backend): |
398 | 75dc539e | Christos Stavrakakis | gnt_instances = backend_mod.get_instances(backend) |
399 | 75dc539e | Christos Stavrakakis | # Filter out non-synnefo instances
|
400 | 75dc539e | Christos Stavrakakis | snf_backend_prefix = settings.BACKEND_PREFIX_ID |
401 | 75dc539e | Christos Stavrakakis | gnt_instances = filter(lambda i: i["name"].startswith(snf_backend_prefix), |
402 | 75dc539e | Christos Stavrakakis | gnt_instances) |
403 | 75dc539e | Christos Stavrakakis | gnt_instances = map(parse_gnt_instance, gnt_instances)
|
404 | 75dc539e | Christos Stavrakakis | return dict([(i["id"], i) for i in gnt_instances if i["id"] is not None]) |
405 | 75dc539e | Christos Stavrakakis | |
406 | 75dc539e | Christos Stavrakakis | |
407 | 75dc539e | Christos Stavrakakis | def parse_gnt_instance(instance): |
408 | 75dc539e | Christos Stavrakakis | try:
|
409 | 75dc539e | Christos Stavrakakis | instance_id = utils.id_from_instance_name(instance['name'])
|
410 | 75dc539e | Christos Stavrakakis | except Exception: |
411 | 75dc539e | Christos Stavrakakis | logger.error("Ignoring instance with malformed name %s",
|
412 | 75dc539e | Christos Stavrakakis | instance['name'])
|
413 | 75dc539e | Christos Stavrakakis | return (None, None) |
414 | 75dc539e | Christos Stavrakakis | |
415 | 75dc539e | Christos Stavrakakis | beparams = instance["beparams"]
|
416 | 75dc539e | Christos Stavrakakis | |
417 | 75dc539e | Christos Stavrakakis | vcpus = beparams["vcpus"]
|
418 | 75dc539e | Christos Stavrakakis | ram = beparams["maxmem"]
|
419 | 75dc539e | Christos Stavrakakis | state = instance["oper_state"] and "STARTED" or "STOPPED" |
420 | 75dc539e | Christos Stavrakakis | |
421 | 75dc539e | Christos Stavrakakis | return {
|
422 | 75dc539e | Christos Stavrakakis | "id": instance_id,
|
423 | 75dc539e | Christos Stavrakakis | "state": state, # FIX |
424 | 75dc539e | Christos Stavrakakis | "updated": datetime.fromtimestamp(instance["mtime"]), |
425 | 75dc539e | Christos Stavrakakis | "disks": disks_from_instance(instance),
|
426 | 75dc539e | Christos Stavrakakis | "nics": nics_from_instance(instance),
|
427 | 75dc539e | Christos Stavrakakis | "flavor": {"vcpus": vcpus, |
428 | 75dc539e | Christos Stavrakakis | "ram": ram},
|
429 | 75dc539e | Christos Stavrakakis | "tags": instance["tags"] |
430 | 75dc539e | Christos Stavrakakis | } |
431 | 75dc539e | Christos Stavrakakis | |
432 | 75dc539e | Christos Stavrakakis | |
433 | 75dc539e | Christos Stavrakakis | def nics_from_instance(i): |
434 | 75dc539e | Christos Stavrakakis | ips = zip(itertools.repeat('ip'), i['nic.ips']) |
435 | a1baa42b | Christos Stavrakakis | names = zip(itertools.repeat('name'), i['nic.names']) |
436 | 75dc539e | Christos Stavrakakis | macs = zip(itertools.repeat('mac'), i['nic.macs']) |
437 | a1baa42b | Christos Stavrakakis | networks = zip(itertools.repeat('network'), i['nic.networks.names']) |
438 | 75dc539e | Christos Stavrakakis | # modes = zip(itertools.repeat('mode'), i['nic.modes'])
|
439 | 75dc539e | Christos Stavrakakis | # links = zip(itertools.repeat('link'), i['nic.links'])
|
440 | 75dc539e | Christos Stavrakakis | # nics = zip(ips,macs,modes,networks,links)
|
441 | a1baa42b | Christos Stavrakakis | nics = zip(ips, names, macs, networks)
|
442 | 75dc539e | Christos Stavrakakis | nics = map(lambda x: dict(x), nics) |
443 | 75dc539e | Christos Stavrakakis | #nics = dict(enumerate(nics))
|
444 | 75dc539e | Christos Stavrakakis | tags = i["tags"]
|
445 | 75dc539e | Christos Stavrakakis | for tag in tags: |
446 | 75dc539e | Christos Stavrakakis | t = tag.split(":")
|
447 | 75dc539e | Christos Stavrakakis | if t[0:2] == ["synnefo", "network"]: |
448 | 75dc539e | Christos Stavrakakis | if len(t) != 4: |
449 | 75dc539e | Christos Stavrakakis | logger.error("Malformed synefo tag %s", tag)
|
450 | 75dc539e | Christos Stavrakakis | continue
|
451 | d0545590 | Christos Stavrakakis | nic_name = t[2]
|
452 | d0545590 | Christos Stavrakakis | firewall = t[3]
|
453 | d0545590 | Christos Stavrakakis | [nic.setdefault("firewall", firewall)
|
454 | d0545590 | Christos Stavrakakis | for nic in nics if nic["name"] == nic_name] |
455 | 75dc539e | Christos Stavrakakis | return nics
|
456 | 9fea53cc | Vangelis Koukis | |
457 | 9fea53cc | Vangelis Koukis | |
458 | 63f9eb8e | Christos Stavrakakis | def get_ganeti_jobs(backend): |
459 | 63f9eb8e | Christos Stavrakakis | gnt_jobs = backend_mod.get_jobs(backend) |
460 | 63f9eb8e | Christos Stavrakakis | return dict([(int(j["id"]), j) for j in gnt_jobs]) |
461 | 63f9eb8e | Christos Stavrakakis | |
462 | 63f9eb8e | Christos Stavrakakis | |
463 | 75dc539e | Christos Stavrakakis | def disks_from_instance(i): |
464 | 75dc539e | Christos Stavrakakis | return dict([(index, {"size": size}) |
465 | 75dc539e | Christos Stavrakakis | for index, size in enumerate(i["disk.sizes"])]) |
466 | 89b2b908 | Christos Stavrakakis | |
467 | 89b2b908 | Christos Stavrakakis | |
468 | 89b2b908 | Christos Stavrakakis | class NetworkReconciler(object): |
469 | 0ccb6461 | Christos Stavrakakis | def __init__(self, logger, fix=False): |
470 | 89b2b908 | Christos Stavrakakis | self.log = logger
|
471 | 89b2b908 | Christos Stavrakakis | self.fix = fix
|
472 | 89b2b908 | Christos Stavrakakis | |
473 | 89b2b908 | Christos Stavrakakis | @transaction.commit_on_success
|
474 | 89b2b908 | Christos Stavrakakis | def reconcile_networks(self): |
475 | 89b2b908 | Christos Stavrakakis | # Get models from DB
|
476 | 0ccb6461 | Christos Stavrakakis | self.backends = Backend.objects.exclude(offline=True) |
477 | 0ccb6461 | Christos Stavrakakis | self.networks = Network.objects.filter(deleted=False) |
478 | 89b2b908 | Christos Stavrakakis | |
479 | 89b2b908 | Christos Stavrakakis | self.event_time = datetime.now()
|
480 | 89b2b908 | Christos Stavrakakis | |
481 | 89b2b908 | Christos Stavrakakis | # Get info from all ganeti backends
|
482 | 0ccb6461 | Christos Stavrakakis | self.ganeti_networks = {}
|
483 | 0ccb6461 | Christos Stavrakakis | self.ganeti_hanging_networks = {}
|
484 | 0ccb6461 | Christos Stavrakakis | for b in self.backends: |
485 | 89b2b908 | Christos Stavrakakis | g_nets = get_networks_from_ganeti(b) |
486 | 0ccb6461 | Christos Stavrakakis | self.ganeti_networks[b] = g_nets
|
487 | 89b2b908 | Christos Stavrakakis | g_hanging_nets = hanging_networks(b, g_nets) |
488 | 0ccb6461 | Christos Stavrakakis | self.ganeti_hanging_networks[b] = g_hanging_nets
|
489 | 89b2b908 | Christos Stavrakakis | |
490 | 0ccb6461 | Christos Stavrakakis | self._reconcile_orphan_networks()
|
491 | 89b2b908 | Christos Stavrakakis | |
492 | 0ccb6461 | Christos Stavrakakis | for network in self.networks: |
493 | 0ccb6461 | Christos Stavrakakis | self._reconcile_network(network)
|
494 | 89b2b908 | Christos Stavrakakis | |
495 | 0ccb6461 | Christos Stavrakakis | @transaction.commit_on_success
|
496 | 0ccb6461 | Christos Stavrakakis | def _reconcile_network(self, network): |
497 | 0ccb6461 | Christos Stavrakakis | """Reconcile a network with corresponging Ganeti networks.
|
498 | 0ccb6461 | Christos Stavrakakis |
|
499 | 0ccb6461 | Christos Stavrakakis | Reconcile a Network and the associated BackendNetworks with the
|
500 | 0ccb6461 | Christos Stavrakakis | corresponding Ganeti networks in all Ganeti backends.
|
501 | 0ccb6461 | Christos Stavrakakis |
|
502 | 0ccb6461 | Christos Stavrakakis | """
|
503 | 4cbd934b | Christos Stavrakakis | if network.subnets.filter(ipversion=4, dhcp=True).exists(): |
504 | 4cbd934b | Christos Stavrakakis | ip_pools = network.get_ip_pools() # X-Lock on IP pools
|
505 | 4cbd934b | Christos Stavrakakis | else:
|
506 | 4cbd934b | Christos Stavrakakis | ip_pools = None
|
507 | 0ccb6461 | Christos Stavrakakis | for bend in self.backends: |
508 | 0ccb6461 | Christos Stavrakakis | bnet = get_backend_network(network, bend) |
509 | 0ccb6461 | Christos Stavrakakis | gnet = self.ganeti_networks[bend].get(network.id)
|
510 | 84a0469b | Christos Stavrakakis | if bnet is None and gnet is not None: |
511 | 84a0469b | Christos Stavrakakis | # Network exists in backend but not in DB for this backend
|
512 | 84a0469b | Christos Stavrakakis | bnet = self.reconcile_parted_network(network, bend)
|
513 | 84a0469b | Christos Stavrakakis | |
514 | 84a0469b | Christos Stavrakakis | if bnet is None: |
515 | 84a0469b | Christos Stavrakakis | continue
|
516 | 84a0469b | Christos Stavrakakis | |
517 | 84a0469b | Christos Stavrakakis | if gnet is None: |
518 | 0ccb6461 | Christos Stavrakakis | # Network does not exist in Ganeti. If the network action
|
519 | 0ccb6461 | Christos Stavrakakis | # is DESTROY, we have to mark as deleted in DB, else we
|
520 | 0ccb6461 | Christos Stavrakakis | # have to create it in Ganeti.
|
521 | 0ccb6461 | Christos Stavrakakis | if network.action == "DESTROY": |
522 | 0ccb6461 | Christos Stavrakakis | if bnet.operstate != "DELETED": |
523 | 0ccb6461 | Christos Stavrakakis | self.reconcile_stale_network(bnet)
|
524 | 0ccb6461 | Christos Stavrakakis | else:
|
525 | 0ccb6461 | Christos Stavrakakis | self.reconcile_missing_network(network, bend)
|
526 | 0ccb6461 | Christos Stavrakakis | # Skip rest reconciliation!
|
527 | 0ccb6461 | Christos Stavrakakis | continue
|
528 | 89b2b908 | Christos Stavrakakis | |
529 | 0ccb6461 | Christos Stavrakakis | try:
|
530 | 0ccb6461 | Christos Stavrakakis | hanging_groups = self.ganeti_hanging_networks[bend][network.id]
|
531 | 0ccb6461 | Christos Stavrakakis | except KeyError: |
532 | 0ccb6461 | Christos Stavrakakis | # Network is connected to all nodegroups
|
533 | 0ccb6461 | Christos Stavrakakis | hanging_groups = [] |
534 | 0ccb6461 | Christos Stavrakakis | |
535 | 0ccb6461 | Christos Stavrakakis | if hanging_groups:
|
536 | 0ccb6461 | Christos Stavrakakis | # CASE-3: Ganeti networks not connected to all nodegroups
|
537 | 0ccb6461 | Christos Stavrakakis | self.reconcile_hanging_groups(network, bend,
|
538 | 0ccb6461 | Christos Stavrakakis | hanging_groups) |
539 | 0ccb6461 | Christos Stavrakakis | continue
|
540 | 89b2b908 | Christos Stavrakakis | |
541 | 0ccb6461 | Christos Stavrakakis | if bnet.operstate != 'ACTIVE': |
542 | 0ccb6461 | Christos Stavrakakis | # CASE-4: Unsynced network state. At this point the network
|
543 | 0ccb6461 | Christos Stavrakakis | # exists and is connected to all nodes so is must be
|
544 | 0ccb6461 | Christos Stavrakakis | # active!
|
545 | 0ccb6461 | Christos Stavrakakis | self.reconcile_unsynced_network(network, bend, bnet)
|
546 | 0ccb6461 | Christos Stavrakakis | |
547 | 0ccb6461 | Christos Stavrakakis | # Check that externally reserved IPs of the network in Ganeti are
|
548 | 0ccb6461 | Christos Stavrakakis | # also externally reserved to the IP pool
|
549 | 0ccb6461 | Christos Stavrakakis | externally_reserved = gnet['external_reservations']
|
550 | 4cbd934b | Christos Stavrakakis | if externally_reserved and ip_pools is not None: |
551 | 98a01362 | Christos Stavrakakis | for ip in externally_reserved.split(","): |
552 | 98a01362 | Christos Stavrakakis | ip = ip.strip() |
553 | 0e0a6ef9 | Christos Stavrakakis | for ip_pool in ip_pools: |
554 | 0e0a6ef9 | Christos Stavrakakis | if ip_pool.contains(ip):
|
555 | 0e0a6ef9 | Christos Stavrakakis | if not ip_pool.is_reserved(ip): |
556 | 0e0a6ef9 | Christos Stavrakakis | msg = ("D: IP '%s' is reserved for network"
|
557 | 0e0a6ef9 | Christos Stavrakakis | " '%s' in backend '%s' but not in DB.")
|
558 | 0e0a6ef9 | Christos Stavrakakis | self.log.info(msg, ip, network, bend)
|
559 | 0e0a6ef9 | Christos Stavrakakis | if self.fix: |
560 | 0e0a6ef9 | Christos Stavrakakis | ip_pool.reserve(ip, external=True)
|
561 | 0e0a6ef9 | Christos Stavrakakis | ip_pool.save() |
562 | 0e0a6ef9 | Christos Stavrakakis | self.log.info("F: Reserved IP '%s'", ip) |
563 | ec4e0acc | Christos Stavrakakis | if network.state != "ACTIVE": |
564 | ec4e0acc | Christos Stavrakakis | network = Network.objects.select_for_update().get(id=network.id) |
565 | ec4e0acc | Christos Stavrakakis | backend_mod.update_network_state(network) |
566 | 89b2b908 | Christos Stavrakakis | |
567 | 89b2b908 | Christos Stavrakakis | def reconcile_parted_network(self, network, backend): |
568 | 89b2b908 | Christos Stavrakakis | self.log.info("D: Missing DB entry for network %s in backend %s", |
569 | 89b2b908 | Christos Stavrakakis | network, backend) |
570 | 89b2b908 | Christos Stavrakakis | if self.fix: |
571 | 89b2b908 | Christos Stavrakakis | network.create_backend_network(backend) |
572 | 89b2b908 | Christos Stavrakakis | self.log.info("F: Created DB entry") |
573 | 89b2b908 | Christos Stavrakakis | bnet = get_backend_network(network, backend) |
574 | 89b2b908 | Christos Stavrakakis | return bnet
|
575 | 89b2b908 | Christos Stavrakakis | |
576 | 89b2b908 | Christos Stavrakakis | def reconcile_stale_network(self, backend_network): |
577 | 89b2b908 | Christos Stavrakakis | self.log.info("D: Stale DB entry for network %s in backend %s", |
578 | 89b2b908 | Christos Stavrakakis | backend_network.network, backend_network.backend) |
579 | 89b2b908 | Christos Stavrakakis | if self.fix: |
580 | 89b2b908 | Christos Stavrakakis | backend_mod.process_network_status( |
581 | 89b2b908 | Christos Stavrakakis | backend_network, self.event_time, 0, |
582 | 89b2b908 | Christos Stavrakakis | "OP_NETWORK_REMOVE",
|
583 | 89b2b908 | Christos Stavrakakis | "success",
|
584 | 89b2b908 | Christos Stavrakakis | "Reconciliation simulated event")
|
585 | 89b2b908 | Christos Stavrakakis | self.log.info("F: Reconciled event: OP_NETWORK_REMOVE") |
586 | 89b2b908 | Christos Stavrakakis | |
587 | 89b2b908 | Christos Stavrakakis | def reconcile_missing_network(self, network, backend): |
588 | 89b2b908 | Christos Stavrakakis | self.log.info("D: Missing Ganeti network %s in backend %s", |
589 | 89b2b908 | Christos Stavrakakis | network, backend) |
590 | 89b2b908 | Christos Stavrakakis | if self.fix: |
591 | 89b2b908 | Christos Stavrakakis | backend_mod.create_network(network, backend) |
592 | 89b2b908 | Christos Stavrakakis | self.log.info("F: Issued OP_NETWORK_CONNECT") |
593 | 89b2b908 | Christos Stavrakakis | |
594 | 89b2b908 | Christos Stavrakakis | def reconcile_hanging_groups(self, network, backend, hanging_groups): |
595 | 89b2b908 | Christos Stavrakakis | self.log.info('D: Network %s in backend %s is not connected to ' |
596 | 89b2b908 | Christos Stavrakakis | 'the following groups:', network, backend)
|
597 | 89b2b908 | Christos Stavrakakis | self.log.info('- ' + '\n- '.join(hanging_groups)) |
598 | 89b2b908 | Christos Stavrakakis | if self.fix: |
599 | 89b2b908 | Christos Stavrakakis | for group in hanging_groups: |
600 | 89b2b908 | Christos Stavrakakis | self.log.info('F: Connecting network %s to nodegroup %s', |
601 | 89b2b908 | Christos Stavrakakis | network, group) |
602 | 89b2b908 | Christos Stavrakakis | backend_mod.connect_network(network, backend, depends=[], |
603 | 89b2b908 | Christos Stavrakakis | group=group) |
604 | 89b2b908 | Christos Stavrakakis | |
605 | 89b2b908 | Christos Stavrakakis | def reconcile_unsynced_network(self, network, backend, backend_network): |
606 | 89b2b908 | Christos Stavrakakis | self.log.info("D: Unsynced network %s in backend %s", network, backend) |
607 | 89b2b908 | Christos Stavrakakis | if self.fix: |
608 | 89b2b908 | Christos Stavrakakis | self.log.info("F: Issuing OP_NETWORK_CONNECT") |
609 | 89b2b908 | Christos Stavrakakis | backend_mod.process_network_status( |
610 | 89b2b908 | Christos Stavrakakis | backend_network, self.event_time, 0, |
611 | 89b2b908 | Christos Stavrakakis | "OP_NETWORK_CONNECT",
|
612 | 89b2b908 | Christos Stavrakakis | "success",
|
613 | 89b2b908 | Christos Stavrakakis | "Reconciliation simulated eventd")
|
614 | 89b2b908 | Christos Stavrakakis | |
615 | 0ccb6461 | Christos Stavrakakis | def _reconcile_orphan_networks(self): |
616 | 0ccb6461 | Christos Stavrakakis | db_networks = self.networks
|
617 | 0ccb6461 | Christos Stavrakakis | ganeti_networks = self.ganeti_networks
|
618 | 89b2b908 | Christos Stavrakakis | # Detect Orphan Networks in Ganeti
|
619 | 89b2b908 | Christos Stavrakakis | db_network_ids = set([net.id for net in db_networks]) |
620 | 89b2b908 | Christos Stavrakakis | for back_end, ganeti_networks in ganeti_networks.items(): |
621 | 89b2b908 | Christos Stavrakakis | ganeti_network_ids = set(ganeti_networks.keys())
|
622 | 89b2b908 | Christos Stavrakakis | orphans = ganeti_network_ids - db_network_ids |
623 | 89b2b908 | Christos Stavrakakis | |
624 | 89b2b908 | Christos Stavrakakis | if len(orphans) > 0: |
625 | 89b2b908 | Christos Stavrakakis | self.log.info('D: Orphan Networks in backend %s:', |
626 | 89b2b908 | Christos Stavrakakis | back_end.clustername) |
627 | 89b2b908 | Christos Stavrakakis | self.log.info('- ' + '\n- '.join([str(o) for o in orphans])) |
628 | 89b2b908 | Christos Stavrakakis | if self.fix: |
629 | 89b2b908 | Christos Stavrakakis | for net_id in orphans: |
630 | 89b2b908 | Christos Stavrakakis | self.log.info('Disconnecting and deleting network %d', |
631 | 89b2b908 | Christos Stavrakakis | net_id) |
632 | 89b2b908 | Christos Stavrakakis | try:
|
633 | 89b2b908 | Christos Stavrakakis | network = Network.objects.get(id=net_id) |
634 | 89b2b908 | Christos Stavrakakis | backend_mod.delete_network(network, |
635 | 89b2b908 | Christos Stavrakakis | backend=back_end) |
636 | 89b2b908 | Christos Stavrakakis | except Network.DoesNotExist:
|
637 | 89b2b908 | Christos Stavrakakis | self.log.info("Not entry for network %s in DB !!", |
638 | 89b2b908 | Christos Stavrakakis | net_id) |
639 | 89b2b908 | Christos Stavrakakis | |
640 | 89b2b908 | Christos Stavrakakis | |
641 | 89b2b908 | Christos Stavrakakis | def get_backend_network(network, backend): |
642 | 89b2b908 | Christos Stavrakakis | try:
|
643 | 89b2b908 | Christos Stavrakakis | return BackendNetwork.objects.get(network=network, backend=backend)
|
644 | 89b2b908 | Christos Stavrakakis | except BackendNetwork.DoesNotExist:
|
645 | 89b2b908 | Christos Stavrakakis | return None |
646 | 89b2b908 | Christos Stavrakakis | |
647 | 89b2b908 | Christos Stavrakakis | |
648 | 0ccb6461 | Christos Stavrakakis | class PoolReconciler(object): |
649 | 0ccb6461 | Christos Stavrakakis | def __init__(self, logger, fix=False): |
650 | 0ccb6461 | Christos Stavrakakis | self.log = logger
|
651 | 0ccb6461 | Christos Stavrakakis | self.fix = fix
|
652 | 0ccb6461 | Christos Stavrakakis | |
653 | 0ccb6461 | Christos Stavrakakis | def reconcile(self): |
654 | 0ccb6461 | Christos Stavrakakis | self.reconcile_bridges()
|
655 | 0ccb6461 | Christos Stavrakakis | self.reconcile_mac_prefixes()
|
656 | 4cbd934b | Christos Stavrakakis | |
657 | 4cbd934b | Christos Stavrakakis | networks = Network.objects.prefetch_related("subnets")\
|
658 | 4cbd934b | Christos Stavrakakis | .filter(deleted=False)
|
659 | 4cbd934b | Christos Stavrakakis | for network in networks: |
660 | 4cbd934b | Christos Stavrakakis | for subnet in network.subnets.all(): |
661 | 4cbd934b | Christos Stavrakakis | if subnet.ipversion == 4 and subnet.dhcp: |
662 | 4cbd934b | Christos Stavrakakis | self.reconcile_ip_pool(network)
|
663 | 0ccb6461 | Christos Stavrakakis | |
664 | 0ccb6461 | Christos Stavrakakis | @transaction.commit_on_success
|
665 | 0ccb6461 | Christos Stavrakakis | def reconcile_bridges(self): |
666 | 0ccb6461 | Christos Stavrakakis | networks = Network.objects.filter(deleted=False,
|
667 | 0ccb6461 | Christos Stavrakakis | flavor="PHYSICAL_VLAN")
|
668 | 0ccb6461 | Christos Stavrakakis | check_unique_values(objects=networks, field='link', logger=self.log) |
669 | 0ccb6461 | Christos Stavrakakis | try:
|
670 | 0ccb6461 | Christos Stavrakakis | pool = BridgePoolTable.get_pool() |
671 | 0ccb6461 | Christos Stavrakakis | except pools.EmptyPool:
|
672 | 0ccb6461 | Christos Stavrakakis | self.log.info("There is no available pool for bridges.") |
673 | 0ccb6461 | Christos Stavrakakis | return
|
674 | 0ccb6461 | Christos Stavrakakis | |
675 | ec4e0acc | Christos Stavrakakis | # Since pool is locked, no new networks may be created
|
676 | 0ccb6461 | Christos Stavrakakis | used_bridges = set(networks.values_list('link', flat=True)) |
677 | 0ccb6461 | Christos Stavrakakis | check_pool_consistent(pool=pool, pool_class=pools.BridgePool, |
678 | 0ccb6461 | Christos Stavrakakis | used_values=used_bridges, fix=self.fix,
|
679 | 0ccb6461 | Christos Stavrakakis | logger=self.log)
|
680 | 89b2b908 | Christos Stavrakakis | |
681 | 0ccb6461 | Christos Stavrakakis | @transaction.commit_on_success
|
682 | 0ccb6461 | Christos Stavrakakis | def reconcile_mac_prefixes(self): |
683 | 0ccb6461 | Christos Stavrakakis | networks = Network.objects.filter(deleted=False, flavor="MAC_FILTERED") |
684 | 0ccb6461 | Christos Stavrakakis | check_unique_values(objects=networks, field='mac_prefix',
|
685 | 0ccb6461 | Christos Stavrakakis | logger=self.log)
|
686 | 0ccb6461 | Christos Stavrakakis | try:
|
687 | 0ccb6461 | Christos Stavrakakis | pool = MacPrefixPoolTable.get_pool() |
688 | 0ccb6461 | Christos Stavrakakis | except pools.EmptyPool:
|
689 | 0ccb6461 | Christos Stavrakakis | self.log.info("There is no available pool for MAC prefixes.") |
690 | 0ccb6461 | Christos Stavrakakis | return
|
691 | 0ccb6461 | Christos Stavrakakis | |
692 | ec4e0acc | Christos Stavrakakis | # Since pool is locked, no new network may be created
|
693 | 0ccb6461 | Christos Stavrakakis | used_mac_prefixes = set(networks.values_list('mac_prefix', flat=True)) |
694 | 0ccb6461 | Christos Stavrakakis | check_pool_consistent(pool=pool, pool_class=pools.MacPrefixPool, |
695 | 0ccb6461 | Christos Stavrakakis | used_values=used_mac_prefixes, fix=self.fix,
|
696 | 0ccb6461 | Christos Stavrakakis | logger=self.log)
|
697 | 89b2b908 | Christos Stavrakakis | |
698 | 0ccb6461 | Christos Stavrakakis | @transaction.commit_on_success
|
699 | 0ccb6461 | Christos Stavrakakis | def reconcile_ip_pool(self, network): |
700 | 0ccb6461 | Christos Stavrakakis | # Check that all NICs have unique IPv4 address
|
701 | 4cbd934b | Christos Stavrakakis | nics = network.ips.exclude(address__isnull=True).all()
|
702 | 0e0a6ef9 | Christos Stavrakakis | check_unique_values(objects=nics, field="address", logger=self.log) |
703 | 0e0a6ef9 | Christos Stavrakakis | |
704 | 0e0a6ef9 | Christos Stavrakakis | for ip_pool in network.get_ip_pools(): |
705 | ec4e0acc | Christos Stavrakakis | # IP pool is now locked, so no new IPs may be created
|
706 | 4cbd934b | Christos Stavrakakis | used_ips = ip_pool.pool_table.subnet\ |
707 | 4cbd934b | Christos Stavrakakis | .ips.exclude(address__isnull=True)\
|
708 | c82f57ad | Christos Stavrakakis | .exclude(deleted=True)\
|
709 | 4cbd934b | Christos Stavrakakis | .values_list("address", flat=True) |
710 | 0e0a6ef9 | Christos Stavrakakis | used_ips = filter(lambda x: ip_pool.contains(x), used_ips) |
711 | 0e0a6ef9 | Christos Stavrakakis | check_pool_consistent(pool=ip_pool, |
712 | 0e0a6ef9 | Christos Stavrakakis | pool_class=pools.IPPool, |
713 | 0e0a6ef9 | Christos Stavrakakis | used_values=used_ips, |
714 | 0e0a6ef9 | Christos Stavrakakis | fix=self.fix, logger=self.log) |
715 | 0ccb6461 | Christos Stavrakakis | |
716 | 0ccb6461 | Christos Stavrakakis | |
717 | 0ccb6461 | Christos Stavrakakis | def check_unique_values(objects, field, logger): |
718 | 0ccb6461 | Christos Stavrakakis | used_values = list(objects.values_list(field, flat=True)) |
719 | 0ccb6461 | Christos Stavrakakis | if len(used_values) != len(set(used_values)): |
720 | 0ccb6461 | Christos Stavrakakis | duplicate_values = [v for v in used_values if used_values.count(v) > 1] |
721 | 0ccb6461 | Christos Stavrakakis | for value in duplicate_values: |
722 | 0ccb6461 | Christos Stavrakakis | filter_args = {field: value} |
723 | 0ccb6461 | Christos Stavrakakis | using_objects = objects.filter(**filter_args) |
724 | 0ccb6461 | Christos Stavrakakis | msg = "Value '%s' is used as %s for more than one objects: %s"
|
725 | 0ccb6461 | Christos Stavrakakis | logger.error(msg, value, field, ",".join(map(str, using_objects))) |
726 | 0ccb6461 | Christos Stavrakakis | return False |
727 | 0ccb6461 | Christos Stavrakakis | logger.debug("Values for field '%s' are unique.", field)
|
728 | 0ccb6461 | Christos Stavrakakis | return True |
729 | 0ccb6461 | Christos Stavrakakis | |
730 | 0ccb6461 | Christos Stavrakakis | |
731 | 0ccb6461 | Christos Stavrakakis | def check_pool_consistent(pool, pool_class, used_values, fix, logger): |
732 | 0ccb6461 | Christos Stavrakakis | dummy_pool = create_empty_pool(pool, pool_class) |
733 | 0ccb6461 | Christos Stavrakakis | [dummy_pool.reserve(value) for value in used_values] |
734 | 0ccb6461 | Christos Stavrakakis | if dummy_pool.available != pool.available:
|
735 | 0ccb6461 | Christos Stavrakakis | msg = "'%s' is not consistent!\nPool: %s\nUsed: %s"
|
736 | 0ccb6461 | Christos Stavrakakis | pool_diff = dummy_pool.available ^ pool.available |
737 | 0ccb6461 | Christos Stavrakakis | for index in pool_diff.itersearch(bitarray.bitarray("1")): |
738 | 0ccb6461 | Christos Stavrakakis | value = pool.index_to_value(int(index))
|
739 | 0ccb6461 | Christos Stavrakakis | msg = "%s is incosistent! Value '%s' is %s but should be %s."
|
740 | 0ccb6461 | Christos Stavrakakis | value1 = pool.is_available(value) and "available" or "unavailable" |
741 | 0ccb6461 | Christos Stavrakakis | value2 = dummy_pool.is_available(value) and "available"\ |
742 | 0ccb6461 | Christos Stavrakakis | or "unavailable" |
743 | 0ccb6461 | Christos Stavrakakis | logger.error(msg, pool, value, value1, value2) |
744 | 0ccb6461 | Christos Stavrakakis | if fix:
|
745 | 0ccb6461 | Christos Stavrakakis | pool.available = dummy_pool.available |
746 | 0ccb6461 | Christos Stavrakakis | pool.save() |
747 | 0ccb6461 | Christos Stavrakakis | logger.info("Fixed available map of pool '%s'", pool)
|
748 | 0ccb6461 | Christos Stavrakakis | |
749 | 0ccb6461 | Christos Stavrakakis | |
750 | 0ccb6461 | Christos Stavrakakis | def create_empty_pool(pool, pool_class): |
751 | 0ccb6461 | Christos Stavrakakis | pool_row = pool.pool_table |
752 | 0ccb6461 | Christos Stavrakakis | pool_row.available_map = ""
|
753 | 0ccb6461 | Christos Stavrakakis | pool_row.reserved_map = ""
|
754 | 0ccb6461 | Christos Stavrakakis | return pool_class(pool_row) |