root / snf-cyclades-app / synnefo / logic / reconciliation.py @ bfd04b01
History | View | Annotate | Download (30.4 kB)
1 | 9fea53cc | Vangelis Koukis | # -*- coding: utf-8 -*-
|
---|---|---|---|
2 | 9fea53cc | Vangelis Koukis | #
|
3 | 75dc539e | Christos Stavrakakis | # Copyright 2011-2013 GRNET S.A. All rights reserved.
|
4 | 9fea53cc | Vangelis Koukis | #
|
5 | 9fea53cc | Vangelis Koukis | # Redistribution and use in source and binary forms, with or
|
6 | 9fea53cc | Vangelis Koukis | # without modification, are permitted provided that the following
|
7 | 9fea53cc | Vangelis Koukis | # conditions are met:
|
8 | 9fea53cc | Vangelis Koukis | #
|
9 | 9fea53cc | Vangelis Koukis | # 1. Redistributions of source code must retain the above
|
10 | 9fea53cc | Vangelis Koukis | # copyright notice, this list of conditions and the following
|
11 | 9fea53cc | Vangelis Koukis | # disclaimer.
|
12 | 9fea53cc | Vangelis Koukis | #
|
13 | 9fea53cc | Vangelis Koukis | # 2. Redistributions in binary form must reproduce the above
|
14 | 9fea53cc | Vangelis Koukis | # copyright notice, this list of conditions and the following
|
15 | 9fea53cc | Vangelis Koukis | # disclaimer in the documentation and/or other materials
|
16 | 9fea53cc | Vangelis Koukis | # provided with the distribution.
|
17 | 9fea53cc | Vangelis Koukis | #
|
18 | 9fea53cc | Vangelis Koukis | # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
|
19 | 9fea53cc | Vangelis Koukis | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
20 | 9fea53cc | Vangelis Koukis | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
21 | 9fea53cc | Vangelis Koukis | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
|
22 | 9fea53cc | Vangelis Koukis | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
23 | 9fea53cc | Vangelis Koukis | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
24 | 9fea53cc | Vangelis Koukis | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
25 | 9fea53cc | Vangelis Koukis | # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
26 | 9fea53cc | Vangelis Koukis | # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
27 | 9fea53cc | Vangelis Koukis | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
28 | 9fea53cc | Vangelis Koukis | # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
29 | 9fea53cc | Vangelis Koukis | # POSSIBILITY OF SUCH DAMAGE.
|
30 | 9fea53cc | Vangelis Koukis | #
|
31 | 9fea53cc | Vangelis Koukis | # The views and conclusions contained in the software and
|
32 | 9fea53cc | Vangelis Koukis | # documentation are those of the authors and should not be
|
33 | 9fea53cc | Vangelis Koukis | # interpreted as representing official policies, either expressed
|
34 | 9fea53cc | Vangelis Koukis | # or implied, of GRNET S.A.
|
35 | 9fea53cc | Vangelis Koukis | #
|
36 | 9fea53cc | Vangelis Koukis | """Business logic for reconciliation
|
37 | 9fea53cc | Vangelis Koukis |
|
38 | 9fea53cc | Vangelis Koukis | Reconcile the contents of the DB with the actual state of the
|
39 | 9fea53cc | Vangelis Koukis | Ganeti backend.
|
40 | 9fea53cc | Vangelis Koukis |
|
41 | 9fea53cc | Vangelis Koukis | Let D be the set of VMs in the DB, G the set of VMs in Ganeti.
|
42 | 9fea53cc | Vangelis Koukis | RULES:
|
43 | 9fea53cc | Vangelis Koukis | R1. Stale servers in DB:
|
44 | 9fea53cc | Vangelis Koukis | For any v in D but not in G:
|
45 | 9fea53cc | Vangelis Koukis | Set deleted=True.
|
46 | 9fea53cc | Vangelis Koukis | R2. Orphan instances in Ganet:
|
47 | 9fea53cc | Vangelis Koukis | For any v in G with deleted=True in D:
|
48 | 9fea53cc | Vangelis Koukis | Issue OP_INSTANCE_DESTROY.
|
49 | 9fea53cc | Vangelis Koukis | R3. Unsynced operstate:
|
50 | 9fea53cc | Vangelis Koukis | For any v whose operating state differs between G and V:
|
51 | 9fea53cc | Vangelis Koukis | Set the operating state in D based on the state in G.
|
52 | 9fea53cc | Vangelis Koukis | In the code, D, G are Python dicts mapping instance ids to operating state.
|
53 | 9fea53cc | Vangelis Koukis | For D, the operating state is chosen from VirtualMachine.OPER_STATES.
|
54 | 9fea53cc | Vangelis Koukis | For G, the operating state is True if the machine is up, False otherwise.
|
55 | 9fea53cc | Vangelis Koukis |
|
56 | 9fea53cc | Vangelis Koukis | """
|
57 | 9fea53cc | Vangelis Koukis | |
58 | 9fea53cc | Vangelis Koukis | |
59 | 0c09b1c0 | Christos Stavrakakis | from django.conf import settings |
60 | 4161cb41 | Christos Stavrakakis | |
61 | 75dc539e | Christos Stavrakakis | import logging |
62 | 75dc539e | Christos Stavrakakis | import itertools |
63 | 89b2b908 | Christos Stavrakakis | import bitarray |
64 | d7ff7f5a | Christos Stavrakakis | from datetime import datetime, timedelta |
65 | 4161cb41 | Christos Stavrakakis | |
66 | 75dc539e | Christos Stavrakakis | from django.db import transaction |
67 | 75dc539e | Christos Stavrakakis | from synnefo.db.models import (Backend, VirtualMachine, Flavor, |
68 | 89b2b908 | Christos Stavrakakis | pooled_rapi_client, Network, |
69 | 0ccb6461 | Christos Stavrakakis | BackendNetwork, BridgePoolTable, |
70 | 0ccb6461 | Christos Stavrakakis | MacPrefixPoolTable) |
71 | 0ccb6461 | Christos Stavrakakis | from synnefo.db import pools |
72 | 75dc539e | Christos Stavrakakis | from synnefo.logic import utils, backend as backend_mod |
73 | 9fea53cc | Vangelis Koukis | |
74 | 75dc539e | Christos Stavrakakis | logger = logging.getLogger() |
75 | 75dc539e | Christos Stavrakakis | logging.basicConfig() |
76 | 9e98ba3c | Giorgos Verigakis | |
77 | e63050ca | Christos Stavrakakis | try:
|
78 | e63050ca | Christos Stavrakakis | CHECK_INTERVAL = settings.RECONCILIATION_CHECK_INTERVAL |
79 | e63050ca | Christos Stavrakakis | except AttributeError: |
80 | e63050ca | Christos Stavrakakis | CHECK_INTERVAL = 60
|
81 | e63050ca | Christos Stavrakakis | |
82 | 63f9eb8e | Christos Stavrakakis | GANETI_JOB_ERROR = "error"
|
83 | 63f9eb8e | Christos Stavrakakis | GANETI_JOBS_PENDING = ["queued", "waiting", "running", "canceling"] |
84 | 63f9eb8e | Christos Stavrakakis | GANETI_JOBS_FINALIZED = ["success", "error", "canceled"] |
85 | 63f9eb8e | Christos Stavrakakis | |
86 | e63050ca | Christos Stavrakakis | |
87 | 75dc539e | Christos Stavrakakis | class BackendReconciler(object): |
88 | 75dc539e | Christos Stavrakakis | def __init__(self, backend, logger, options=None): |
89 | 75dc539e | Christos Stavrakakis | self.backend = backend
|
90 | 75dc539e | Christos Stavrakakis | self.log = logger
|
91 | 75dc539e | Christos Stavrakakis | self.client = backend.get_client()
|
92 | 75dc539e | Christos Stavrakakis | if options is None: |
93 | 75dc539e | Christos Stavrakakis | self.options = {}
|
94 | 4161cb41 | Christos Stavrakakis | else:
|
95 | 75dc539e | Christos Stavrakakis | self.options = options
|
96 | 75dc539e | Christos Stavrakakis | |
97 | 75dc539e | Christos Stavrakakis | def close(self): |
98 | 75dc539e | Christos Stavrakakis | self.backend.put_client(self.client) |
99 | 75dc539e | Christos Stavrakakis | |
100 | 75dc539e | Christos Stavrakakis | @transaction.commit_on_success
|
101 | 75dc539e | Christos Stavrakakis | def reconcile(self): |
102 | 75dc539e | Christos Stavrakakis | log = self.log
|
103 | 75dc539e | Christos Stavrakakis | backend = self.backend
|
104 | 75dc539e | Christos Stavrakakis | log.debug("Reconciling backend %s", backend)
|
105 | 75dc539e | Christos Stavrakakis | |
106 | 75dc539e | Christos Stavrakakis | self.db_servers = get_database_servers(backend)
|
107 | 75dc539e | Christos Stavrakakis | self.db_servers_keys = set(self.db_servers.keys()) |
108 | 75dc539e | Christos Stavrakakis | log.debug("Got servers info from database.")
|
109 | 75dc539e | Christos Stavrakakis | |
110 | 75dc539e | Christos Stavrakakis | self.gnt_servers = get_ganeti_servers(backend)
|
111 | 75dc539e | Christos Stavrakakis | self.gnt_servers_keys = set(self.gnt_servers.keys()) |
112 | 75dc539e | Christos Stavrakakis | log.debug("Got servers info from Ganeti backend.")
|
113 | 75dc539e | Christos Stavrakakis | |
114 | 63f9eb8e | Christos Stavrakakis | self.gnt_jobs = get_ganeti_jobs(backend)
|
115 | 63f9eb8e | Christos Stavrakakis | log.debug("Got jobs from Ganeti backend")
|
116 | 63f9eb8e | Christos Stavrakakis | |
117 | 75dc539e | Christos Stavrakakis | self.event_time = datetime.now()
|
118 | 75dc539e | Christos Stavrakakis | |
119 | 75dc539e | Christos Stavrakakis | self.stale_servers = self.reconcile_stale_servers() |
120 | 75dc539e | Christos Stavrakakis | self.orphan_servers = self.reconcile_orphan_servers() |
121 | 75dc539e | Christos Stavrakakis | self.unsynced_servers = self.reconcile_unsynced_servers() |
122 | 75dc539e | Christos Stavrakakis | self.close()
|
123 | 75dc539e | Christos Stavrakakis | |
124 | 75dc539e | Christos Stavrakakis | def get_build_status(self, db_server): |
125 | 63f9eb8e | Christos Stavrakakis | job_id = db_server.backendjobid |
126 | 63f9eb8e | Christos Stavrakakis | if job_id in self.gnt_jobs: |
127 | 63f9eb8e | Christos Stavrakakis | gnt_job_status = self.gnt_jobs[job_id]["status"] |
128 | 63f9eb8e | Christos Stavrakakis | if gnt_job_status == GANETI_JOB_ERROR:
|
129 | 63f9eb8e | Christos Stavrakakis | return "ERROR" |
130 | 63f9eb8e | Christos Stavrakakis | elif gnt_job_status not in GANETI_JOBS_FINALIZED: |
131 | 75dc539e | Christos Stavrakakis | return "RUNNING" |
132 | 75dc539e | Christos Stavrakakis | else:
|
133 | 63f9eb8e | Christos Stavrakakis | return "FINALIZED" |
134 | 75dc539e | Christos Stavrakakis | else:
|
135 | 63f9eb8e | Christos Stavrakakis | return "ERROR" |
136 | 75dc539e | Christos Stavrakakis | |
137 | 75dc539e | Christos Stavrakakis | def reconcile_stale_servers(self): |
138 | 75dc539e | Christos Stavrakakis | # Detect stale servers
|
139 | 75dc539e | Christos Stavrakakis | stale = [] |
140 | 75dc539e | Christos Stavrakakis | stale_keys = self.db_servers_keys - self.gnt_servers_keys |
141 | 75dc539e | Christos Stavrakakis | for server_id in stale_keys: |
142 | 75dc539e | Christos Stavrakakis | db_server = self.db_servers[server_id]
|
143 | 75dc539e | Christos Stavrakakis | if db_server.operstate == "BUILD": |
144 | 75dc539e | Christos Stavrakakis | build_status = self.get_build_status(db_server)
|
145 | 75dc539e | Christos Stavrakakis | if build_status == "ERROR": |
146 | 75dc539e | Christos Stavrakakis | # Special handling of BUILD eerrors
|
147 | 75dc539e | Christos Stavrakakis | self.reconcile_building_server(db_server)
|
148 | 75dc539e | Christos Stavrakakis | elif build_status != "RUNNING": |
149 | 75dc539e | Christos Stavrakakis | stale.append(server_id) |
150 | 1464a17b | Christos Stavrakakis | elif (db_server.operstate == "ERROR" and |
151 | 1464a17b | Christos Stavrakakis | db_server.action != "DESTROY"):
|
152 | 1464a17b | Christos Stavrakakis | # Servers at building ERROR are stale only if the user has
|
153 | 1464a17b | Christos Stavrakakis | # asked to destroy them.
|
154 | 1464a17b | Christos Stavrakakis | pass
|
155 | 75dc539e | Christos Stavrakakis | else:
|
156 | 75dc539e | Christos Stavrakakis | stale.append(server_id) |
157 | 75dc539e | Christos Stavrakakis | |
158 | 75dc539e | Christos Stavrakakis | # Report them
|
159 | 75dc539e | Christos Stavrakakis | if stale:
|
160 | 75dc539e | Christos Stavrakakis | self.log.info("Found stale servers %s at backend %s", |
161 | 75dc539e | Christos Stavrakakis | ", ".join(map(str, stale)), self.backend) |
162 | 75dc539e | Christos Stavrakakis | else:
|
163 | 75dc539e | Christos Stavrakakis | self.log.debug("No stale servers at backend %s", self.backend) |
164 | 75dc539e | Christos Stavrakakis | |
165 | 75dc539e | Christos Stavrakakis | # Fix them
|
166 | 75dc539e | Christos Stavrakakis | if stale and self.options["fix_stale"]: |
167 | 75dc539e | Christos Stavrakakis | for server_id in stale: |
168 | 75dc539e | Christos Stavrakakis | db_server = self.db_servers[server_id]
|
169 | 75dc539e | Christos Stavrakakis | backend_mod.process_op_status( |
170 | 75dc539e | Christos Stavrakakis | vm=db_server, |
171 | 75dc539e | Christos Stavrakakis | etime=self.event_time,
|
172 | 75dc539e | Christos Stavrakakis | jobid=-0,
|
173 | 75dc539e | Christos Stavrakakis | opcode='OP_INSTANCE_REMOVE', status='success', |
174 | 75dc539e | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
175 | 75dc539e | Christos Stavrakakis | self.log.debug("Simulated Ganeti removal for stale servers.") |
176 | 75dc539e | Christos Stavrakakis | |
177 | 75dc539e | Christos Stavrakakis | def reconcile_orphan_servers(self): |
178 | 75dc539e | Christos Stavrakakis | orphans = self.gnt_servers_keys - self.db_servers_keys |
179 | 75dc539e | Christos Stavrakakis | if orphans:
|
180 | 75dc539e | Christos Stavrakakis | self.log.info("Found orphan servers %s at backend %s", |
181 | 75dc539e | Christos Stavrakakis | ", ".join(map(str, orphans)), self.backend) |
182 | 75dc539e | Christos Stavrakakis | else:
|
183 | 75dc539e | Christos Stavrakakis | self.log.debug("No orphan servers at backend %s", self.backend) |
184 | 75dc539e | Christos Stavrakakis | |
185 | 75dc539e | Christos Stavrakakis | if orphans and self.options["fix_orphans"]: |
186 | 75dc539e | Christos Stavrakakis | for server_id in orphans: |
187 | 75dc539e | Christos Stavrakakis | server_name = utils.id_to_instance_name(server_id) |
188 | 75dc539e | Christos Stavrakakis | self.client.DeleteInstance(server_name)
|
189 | 75dc539e | Christos Stavrakakis | self.log.debug("Issued OP_INSTANCE_REMOVE for orphan servers.") |
190 | 75dc539e | Christos Stavrakakis | |
191 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_servers(self): |
192 | 75dc539e | Christos Stavrakakis | #log = self.log
|
193 | 75dc539e | Christos Stavrakakis | for server_id in self.db_servers_keys & self.gnt_servers_keys: |
194 | 75dc539e | Christos Stavrakakis | db_server = self.db_servers[server_id]
|
195 | 75dc539e | Christos Stavrakakis | gnt_server = self.gnt_servers[server_id]
|
196 | 75dc539e | Christos Stavrakakis | if db_server.operstate == "BUILD": |
197 | 75dc539e | Christos Stavrakakis | build_status = self.get_build_status(db_server)
|
198 | 75dc539e | Christos Stavrakakis | if build_status == "RUNNING": |
199 | 75dc539e | Christos Stavrakakis | # Do not reconcile building VMs
|
200 | 75dc539e | Christos Stavrakakis | continue
|
201 | 75dc539e | Christos Stavrakakis | elif build_status == "ERROR": |
202 | 75dc539e | Christos Stavrakakis | # Special handling of build errors
|
203 | 75dc539e | Christos Stavrakakis | self.reconcile_building_server(db_server)
|
204 | 75dc539e | Christos Stavrakakis | continue
|
205 | 75dc539e | Christos Stavrakakis | |
206 | 75dc539e | Christos Stavrakakis | self.reconcile_unsynced_operstate(server_id, db_server,
|
207 | 75dc539e | Christos Stavrakakis | gnt_server) |
208 | 75dc539e | Christos Stavrakakis | self.reconcile_unsynced_flavor(server_id, db_server,
|
209 | 75dc539e | Christos Stavrakakis | gnt_server) |
210 | 75dc539e | Christos Stavrakakis | self.reconcile_unsynced_nics(server_id, db_server, gnt_server)
|
211 | 75dc539e | Christos Stavrakakis | self.reconcile_unsynced_disks(server_id, db_server, gnt_server)
|
212 | 63f9eb8e | Christos Stavrakakis | if db_server.task is not None: |
213 | 63f9eb8e | Christos Stavrakakis | self.reconcile_pending_task(server_id, db_server)
|
214 | 75dc539e | Christos Stavrakakis | |
215 | 75dc539e | Christos Stavrakakis | def reconcile_building_server(self, db_server): |
216 | 75dc539e | Christos Stavrakakis | self.log.info("Server '%s' is BUILD in DB, but 'ERROR' in Ganeti.", |
217 | 75dc539e | Christos Stavrakakis | db_server.id) |
218 | 75dc539e | Christos Stavrakakis | if self.options["fix_unsynced"]: |
219 | 75dc539e | Christos Stavrakakis | fix_opcode = "OP_INSTANCE_CREATE"
|
220 | 75dc539e | Christos Stavrakakis | backend_mod.process_op_status( |
221 | 75dc539e | Christos Stavrakakis | vm=db_server, |
222 | 75dc539e | Christos Stavrakakis | etime=self.event_time,
|
223 | 75dc539e | Christos Stavrakakis | jobid=-0,
|
224 | 75dc539e | Christos Stavrakakis | opcode=fix_opcode, status='error',
|
225 | 75dc539e | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
226 | 75dc539e | Christos Stavrakakis | self.log.debug("Simulated Ganeti error build event for" |
227 | 75dc539e | Christos Stavrakakis | " server '%s'", db_server.id)
|
228 | 75dc539e | Christos Stavrakakis | |
229 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_operstate(self, server_id, db_server, gnt_server): |
230 | 75dc539e | Christos Stavrakakis | if db_server.operstate != gnt_server["state"]: |
231 | 75dc539e | Christos Stavrakakis | self.log.info("Server '%s' is '%s' in DB and '%s' in Ganeti.", |
232 | 75dc539e | Christos Stavrakakis | server_id, db_server.operstate, gnt_server["state"])
|
233 | 75dc539e | Christos Stavrakakis | if self.options["fix_unsynced"]: |
234 | 5d3e597a | Christos Stavrakakis | # If server is in building state, you will have first to
|
235 | 5d3e597a | Christos Stavrakakis | # reconcile it's creation, to avoid wrong quotas
|
236 | 5d3e597a | Christos Stavrakakis | if db_server.operstate == "BUILD": |
237 | 5d3e597a | Christos Stavrakakis | backend_mod.process_op_status( |
238 | 5d3e597a | Christos Stavrakakis | vm=db_server, etime=self.event_time, jobid=-0, |
239 | 5d3e597a | Christos Stavrakakis | opcode="OP_INSTANCE_CREATE", status='success', |
240 | 5d3e597a | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
241 | 5d3e597a | Christos Stavrakakis | fix_opcode = "OP_INSTANCE_STARTUP"\
|
242 | 5d3e597a | Christos Stavrakakis | if gnt_server["state"] == "STARTED"\ |
243 | 75dc539e | Christos Stavrakakis | else "OP_INSTANCE_SHUTDOWN" |
244 | 75dc539e | Christos Stavrakakis | backend_mod.process_op_status( |
245 | 5d3e597a | Christos Stavrakakis | vm=db_server, etime=self.event_time, jobid=-0, |
246 | 75dc539e | Christos Stavrakakis | opcode=fix_opcode, status='success',
|
247 | 75dc539e | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
248 | 75dc539e | Christos Stavrakakis | self.log.debug("Simulated Ganeti state event for server '%s'", |
249 | 75dc539e | Christos Stavrakakis | server_id) |
250 | 75dc539e | Christos Stavrakakis | |
251 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_flavor(self, server_id, db_server, gnt_server): |
252 | 75dc539e | Christos Stavrakakis | db_flavor = db_server.flavor |
253 | 75dc539e | Christos Stavrakakis | gnt_flavor = gnt_server["flavor"]
|
254 | 75dc539e | Christos Stavrakakis | if (db_flavor.ram != gnt_flavor["ram"] or |
255 | 5d3e597a | Christos Stavrakakis | db_flavor.cpu != gnt_flavor["vcpus"]):
|
256 | a67419d8 | Christos Stavrakakis | try:
|
257 | a67419d8 | Christos Stavrakakis | gnt_flavor = Flavor.objects.get( |
258 | 75dc539e | Christos Stavrakakis | ram=gnt_flavor["ram"],
|
259 | 75dc539e | Christos Stavrakakis | cpu=gnt_flavor["vcpus"],
|
260 | 75dc539e | Christos Stavrakakis | disk=db_flavor.disk, |
261 | 75dc539e | Christos Stavrakakis | disk_template=db_flavor.disk_template) |
262 | a67419d8 | Christos Stavrakakis | except Flavor.DoesNotExist:
|
263 | 75dc539e | Christos Stavrakakis | self.log.warning("Server '%s' has unknown flavor.", server_id) |
264 | 75dc539e | Christos Stavrakakis | return
|
265 | 75dc539e | Christos Stavrakakis | |
266 | 4111c53e | Christos Stavrakakis | self.log.info("Server '%s' has flavor '%s' in DB and '%s' in" |
267 | 75dc539e | Christos Stavrakakis | " Ganeti", server_id, db_flavor, gnt_flavor)
|
268 | 75dc539e | Christos Stavrakakis | if self.options["fix_unsynced_flavors"]: |
269 | 75dc539e | Christos Stavrakakis | old_state = db_server.operstate |
270 | 75dc539e | Christos Stavrakakis | opcode = "OP_INSTANCE_SET_PARAMS"
|
271 | 75dc539e | Christos Stavrakakis | beparams = {"vcpus": gnt_flavor.cpu,
|
272 | 75dc539e | Christos Stavrakakis | "minmem": gnt_flavor.ram,
|
273 | 75dc539e | Christos Stavrakakis | "maxmem": gnt_flavor.ram}
|
274 | 75dc539e | Christos Stavrakakis | backend_mod.process_op_status( |
275 | 75dc539e | Christos Stavrakakis | vm=db_server, etime=self.event_time, jobid=-0, |
276 | 75dc539e | Christos Stavrakakis | opcode=opcode, status='success',
|
277 | e6fbada1 | Christos Stavrakakis | job_fields={"beparams": beparams},
|
278 | 75dc539e | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
279 | 75dc539e | Christos Stavrakakis | # process_op_status with beparams will set the vmstate to
|
280 | 75dc539e | Christos Stavrakakis | # shutdown. Fix this be returning it to old state
|
281 | 75dc539e | Christos Stavrakakis | vm = VirtualMachine.objects.get(pk=server_id) |
282 | 75dc539e | Christos Stavrakakis | vm.operstate = old_state |
283 | 75dc539e | Christos Stavrakakis | vm.save() |
284 | 75dc539e | Christos Stavrakakis | self.log.debug("Simulated Ganeti flavor event for server '%s'", |
285 | 75dc539e | Christos Stavrakakis | server_id) |
286 | 75dc539e | Christos Stavrakakis | |
287 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_nics(self, server_id, db_server, gnt_server): |
288 | d7ff7f5a | Christos Stavrakakis | building_time = (self.event_time -
|
289 | d7ff7f5a | Christos Stavrakakis | timedelta(seconds=backend_mod.BUILDING_NIC_TIMEOUT)) |
290 | d7ff7f5a | Christos Stavrakakis | db_nics = db_server.nics.exclude(state="BUILDING",
|
291 | d7ff7f5a | Christos Stavrakakis | created__lte=building_time) \ |
292 | d7ff7f5a | Christos Stavrakakis | .order_by("index")
|
293 | 75dc539e | Christos Stavrakakis | gnt_nics = gnt_server["nics"]
|
294 | 75dc539e | Christos Stavrakakis | gnt_nics_parsed = backend_mod.process_ganeti_nics(gnt_nics) |
295 | a1baa42b | Christos Stavrakakis | nics_changed = len(db_nics) != len(gnt_nics) |
296 | a1baa42b | Christos Stavrakakis | for db_nic, gnt_nic in zip(db_nics, sorted(gnt_nics_parsed.items())): |
297 | a1baa42b | Christos Stavrakakis | gnt_nic_id, gnt_nic = gnt_nic |
298 | a1baa42b | Christos Stavrakakis | if (db_nic.id == gnt_nic_id) and\ |
299 | a1baa42b | Christos Stavrakakis | backend_mod.nics_are_equal(db_nic, gnt_nic): |
300 | a1baa42b | Christos Stavrakakis | continue
|
301 | a1baa42b | Christos Stavrakakis | else:
|
302 | a1baa42b | Christos Stavrakakis | nics_changed = True
|
303 | a1baa42b | Christos Stavrakakis | break
|
304 | a1baa42b | Christos Stavrakakis | if nics_changed:
|
305 | a1baa42b | Christos Stavrakakis | msg = "Found unsynced NICs for server '%s'.\n"\
|
306 | a1baa42b | Christos Stavrakakis | "\tDB:\n\t\t%s\n\tGaneti:\n\t\t%s"
|
307 | a1baa42b | Christos Stavrakakis | db_nics_str = "\n\t\t".join(map(format_db_nic, db_nics)) |
308 | a1baa42b | Christos Stavrakakis | gnt_nics_str = "\n\t\t".join(map(format_gnt_nic, |
309 | a1baa42b | Christos Stavrakakis | gnt_nics_parsed.items())) |
310 | 75dc539e | Christos Stavrakakis | self.log.info(msg, server_id, db_nics_str, gnt_nics_str)
|
311 | 75dc539e | Christos Stavrakakis | if self.options["fix_unsynced_nics"]: |
312 | 75dc539e | Christos Stavrakakis | backend_mod.process_net_status(vm=db_server, |
313 | 75dc539e | Christos Stavrakakis | etime=self.event_time,
|
314 | 75dc539e | Christos Stavrakakis | nics=gnt_nics) |
315 | 75dc539e | Christos Stavrakakis | |
316 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_disks(self, server_id, db_server, gnt_server): |
317 | 75dc539e | Christos Stavrakakis | pass
|
318 | 75dc539e | Christos Stavrakakis | |
319 | 63f9eb8e | Christos Stavrakakis | def reconcile_pending_task(self, server_id, db_server): |
320 | 63f9eb8e | Christos Stavrakakis | job_id = db_server.task_job_id |
321 | 63f9eb8e | Christos Stavrakakis | pending_task = False
|
322 | 63f9eb8e | Christos Stavrakakis | if job_id not in self.gnt_jobs: |
323 | 63f9eb8e | Christos Stavrakakis | pending_task = True
|
324 | 63f9eb8e | Christos Stavrakakis | else:
|
325 | ce55f724 | Christos Stavrakakis | gnt_job_status = self.gnt_jobs[job_id]["status"] |
326 | 63f9eb8e | Christos Stavrakakis | if gnt_job_status in GANETI_JOBS_FINALIZED: |
327 | 63f9eb8e | Christos Stavrakakis | pending_task = True
|
328 | 63f9eb8e | Christos Stavrakakis | |
329 | 63f9eb8e | Christos Stavrakakis | if pending_task:
|
330 | 63f9eb8e | Christos Stavrakakis | self.log.info("Found server '%s' with pending task: '%s'", |
331 | 63f9eb8e | Christos Stavrakakis | server_id, db_server.task) |
332 | 160c81a1 | Christos Stavrakakis | if self.options["fix_pending_tasks"]: |
333 | 63f9eb8e | Christos Stavrakakis | db_server.task = None
|
334 | 63f9eb8e | Christos Stavrakakis | db_server.task_job_id = None
|
335 | 63f9eb8e | Christos Stavrakakis | db_server.save() |
336 | 63f9eb8e | Christos Stavrakakis | self.log.info("Cleared pending task for server '%s", server_id) |
337 | 63f9eb8e | Christos Stavrakakis | |
338 | 75dc539e | Christos Stavrakakis | |
339 | bfd04b01 | Christos Stavrakakis | NIC_MSG = ": %s\t".join(["ID", "State", "IP", "Network", "MAC", "Index", |
340 | bfd04b01 | Christos Stavrakakis | "Firewall"]) + ": %s" |
341 | a1baa42b | Christos Stavrakakis | |
342 | a1baa42b | Christos Stavrakakis | |
343 | 75dc539e | Christos Stavrakakis | def format_db_nic(nic): |
344 | a1baa42b | Christos Stavrakakis | return NIC_MSG % (nic.id, nic.state, nic.ipv4, nic.network_id, nic.mac,
|
345 | bfd04b01 | Christos Stavrakakis | nic.index, nic.firewall_profile) |
346 | 75dc539e | Christos Stavrakakis | |
347 | 75dc539e | Christos Stavrakakis | |
348 | 75dc539e | Christos Stavrakakis | def format_gnt_nic(nic): |
349 | a1baa42b | Christos Stavrakakis | nic_name, nic = nic |
350 | a1baa42b | Christos Stavrakakis | return NIC_MSG % (nic_name, nic["state"], nic["ipv4"], nic["network"], |
351 | bfd04b01 | Christos Stavrakakis | nic["mac"], nic["index"], nic["firewall_profile"]) |
352 | 9fea53cc | Vangelis Koukis | |
353 | cc92b70f | Christos Stavrakakis | |
354 | 0e9a423f | Christos Stavrakakis | #
|
355 | 0e9a423f | Christos Stavrakakis | # Networks
|
356 | 0e9a423f | Christos Stavrakakis | #
|
357 | 3524241a | Christos Stavrakakis | |
358 | 3524241a | Christos Stavrakakis | |
359 | 0e9a423f | Christos Stavrakakis | def get_networks_from_ganeti(backend): |
360 | 44e2c577 | Christos Stavrakakis | prefix = settings.BACKEND_PREFIX_ID + 'net-'
|
361 | 0e9a423f | Christos Stavrakakis | |
362 | 0e9a423f | Christos Stavrakakis | networks = {} |
363 | 3524241a | Christos Stavrakakis | with pooled_rapi_client(backend) as c: |
364 | 3524241a | Christos Stavrakakis | for net in c.GetNetworks(bulk=True): |
365 | 3524241a | Christos Stavrakakis | if net['name'].startswith(prefix): |
366 | 3524241a | Christos Stavrakakis | id = utils.id_from_network_name(net['name'])
|
367 | 3524241a | Christos Stavrakakis | networks[id] = net
|
368 | 0e9a423f | Christos Stavrakakis | |
369 | 0e9a423f | Christos Stavrakakis | return networks
|
370 | 0e9a423f | Christos Stavrakakis | |
371 | 0e9a423f | Christos Stavrakakis | |
372 | 0e9a423f | Christos Stavrakakis | def hanging_networks(backend, GNets): |
373 | 0e9a423f | Christos Stavrakakis | """Get networks that are not connected to all Nodegroups.
|
374 | 0e9a423f | Christos Stavrakakis |
|
375 | 0e9a423f | Christos Stavrakakis | """
|
376 | 0e9a423f | Christos Stavrakakis | def get_network_groups(group_list): |
377 | 0e9a423f | Christos Stavrakakis | groups = set()
|
378 | a9bb2a1a | Christos Stavrakakis | for (name, mode, link) in group_list: |
379 | a9bb2a1a | Christos Stavrakakis | groups.add(name) |
380 | 0e9a423f | Christos Stavrakakis | return groups
|
381 | 0e9a423f | Christos Stavrakakis | |
382 | 3524241a | Christos Stavrakakis | with pooled_rapi_client(backend) as c: |
383 | 3524241a | Christos Stavrakakis | groups = set(c.GetGroups())
|
384 | 0e9a423f | Christos Stavrakakis | |
385 | 0e9a423f | Christos Stavrakakis | hanging = {} |
386 | 0e9a423f | Christos Stavrakakis | for id, info in GNets.items(): |
387 | 0e9a423f | Christos Stavrakakis | group_list = get_network_groups(info['group_list'])
|
388 | 0e9a423f | Christos Stavrakakis | if group_list != groups:
|
389 | 0e9a423f | Christos Stavrakakis | hanging[id] = groups - group_list
|
390 | 0e9a423f | Christos Stavrakakis | return hanging
|
391 | 0e9a423f | Christos Stavrakakis | |
392 | 9fea53cc | Vangelis Koukis | |
393 | 75dc539e | Christos Stavrakakis | def get_online_backends(): |
394 | 75dc539e | Christos Stavrakakis | return Backend.objects.filter(offline=False) |
395 | 75dc539e | Christos Stavrakakis | |
396 | 75dc539e | Christos Stavrakakis | |
397 | 75dc539e | Christos Stavrakakis | def get_database_servers(backend): |
398 | 75dc539e | Christos Stavrakakis | servers = backend.virtual_machines.select_related("nics", "flavor")\ |
399 | 75dc539e | Christos Stavrakakis | .filter(deleted=False)
|
400 | 75dc539e | Christos Stavrakakis | return dict([(s.id, s) for s in servers]) |
401 | 75dc539e | Christos Stavrakakis | |
402 | 75dc539e | Christos Stavrakakis | |
403 | 75dc539e | Christos Stavrakakis | def get_ganeti_servers(backend): |
404 | 75dc539e | Christos Stavrakakis | gnt_instances = backend_mod.get_instances(backend) |
405 | 75dc539e | Christos Stavrakakis | # Filter out non-synnefo instances
|
406 | 75dc539e | Christos Stavrakakis | snf_backend_prefix = settings.BACKEND_PREFIX_ID |
407 | 75dc539e | Christos Stavrakakis | gnt_instances = filter(lambda i: i["name"].startswith(snf_backend_prefix), |
408 | 75dc539e | Christos Stavrakakis | gnt_instances) |
409 | 75dc539e | Christos Stavrakakis | gnt_instances = map(parse_gnt_instance, gnt_instances)
|
410 | 75dc539e | Christos Stavrakakis | return dict([(i["id"], i) for i in gnt_instances if i["id"] is not None]) |
411 | 75dc539e | Christos Stavrakakis | |
412 | 75dc539e | Christos Stavrakakis | |
413 | 75dc539e | Christos Stavrakakis | def parse_gnt_instance(instance): |
414 | 75dc539e | Christos Stavrakakis | try:
|
415 | 75dc539e | Christos Stavrakakis | instance_id = utils.id_from_instance_name(instance['name'])
|
416 | 75dc539e | Christos Stavrakakis | except Exception: |
417 | 75dc539e | Christos Stavrakakis | logger.error("Ignoring instance with malformed name %s",
|
418 | 75dc539e | Christos Stavrakakis | instance['name'])
|
419 | 75dc539e | Christos Stavrakakis | return (None, None) |
420 | 75dc539e | Christos Stavrakakis | |
421 | 75dc539e | Christos Stavrakakis | beparams = instance["beparams"]
|
422 | 75dc539e | Christos Stavrakakis | |
423 | 75dc539e | Christos Stavrakakis | vcpus = beparams["vcpus"]
|
424 | 75dc539e | Christos Stavrakakis | ram = beparams["maxmem"]
|
425 | 75dc539e | Christos Stavrakakis | state = instance["oper_state"] and "STARTED" or "STOPPED" |
426 | 75dc539e | Christos Stavrakakis | |
427 | 75dc539e | Christos Stavrakakis | return {
|
428 | 75dc539e | Christos Stavrakakis | "id": instance_id,
|
429 | 75dc539e | Christos Stavrakakis | "state": state, # FIX |
430 | 75dc539e | Christos Stavrakakis | "updated": datetime.fromtimestamp(instance["mtime"]), |
431 | 75dc539e | Christos Stavrakakis | "disks": disks_from_instance(instance),
|
432 | 75dc539e | Christos Stavrakakis | "nics": nics_from_instance(instance),
|
433 | 75dc539e | Christos Stavrakakis | "flavor": {"vcpus": vcpus, |
434 | 75dc539e | Christos Stavrakakis | "ram": ram},
|
435 | 75dc539e | Christos Stavrakakis | "tags": instance["tags"] |
436 | 75dc539e | Christos Stavrakakis | } |
437 | 75dc539e | Christos Stavrakakis | |
438 | 75dc539e | Christos Stavrakakis | |
439 | 75dc539e | Christos Stavrakakis | def nics_from_instance(i): |
440 | 75dc539e | Christos Stavrakakis | ips = zip(itertools.repeat('ip'), i['nic.ips']) |
441 | a1baa42b | Christos Stavrakakis | names = zip(itertools.repeat('name'), i['nic.names']) |
442 | 75dc539e | Christos Stavrakakis | macs = zip(itertools.repeat('mac'), i['nic.macs']) |
443 | a1baa42b | Christos Stavrakakis | networks = zip(itertools.repeat('network'), i['nic.networks.names']) |
444 | 75dc539e | Christos Stavrakakis | # modes = zip(itertools.repeat('mode'), i['nic.modes'])
|
445 | 75dc539e | Christos Stavrakakis | # links = zip(itertools.repeat('link'), i['nic.links'])
|
446 | 75dc539e | Christos Stavrakakis | # nics = zip(ips,macs,modes,networks,links)
|
447 | a1baa42b | Christos Stavrakakis | nics = zip(ips, names, macs, networks)
|
448 | 75dc539e | Christos Stavrakakis | nics = map(lambda x: dict(x), nics) |
449 | 75dc539e | Christos Stavrakakis | #nics = dict(enumerate(nics))
|
450 | 75dc539e | Christos Stavrakakis | tags = i["tags"]
|
451 | 75dc539e | Christos Stavrakakis | for tag in tags: |
452 | 75dc539e | Christos Stavrakakis | t = tag.split(":")
|
453 | 75dc539e | Christos Stavrakakis | if t[0:2] == ["synnefo", "network"]: |
454 | 75dc539e | Christos Stavrakakis | if len(t) != 4: |
455 | 75dc539e | Christos Stavrakakis | logger.error("Malformed synefo tag %s", tag)
|
456 | 75dc539e | Christos Stavrakakis | continue
|
457 | 75dc539e | Christos Stavrakakis | try:
|
458 | 75dc539e | Christos Stavrakakis | index = int(t[2]) |
459 | 75dc539e | Christos Stavrakakis | nics[index]['firewall'] = t[3] |
460 | 75dc539e | Christos Stavrakakis | except ValueError: |
461 | 75dc539e | Christos Stavrakakis | logger.error("Malformed synnefo tag %s", tag)
|
462 | 75dc539e | Christos Stavrakakis | except IndexError: |
463 | 75dc539e | Christos Stavrakakis | logger.error("Found tag %s for non-existent NIC %d",
|
464 | 75dc539e | Christos Stavrakakis | tag, index) |
465 | 75dc539e | Christos Stavrakakis | return nics
|
466 | 9fea53cc | Vangelis Koukis | |
467 | 9fea53cc | Vangelis Koukis | |
468 | 63f9eb8e | Christos Stavrakakis | def get_ganeti_jobs(backend): |
469 | 63f9eb8e | Christos Stavrakakis | gnt_jobs = backend_mod.get_jobs(backend) |
470 | 63f9eb8e | Christos Stavrakakis | return dict([(int(j["id"]), j) for j in gnt_jobs]) |
471 | 63f9eb8e | Christos Stavrakakis | |
472 | 63f9eb8e | Christos Stavrakakis | |
473 | 75dc539e | Christos Stavrakakis | def disks_from_instance(i): |
474 | 75dc539e | Christos Stavrakakis | return dict([(index, {"size": size}) |
475 | 75dc539e | Christos Stavrakakis | for index, size in enumerate(i["disk.sizes"])]) |
476 | 89b2b908 | Christos Stavrakakis | |
477 | 89b2b908 | Christos Stavrakakis | |
478 | 89b2b908 | Christos Stavrakakis | class NetworkReconciler(object): |
479 | 0ccb6461 | Christos Stavrakakis | def __init__(self, logger, fix=False): |
480 | 89b2b908 | Christos Stavrakakis | self.log = logger
|
481 | 89b2b908 | Christos Stavrakakis | self.fix = fix
|
482 | 89b2b908 | Christos Stavrakakis | |
483 | 89b2b908 | Christos Stavrakakis | @transaction.commit_on_success
|
484 | 89b2b908 | Christos Stavrakakis | def reconcile_networks(self): |
485 | 89b2b908 | Christos Stavrakakis | # Get models from DB
|
486 | 0ccb6461 | Christos Stavrakakis | self.backends = Backend.objects.exclude(offline=True) |
487 | 0ccb6461 | Christos Stavrakakis | self.networks = Network.objects.filter(deleted=False) |
488 | 89b2b908 | Christos Stavrakakis | |
489 | 89b2b908 | Christos Stavrakakis | self.event_time = datetime.now()
|
490 | 89b2b908 | Christos Stavrakakis | |
491 | 89b2b908 | Christos Stavrakakis | # Get info from all ganeti backends
|
492 | 0ccb6461 | Christos Stavrakakis | self.ganeti_networks = {}
|
493 | 0ccb6461 | Christos Stavrakakis | self.ganeti_hanging_networks = {}
|
494 | 0ccb6461 | Christos Stavrakakis | for b in self.backends: |
495 | 89b2b908 | Christos Stavrakakis | g_nets = get_networks_from_ganeti(b) |
496 | 0ccb6461 | Christos Stavrakakis | self.ganeti_networks[b] = g_nets
|
497 | 89b2b908 | Christos Stavrakakis | g_hanging_nets = hanging_networks(b, g_nets) |
498 | 0ccb6461 | Christos Stavrakakis | self.ganeti_hanging_networks[b] = g_hanging_nets
|
499 | 89b2b908 | Christos Stavrakakis | |
500 | 0ccb6461 | Christos Stavrakakis | self._reconcile_orphan_networks()
|
501 | 89b2b908 | Christos Stavrakakis | |
502 | 0ccb6461 | Christos Stavrakakis | for network in self.networks: |
503 | 0ccb6461 | Christos Stavrakakis | self._reconcile_network(network)
|
504 | 89b2b908 | Christos Stavrakakis | |
505 | 0ccb6461 | Christos Stavrakakis | @transaction.commit_on_success
|
506 | 0ccb6461 | Christos Stavrakakis | def _reconcile_network(self, network): |
507 | 0ccb6461 | Christos Stavrakakis | """Reconcile a network with corresponging Ganeti networks.
|
508 | 0ccb6461 | Christos Stavrakakis |
|
509 | 0ccb6461 | Christos Stavrakakis | Reconcile a Network and the associated BackendNetworks with the
|
510 | 0ccb6461 | Christos Stavrakakis | corresponding Ganeti networks in all Ganeti backends.
|
511 | 0ccb6461 | Christos Stavrakakis |
|
512 | 0ccb6461 | Christos Stavrakakis | """
|
513 | 0ccb6461 | Christos Stavrakakis | network_ip_pool = network.get_pool() # X-Lock on IP Pool
|
514 | 0ccb6461 | Christos Stavrakakis | for bend in self.backends: |
515 | 0ccb6461 | Christos Stavrakakis | bnet = get_backend_network(network, bend) |
516 | 0ccb6461 | Christos Stavrakakis | gnet = self.ganeti_networks[bend].get(network.id)
|
517 | 0ccb6461 | Christos Stavrakakis | if not bnet: |
518 | 0ccb6461 | Christos Stavrakakis | if network.floating_ip_pool:
|
519 | 0ccb6461 | Christos Stavrakakis | # Network is a floating IP pool and does not exist in
|
520 | 0ccb6461 | Christos Stavrakakis | # backend. We need to create it
|
521 | 0ccb6461 | Christos Stavrakakis | bnet = self.reconcile_parted_network(network, bend)
|
522 | 0ccb6461 | Christos Stavrakakis | elif not gnet: |
523 | 0ccb6461 | Christos Stavrakakis | # Network does not exist either in Ganeti nor in BD.
|
524 | 0ccb6461 | Christos Stavrakakis | continue
|
525 | 0ccb6461 | Christos Stavrakakis | else:
|
526 | 0ccb6461 | Christos Stavrakakis | # Network exists in Ganeti and not in DB.
|
527 | 0ccb6461 | Christos Stavrakakis | if network.action != "DESTROY" and not network.public: |
528 | 0ccb6461 | Christos Stavrakakis | bnet = self.reconcile_parted_network(network, bend)
|
529 | 0ccb6461 | Christos Stavrakakis | else:
|
530 | 0ccb6461 | Christos Stavrakakis | continue
|
531 | 89b2b908 | Christos Stavrakakis | |
532 | 0ccb6461 | Christos Stavrakakis | if not gnet: |
533 | 0ccb6461 | Christos Stavrakakis | # Network does not exist in Ganeti. If the network action
|
534 | 0ccb6461 | Christos Stavrakakis | # is DESTROY, we have to mark as deleted in DB, else we
|
535 | 0ccb6461 | Christos Stavrakakis | # have to create it in Ganeti.
|
536 | 0ccb6461 | Christos Stavrakakis | if network.action == "DESTROY": |
537 | 0ccb6461 | Christos Stavrakakis | if bnet.operstate != "DELETED": |
538 | 0ccb6461 | Christos Stavrakakis | self.reconcile_stale_network(bnet)
|
539 | 0ccb6461 | Christos Stavrakakis | else:
|
540 | 0ccb6461 | Christos Stavrakakis | self.reconcile_missing_network(network, bend)
|
541 | 0ccb6461 | Christos Stavrakakis | # Skip rest reconciliation!
|
542 | 0ccb6461 | Christos Stavrakakis | continue
|
543 | 89b2b908 | Christos Stavrakakis | |
544 | 0ccb6461 | Christos Stavrakakis | try:
|
545 | 0ccb6461 | Christos Stavrakakis | hanging_groups = self.ganeti_hanging_networks[bend][network.id]
|
546 | 0ccb6461 | Christos Stavrakakis | except KeyError: |
547 | 0ccb6461 | Christos Stavrakakis | # Network is connected to all nodegroups
|
548 | 0ccb6461 | Christos Stavrakakis | hanging_groups = [] |
549 | 0ccb6461 | Christos Stavrakakis | |
550 | 0ccb6461 | Christos Stavrakakis | if hanging_groups:
|
551 | 0ccb6461 | Christos Stavrakakis | # CASE-3: Ganeti networks not connected to all nodegroups
|
552 | 0ccb6461 | Christos Stavrakakis | self.reconcile_hanging_groups(network, bend,
|
553 | 0ccb6461 | Christos Stavrakakis | hanging_groups) |
554 | 0ccb6461 | Christos Stavrakakis | continue
|
555 | 89b2b908 | Christos Stavrakakis | |
556 | 0ccb6461 | Christos Stavrakakis | if bnet.operstate != 'ACTIVE': |
557 | 0ccb6461 | Christos Stavrakakis | # CASE-4: Unsynced network state. At this point the network
|
558 | 0ccb6461 | Christos Stavrakakis | # exists and is connected to all nodes so is must be
|
559 | 0ccb6461 | Christos Stavrakakis | # active!
|
560 | 0ccb6461 | Christos Stavrakakis | self.reconcile_unsynced_network(network, bend, bnet)
|
561 | 0ccb6461 | Christos Stavrakakis | |
562 | 0ccb6461 | Christos Stavrakakis | # Check that externally reserved IPs of the network in Ganeti are
|
563 | 0ccb6461 | Christos Stavrakakis | # also externally reserved to the IP pool
|
564 | 0ccb6461 | Christos Stavrakakis | externally_reserved = gnet['external_reservations']
|
565 | 98a01362 | Christos Stavrakakis | if externally_reserved:
|
566 | 98a01362 | Christos Stavrakakis | for ip in externally_reserved.split(","): |
567 | 98a01362 | Christos Stavrakakis | ip = ip.strip() |
568 | 98a01362 | Christos Stavrakakis | if not network_ip_pool.is_reserved(ip): |
569 | 98a01362 | Christos Stavrakakis | msg = ("D: IP '%s' is reserved for network '%s' in"
|
570 | 98a01362 | Christos Stavrakakis | " backend '%s' but not in DB.")
|
571 | 98a01362 | Christos Stavrakakis | self.log.info(msg, ip, network, bend)
|
572 | 98a01362 | Christos Stavrakakis | if self.fix: |
573 | 98a01362 | Christos Stavrakakis | network_ip_pool.reserve(ip, external=True)
|
574 | 98a01362 | Christos Stavrakakis | network_ip_pool.save() |
575 | 98a01362 | Christos Stavrakakis | self.log.info("F: Reserved IP '%s'", ip) |
576 | 89b2b908 | Christos Stavrakakis | |
577 | 89b2b908 | Christos Stavrakakis | def reconcile_parted_network(self, network, backend): |
578 | 89b2b908 | Christos Stavrakakis | self.log.info("D: Missing DB entry for network %s in backend %s", |
579 | 89b2b908 | Christos Stavrakakis | network, backend) |
580 | 89b2b908 | Christos Stavrakakis | if self.fix: |
581 | 89b2b908 | Christos Stavrakakis | network.create_backend_network(backend) |
582 | 89b2b908 | Christos Stavrakakis | self.log.info("F: Created DB entry") |
583 | 89b2b908 | Christos Stavrakakis | bnet = get_backend_network(network, backend) |
584 | 89b2b908 | Christos Stavrakakis | return bnet
|
585 | 89b2b908 | Christos Stavrakakis | |
586 | 89b2b908 | Christos Stavrakakis | def reconcile_stale_network(self, backend_network): |
587 | 89b2b908 | Christos Stavrakakis | self.log.info("D: Stale DB entry for network %s in backend %s", |
588 | 89b2b908 | Christos Stavrakakis | backend_network.network, backend_network.backend) |
589 | 89b2b908 | Christos Stavrakakis | if self.fix: |
590 | 89b2b908 | Christos Stavrakakis | backend_mod.process_network_status( |
591 | 89b2b908 | Christos Stavrakakis | backend_network, self.event_time, 0, |
592 | 89b2b908 | Christos Stavrakakis | "OP_NETWORK_REMOVE",
|
593 | 89b2b908 | Christos Stavrakakis | "success",
|
594 | 89b2b908 | Christos Stavrakakis | "Reconciliation simulated event")
|
595 | 89b2b908 | Christos Stavrakakis | self.log.info("F: Reconciled event: OP_NETWORK_REMOVE") |
596 | 89b2b908 | Christos Stavrakakis | |
597 | 89b2b908 | Christos Stavrakakis | def reconcile_missing_network(self, network, backend): |
598 | 89b2b908 | Christos Stavrakakis | self.log.info("D: Missing Ganeti network %s in backend %s", |
599 | 89b2b908 | Christos Stavrakakis | network, backend) |
600 | 89b2b908 | Christos Stavrakakis | if self.fix: |
601 | 89b2b908 | Christos Stavrakakis | backend_mod.create_network(network, backend) |
602 | 89b2b908 | Christos Stavrakakis | self.log.info("F: Issued OP_NETWORK_CONNECT") |
603 | 89b2b908 | Christos Stavrakakis | |
604 | 89b2b908 | Christos Stavrakakis | def reconcile_hanging_groups(self, network, backend, hanging_groups): |
605 | 89b2b908 | Christos Stavrakakis | self.log.info('D: Network %s in backend %s is not connected to ' |
606 | 89b2b908 | Christos Stavrakakis | 'the following groups:', network, backend)
|
607 | 89b2b908 | Christos Stavrakakis | self.log.info('- ' + '\n- '.join(hanging_groups)) |
608 | 89b2b908 | Christos Stavrakakis | if self.fix: |
609 | 89b2b908 | Christos Stavrakakis | for group in hanging_groups: |
610 | 89b2b908 | Christos Stavrakakis | self.log.info('F: Connecting network %s to nodegroup %s', |
611 | 89b2b908 | Christos Stavrakakis | network, group) |
612 | 89b2b908 | Christos Stavrakakis | backend_mod.connect_network(network, backend, depends=[], |
613 | 89b2b908 | Christos Stavrakakis | group=group) |
614 | 89b2b908 | Christos Stavrakakis | |
615 | 89b2b908 | Christos Stavrakakis | def reconcile_unsynced_network(self, network, backend, backend_network): |
616 | 89b2b908 | Christos Stavrakakis | self.log.info("D: Unsynced network %s in backend %s", network, backend) |
617 | 89b2b908 | Christos Stavrakakis | if self.fix: |
618 | 89b2b908 | Christos Stavrakakis | self.log.info("F: Issuing OP_NETWORK_CONNECT") |
619 | 89b2b908 | Christos Stavrakakis | backend_mod.process_network_status( |
620 | 89b2b908 | Christos Stavrakakis | backend_network, self.event_time, 0, |
621 | 89b2b908 | Christos Stavrakakis | "OP_NETWORK_CONNECT",
|
622 | 89b2b908 | Christos Stavrakakis | "success",
|
623 | 89b2b908 | Christos Stavrakakis | "Reconciliation simulated eventd")
|
624 | 89b2b908 | Christos Stavrakakis | |
625 | 0ccb6461 | Christos Stavrakakis | def _reconcile_orphan_networks(self): |
626 | 0ccb6461 | Christos Stavrakakis | db_networks = self.networks
|
627 | 0ccb6461 | Christos Stavrakakis | ganeti_networks = self.ganeti_networks
|
628 | 89b2b908 | Christos Stavrakakis | # Detect Orphan Networks in Ganeti
|
629 | 89b2b908 | Christos Stavrakakis | db_network_ids = set([net.id for net in db_networks]) |
630 | 89b2b908 | Christos Stavrakakis | for back_end, ganeti_networks in ganeti_networks.items(): |
631 | 89b2b908 | Christos Stavrakakis | ganeti_network_ids = set(ganeti_networks.keys())
|
632 | 89b2b908 | Christos Stavrakakis | orphans = ganeti_network_ids - db_network_ids |
633 | 89b2b908 | Christos Stavrakakis | |
634 | 89b2b908 | Christos Stavrakakis | if len(orphans) > 0: |
635 | 89b2b908 | Christos Stavrakakis | self.log.info('D: Orphan Networks in backend %s:', |
636 | 89b2b908 | Christos Stavrakakis | back_end.clustername) |
637 | 89b2b908 | Christos Stavrakakis | self.log.info('- ' + '\n- '.join([str(o) for o in orphans])) |
638 | 89b2b908 | Christos Stavrakakis | if self.fix: |
639 | 89b2b908 | Christos Stavrakakis | for net_id in orphans: |
640 | 89b2b908 | Christos Stavrakakis | self.log.info('Disconnecting and deleting network %d', |
641 | 89b2b908 | Christos Stavrakakis | net_id) |
642 | 89b2b908 | Christos Stavrakakis | try:
|
643 | 89b2b908 | Christos Stavrakakis | network = Network.objects.get(id=net_id) |
644 | 89b2b908 | Christos Stavrakakis | backend_mod.delete_network(network, |
645 | 89b2b908 | Christos Stavrakakis | backend=back_end) |
646 | 89b2b908 | Christos Stavrakakis | except Network.DoesNotExist:
|
647 | 89b2b908 | Christos Stavrakakis | self.log.info("Not entry for network %s in DB !!", |
648 | 89b2b908 | Christos Stavrakakis | net_id) |
649 | 89b2b908 | Christos Stavrakakis | |
650 | 89b2b908 | Christos Stavrakakis | |
651 | 89b2b908 | Christos Stavrakakis | def get_backend_network(network, backend): |
652 | 89b2b908 | Christos Stavrakakis | try:
|
653 | 89b2b908 | Christos Stavrakakis | return BackendNetwork.objects.get(network=network, backend=backend)
|
654 | 89b2b908 | Christos Stavrakakis | except BackendNetwork.DoesNotExist:
|
655 | 89b2b908 | Christos Stavrakakis | return None |
656 | 89b2b908 | Christos Stavrakakis | |
657 | 89b2b908 | Christos Stavrakakis | |
658 | 0ccb6461 | Christos Stavrakakis | class PoolReconciler(object): |
659 | 0ccb6461 | Christos Stavrakakis | def __init__(self, logger, fix=False): |
660 | 0ccb6461 | Christos Stavrakakis | self.log = logger
|
661 | 0ccb6461 | Christos Stavrakakis | self.fix = fix
|
662 | 0ccb6461 | Christos Stavrakakis | |
663 | 0ccb6461 | Christos Stavrakakis | def reconcile(self): |
664 | 0ccb6461 | Christos Stavrakakis | self.reconcile_bridges()
|
665 | 0ccb6461 | Christos Stavrakakis | self.reconcile_mac_prefixes()
|
666 | 0ccb6461 | Christos Stavrakakis | for network in Network.objects.filter(deleted=False): |
667 | 0ccb6461 | Christos Stavrakakis | self.reconcile_ip_pool(network)
|
668 | 0ccb6461 | Christos Stavrakakis | |
669 | 0ccb6461 | Christos Stavrakakis | @transaction.commit_on_success
|
670 | 0ccb6461 | Christos Stavrakakis | def reconcile_bridges(self): |
671 | 0ccb6461 | Christos Stavrakakis | networks = Network.objects.filter(deleted=False,
|
672 | 0ccb6461 | Christos Stavrakakis | flavor="PHYSICAL_VLAN")
|
673 | 0ccb6461 | Christos Stavrakakis | check_unique_values(objects=networks, field='link', logger=self.log) |
674 | 0ccb6461 | Christos Stavrakakis | try:
|
675 | 0ccb6461 | Christos Stavrakakis | pool = BridgePoolTable.get_pool() |
676 | 0ccb6461 | Christos Stavrakakis | except pools.EmptyPool:
|
677 | 0ccb6461 | Christos Stavrakakis | self.log.info("There is no available pool for bridges.") |
678 | 0ccb6461 | Christos Stavrakakis | return
|
679 | 0ccb6461 | Christos Stavrakakis | |
680 | 0ccb6461 | Christos Stavrakakis | used_bridges = set(networks.values_list('link', flat=True)) |
681 | 0ccb6461 | Christos Stavrakakis | check_pool_consistent(pool=pool, pool_class=pools.BridgePool, |
682 | 0ccb6461 | Christos Stavrakakis | used_values=used_bridges, fix=self.fix,
|
683 | 0ccb6461 | Christos Stavrakakis | logger=self.log)
|
684 | 89b2b908 | Christos Stavrakakis | |
685 | 0ccb6461 | Christos Stavrakakis | @transaction.commit_on_success
|
686 | 0ccb6461 | Christos Stavrakakis | def reconcile_mac_prefixes(self): |
687 | 0ccb6461 | Christos Stavrakakis | networks = Network.objects.filter(deleted=False, flavor="MAC_FILTERED") |
688 | 0ccb6461 | Christos Stavrakakis | check_unique_values(objects=networks, field='mac_prefix',
|
689 | 0ccb6461 | Christos Stavrakakis | logger=self.log)
|
690 | 0ccb6461 | Christos Stavrakakis | try:
|
691 | 0ccb6461 | Christos Stavrakakis | pool = MacPrefixPoolTable.get_pool() |
692 | 0ccb6461 | Christos Stavrakakis | except pools.EmptyPool:
|
693 | 0ccb6461 | Christos Stavrakakis | self.log.info("There is no available pool for MAC prefixes.") |
694 | 0ccb6461 | Christos Stavrakakis | return
|
695 | 0ccb6461 | Christos Stavrakakis | |
696 | 0ccb6461 | Christos Stavrakakis | used_mac_prefixes = set(networks.values_list('mac_prefix', flat=True)) |
697 | 0ccb6461 | Christos Stavrakakis | check_pool_consistent(pool=pool, pool_class=pools.MacPrefixPool, |
698 | 0ccb6461 | Christos Stavrakakis | used_values=used_mac_prefixes, fix=self.fix,
|
699 | 0ccb6461 | Christos Stavrakakis | logger=self.log)
|
700 | 89b2b908 | Christos Stavrakakis | |
701 | 0ccb6461 | Christos Stavrakakis | @transaction.commit_on_success
|
702 | 0ccb6461 | Christos Stavrakakis | def reconcile_ip_pool(self, network): |
703 | 0ccb6461 | Christos Stavrakakis | # Check that all NICs have unique IPv4 address
|
704 | 0ccb6461 | Christos Stavrakakis | nics = network.nics.filter(ipv4__isnull=False)
|
705 | 0ccb6461 | Christos Stavrakakis | check_unique_values(objects=nics, field='ipv4', logger=self.log) |
706 | 0ccb6461 | Christos Stavrakakis | |
707 | 0ccb6461 | Christos Stavrakakis | # Check that all Floating IPs have unique IPv4 address
|
708 | 0ccb6461 | Christos Stavrakakis | floating_ips = network.floating_ips.filter(deleted=False)
|
709 | 0ccb6461 | Christos Stavrakakis | check_unique_values(objects=floating_ips, field='ipv4',
|
710 | 0ccb6461 | Christos Stavrakakis | logger=self.log)
|
711 | 0ccb6461 | Christos Stavrakakis | |
712 | 0ccb6461 | Christos Stavrakakis | # First get(lock) the IP pool of the network to prevent new NICs
|
713 | 0ccb6461 | Christos Stavrakakis | # from being created.
|
714 | 0ccb6461 | Christos Stavrakakis | network_ip_pool = network.get_pool() |
715 | 0ccb6461 | Christos Stavrakakis | used_ips = set(list(nics.values_list("ipv4", flat=True)) + |
716 | 0ccb6461 | Christos Stavrakakis | list(floating_ips.values_list("ipv4", flat=True))) |
717 | 0ccb6461 | Christos Stavrakakis | |
718 | 0ccb6461 | Christos Stavrakakis | check_pool_consistent(pool=network_ip_pool, |
719 | 0ccb6461 | Christos Stavrakakis | pool_class=pools.IPPool, |
720 | 0ccb6461 | Christos Stavrakakis | used_values=used_ips, |
721 | 0ccb6461 | Christos Stavrakakis | fix=self.fix, logger=self.log) |
722 | 0ccb6461 | Christos Stavrakakis | |
723 | 0ccb6461 | Christos Stavrakakis | |
724 | 0ccb6461 | Christos Stavrakakis | def check_unique_values(objects, field, logger): |
725 | 0ccb6461 | Christos Stavrakakis | used_values = list(objects.values_list(field, flat=True)) |
726 | 0ccb6461 | Christos Stavrakakis | if len(used_values) != len(set(used_values)): |
727 | 0ccb6461 | Christos Stavrakakis | duplicate_values = [v for v in used_values if used_values.count(v) > 1] |
728 | 0ccb6461 | Christos Stavrakakis | for value in duplicate_values: |
729 | 0ccb6461 | Christos Stavrakakis | filter_args = {field: value} |
730 | 0ccb6461 | Christos Stavrakakis | using_objects = objects.filter(**filter_args) |
731 | 0ccb6461 | Christos Stavrakakis | msg = "Value '%s' is used as %s for more than one objects: %s"
|
732 | 0ccb6461 | Christos Stavrakakis | logger.error(msg, value, field, ",".join(map(str, using_objects))) |
733 | 0ccb6461 | Christos Stavrakakis | return False |
734 | 0ccb6461 | Christos Stavrakakis | logger.debug("Values for field '%s' are unique.", field)
|
735 | 0ccb6461 | Christos Stavrakakis | return True |
736 | 0ccb6461 | Christos Stavrakakis | |
737 | 0ccb6461 | Christos Stavrakakis | |
738 | 0ccb6461 | Christos Stavrakakis | def check_pool_consistent(pool, pool_class, used_values, fix, logger): |
739 | 0ccb6461 | Christos Stavrakakis | dummy_pool = create_empty_pool(pool, pool_class) |
740 | 0ccb6461 | Christos Stavrakakis | [dummy_pool.reserve(value) for value in used_values] |
741 | 0ccb6461 | Christos Stavrakakis | if dummy_pool.available != pool.available:
|
742 | 0ccb6461 | Christos Stavrakakis | msg = "'%s' is not consistent!\nPool: %s\nUsed: %s"
|
743 | 0ccb6461 | Christos Stavrakakis | pool_diff = dummy_pool.available ^ pool.available |
744 | 0ccb6461 | Christos Stavrakakis | for index in pool_diff.itersearch(bitarray.bitarray("1")): |
745 | 0ccb6461 | Christos Stavrakakis | value = pool.index_to_value(int(index))
|
746 | 0ccb6461 | Christos Stavrakakis | msg = "%s is incosistent! Value '%s' is %s but should be %s."
|
747 | 0ccb6461 | Christos Stavrakakis | value1 = pool.is_available(value) and "available" or "unavailable" |
748 | 0ccb6461 | Christos Stavrakakis | value2 = dummy_pool.is_available(value) and "available"\ |
749 | 0ccb6461 | Christos Stavrakakis | or "unavailable" |
750 | 0ccb6461 | Christos Stavrakakis | logger.error(msg, pool, value, value1, value2) |
751 | 0ccb6461 | Christos Stavrakakis | if fix:
|
752 | 0ccb6461 | Christos Stavrakakis | pool.available = dummy_pool.available |
753 | 0ccb6461 | Christos Stavrakakis | pool.save() |
754 | 0ccb6461 | Christos Stavrakakis | logger.info("Fixed available map of pool '%s'", pool)
|
755 | 0ccb6461 | Christos Stavrakakis | |
756 | 0ccb6461 | Christos Stavrakakis | |
757 | 0ccb6461 | Christos Stavrakakis | def create_empty_pool(pool, pool_class): |
758 | 0ccb6461 | Christos Stavrakakis | pool_row = pool.pool_table |
759 | 0ccb6461 | Christos Stavrakakis | pool_row.available_map = ""
|
760 | 0ccb6461 | Christos Stavrakakis | pool_row.reserved_map = ""
|
761 | 0ccb6461 | Christos Stavrakakis | return pool_class(pool_row) |