root / snf-cyclades-app / synnefo / logic / reconciliation.py @ 6fec4302
History | View | Annotate | Download (31.8 kB)
1 | 9fea53cc | Vangelis Koukis | # -*- coding: utf-8 -*-
|
---|---|---|---|
2 | 9fea53cc | Vangelis Koukis | #
|
3 | 75dc539e | Christos Stavrakakis | # Copyright 2011-2013 GRNET S.A. All rights reserved.
|
4 | 9fea53cc | Vangelis Koukis | #
|
5 | 9fea53cc | Vangelis Koukis | # Redistribution and use in source and binary forms, with or
|
6 | 9fea53cc | Vangelis Koukis | # without modification, are permitted provided that the following
|
7 | 9fea53cc | Vangelis Koukis | # conditions are met:
|
8 | 9fea53cc | Vangelis Koukis | #
|
9 | 9fea53cc | Vangelis Koukis | # 1. Redistributions of source code must retain the above
|
10 | 9fea53cc | Vangelis Koukis | # copyright notice, this list of conditions and the following
|
11 | 9fea53cc | Vangelis Koukis | # disclaimer.
|
12 | 9fea53cc | Vangelis Koukis | #
|
13 | 9fea53cc | Vangelis Koukis | # 2. Redistributions in binary form must reproduce the above
|
14 | 9fea53cc | Vangelis Koukis | # copyright notice, this list of conditions and the following
|
15 | 9fea53cc | Vangelis Koukis | # disclaimer in the documentation and/or other materials
|
16 | 9fea53cc | Vangelis Koukis | # provided with the distribution.
|
17 | 9fea53cc | Vangelis Koukis | #
|
18 | 9fea53cc | Vangelis Koukis | # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
|
19 | 9fea53cc | Vangelis Koukis | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
20 | 9fea53cc | Vangelis Koukis | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
21 | 9fea53cc | Vangelis Koukis | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
|
22 | 9fea53cc | Vangelis Koukis | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
23 | 9fea53cc | Vangelis Koukis | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
24 | 9fea53cc | Vangelis Koukis | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
25 | 9fea53cc | Vangelis Koukis | # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
26 | 9fea53cc | Vangelis Koukis | # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
27 | 9fea53cc | Vangelis Koukis | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
28 | 9fea53cc | Vangelis Koukis | # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
29 | 9fea53cc | Vangelis Koukis | # POSSIBILITY OF SUCH DAMAGE.
|
30 | 9fea53cc | Vangelis Koukis | #
|
31 | 9fea53cc | Vangelis Koukis | # The views and conclusions contained in the software and
|
32 | 9fea53cc | Vangelis Koukis | # documentation are those of the authors and should not be
|
33 | 9fea53cc | Vangelis Koukis | # interpreted as representing official policies, either expressed
|
34 | 9fea53cc | Vangelis Koukis | # or implied, of GRNET S.A.
|
35 | 9fea53cc | Vangelis Koukis | #
|
36 | 9fea53cc | Vangelis Koukis | """Business logic for reconciliation
|
37 | 9fea53cc | Vangelis Koukis |
|
38 | 9fea53cc | Vangelis Koukis | Reconcile the contents of the DB with the actual state of the
|
39 | 9fea53cc | Vangelis Koukis | Ganeti backend.
|
40 | 9fea53cc | Vangelis Koukis |
|
41 | 9fea53cc | Vangelis Koukis | Let D be the set of VMs in the DB, G the set of VMs in Ganeti.
|
42 | 9fea53cc | Vangelis Koukis | RULES:
|
43 | 9fea53cc | Vangelis Koukis | R1. Stale servers in DB:
|
44 | 9fea53cc | Vangelis Koukis | For any v in D but not in G:
|
45 | 9fea53cc | Vangelis Koukis | Set deleted=True.
|
46 | 9fea53cc | Vangelis Koukis | R2. Orphan instances in Ganet:
|
47 | 9fea53cc | Vangelis Koukis | For any v in G with deleted=True in D:
|
48 | 9fea53cc | Vangelis Koukis | Issue OP_INSTANCE_DESTROY.
|
49 | 9fea53cc | Vangelis Koukis | R3. Unsynced operstate:
|
50 | 9fea53cc | Vangelis Koukis | For any v whose operating state differs between G and V:
|
51 | 9fea53cc | Vangelis Koukis | Set the operating state in D based on the state in G.
|
52 | 9fea53cc | Vangelis Koukis | In the code, D, G are Python dicts mapping instance ids to operating state.
|
53 | 9fea53cc | Vangelis Koukis | For D, the operating state is chosen from VirtualMachine.OPER_STATES.
|
54 | 9fea53cc | Vangelis Koukis | For G, the operating state is True if the machine is up, False otherwise.
|
55 | 9fea53cc | Vangelis Koukis |
|
56 | 9fea53cc | Vangelis Koukis | """
|
57 | 9fea53cc | Vangelis Koukis | |
58 | 9fea53cc | Vangelis Koukis | |
59 | 0c09b1c0 | Christos Stavrakakis | from django.conf import settings |
60 | 4161cb41 | Christos Stavrakakis | |
61 | 75dc539e | Christos Stavrakakis | import logging |
62 | 75dc539e | Christos Stavrakakis | import itertools |
63 | 89b2b908 | Christos Stavrakakis | import bitarray |
64 | 1cb7846c | Christos Stavrakakis | from datetime import datetime, timedelta |
65 | 4161cb41 | Christos Stavrakakis | |
66 | 75dc539e | Christos Stavrakakis | from django.db import transaction |
67 | 75dc539e | Christos Stavrakakis | from synnefo.db.models import (Backend, VirtualMachine, Flavor, |
68 | 89b2b908 | Christos Stavrakakis | pooled_rapi_client, Network, |
69 | 0ccb6461 | Christos Stavrakakis | BackendNetwork, BridgePoolTable, |
70 | 0ccb6461 | Christos Stavrakakis | MacPrefixPoolTable) |
71 | 0ccb6461 | Christos Stavrakakis | from synnefo.db import pools |
72 | edbc1d5a | Christos Stavrakakis | from synnefo.logic import utils, rapi, backend as backend_mod |
73 | 7ef05bd4 | Christos Stavrakakis | from synnefo.lib.utils import merge_time |
74 | 9fea53cc | Vangelis Koukis | |
75 | 75dc539e | Christos Stavrakakis | logger = logging.getLogger() |
76 | 75dc539e | Christos Stavrakakis | logging.basicConfig() |
77 | 9e98ba3c | Giorgos Verigakis | |
78 | 1cb7846c | Christos Stavrakakis | BUILDING_NIC_TIMEOUT = timedelta(seconds=120)
|
79 | e63050ca | Christos Stavrakakis | |
80 | e63050ca | Christos Stavrakakis | |
81 | 75dc539e | Christos Stavrakakis | class BackendReconciler(object): |
82 | 75dc539e | Christos Stavrakakis | def __init__(self, backend, logger, options=None): |
83 | 75dc539e | Christos Stavrakakis | self.backend = backend
|
84 | 75dc539e | Christos Stavrakakis | self.log = logger
|
85 | 75dc539e | Christos Stavrakakis | self.client = backend.get_client()
|
86 | 75dc539e | Christos Stavrakakis | if options is None: |
87 | 75dc539e | Christos Stavrakakis | self.options = {}
|
88 | 4161cb41 | Christos Stavrakakis | else:
|
89 | 75dc539e | Christos Stavrakakis | self.options = options
|
90 | 75dc539e | Christos Stavrakakis | |
91 | 75dc539e | Christos Stavrakakis | def close(self): |
92 | 75dc539e | Christos Stavrakakis | self.backend.put_client(self.client) |
93 | 75dc539e | Christos Stavrakakis | |
94 | 75dc539e | Christos Stavrakakis | @transaction.commit_on_success
|
95 | 75dc539e | Christos Stavrakakis | def reconcile(self): |
96 | 75dc539e | Christos Stavrakakis | log = self.log
|
97 | 75dc539e | Christos Stavrakakis | backend = self.backend
|
98 | 75dc539e | Christos Stavrakakis | log.debug("Reconciling backend %s", backend)
|
99 | 75dc539e | Christos Stavrakakis | |
100 | 7ef05bd4 | Christos Stavrakakis | self.event_time = datetime.now()
|
101 | 7ef05bd4 | Christos Stavrakakis | |
102 | 75dc539e | Christos Stavrakakis | self.db_servers = get_database_servers(backend)
|
103 | 75dc539e | Christos Stavrakakis | self.db_servers_keys = set(self.db_servers.keys()) |
104 | 75dc539e | Christos Stavrakakis | log.debug("Got servers info from database.")
|
105 | 75dc539e | Christos Stavrakakis | |
106 | 75dc539e | Christos Stavrakakis | self.gnt_servers = get_ganeti_servers(backend)
|
107 | 75dc539e | Christos Stavrakakis | self.gnt_servers_keys = set(self.gnt_servers.keys()) |
108 | 75dc539e | Christos Stavrakakis | log.debug("Got servers info from Ganeti backend.")
|
109 | 75dc539e | Christos Stavrakakis | |
110 | 63f9eb8e | Christos Stavrakakis | self.gnt_jobs = get_ganeti_jobs(backend)
|
111 | 63f9eb8e | Christos Stavrakakis | log.debug("Got jobs from Ganeti backend")
|
112 | 63f9eb8e | Christos Stavrakakis | |
113 | 75dc539e | Christos Stavrakakis | self.stale_servers = self.reconcile_stale_servers() |
114 | 75dc539e | Christos Stavrakakis | self.orphan_servers = self.reconcile_orphan_servers() |
115 | 75dc539e | Christos Stavrakakis | self.unsynced_servers = self.reconcile_unsynced_servers() |
116 | 75dc539e | Christos Stavrakakis | self.close()
|
117 | 75dc539e | Christos Stavrakakis | |
118 | 75dc539e | Christos Stavrakakis | def get_build_status(self, db_server): |
119 | 7ef05bd4 | Christos Stavrakakis | """Return the status of the build job.
|
120 | 7ef05bd4 | Christos Stavrakakis |
|
121 | 7ef05bd4 | Christos Stavrakakis | Return whether the job is RUNNING, FINALIZED or ERROR, together
|
122 | 7ef05bd4 | Christos Stavrakakis | with the timestamp that the job finished (if any).
|
123 | 7ef05bd4 | Christos Stavrakakis |
|
124 | 7ef05bd4 | Christos Stavrakakis | """
|
125 | 63f9eb8e | Christos Stavrakakis | job_id = db_server.backendjobid |
126 | 63f9eb8e | Christos Stavrakakis | if job_id in self.gnt_jobs: |
127 | 7ef05bd4 | Christos Stavrakakis | job = self.gnt_jobs[job_id]
|
128 | 7ef05bd4 | Christos Stavrakakis | gnt_job_status = job["status"]
|
129 | a8817717 | Christos Stavrakakis | end_timestamp = job["end_ts"]
|
130 | a8817717 | Christos Stavrakakis | if end_timestamp is not None: |
131 | a8817717 | Christos Stavrakakis | end_timestamp = merge_time(end_timestamp) |
132 | edbc1d5a | Christos Stavrakakis | if gnt_job_status == rapi.JOB_STATUS_ERROR:
|
133 | 7ef05bd4 | Christos Stavrakakis | return "ERROR", end_timestamp |
134 | edbc1d5a | Christos Stavrakakis | elif gnt_job_status not in rapi.JOB_STATUS_FINALIZED: |
135 | 7ef05bd4 | Christos Stavrakakis | return "RUNNING", None |
136 | 75dc539e | Christos Stavrakakis | else:
|
137 | 7ef05bd4 | Christos Stavrakakis | return "FINALIZED", end_timestamp |
138 | 75dc539e | Christos Stavrakakis | else:
|
139 | 7ef05bd4 | Christos Stavrakakis | return "ERROR", None |
140 | 75dc539e | Christos Stavrakakis | |
141 | 75dc539e | Christos Stavrakakis | def reconcile_stale_servers(self): |
142 | 75dc539e | Christos Stavrakakis | # Detect stale servers
|
143 | 75dc539e | Christos Stavrakakis | stale = [] |
144 | 75dc539e | Christos Stavrakakis | stale_keys = self.db_servers_keys - self.gnt_servers_keys |
145 | 75dc539e | Christos Stavrakakis | for server_id in stale_keys: |
146 | 75dc539e | Christos Stavrakakis | db_server = self.db_servers[server_id]
|
147 | 75dc539e | Christos Stavrakakis | if db_server.operstate == "BUILD": |
148 | 7ef05bd4 | Christos Stavrakakis | build_status, end_timestamp = self.get_build_status(db_server)
|
149 | 75dc539e | Christos Stavrakakis | if build_status == "ERROR": |
150 | 75dc539e | Christos Stavrakakis | # Special handling of BUILD eerrors
|
151 | 75dc539e | Christos Stavrakakis | self.reconcile_building_server(db_server)
|
152 | 75dc539e | Christos Stavrakakis | elif build_status != "RUNNING": |
153 | 75dc539e | Christos Stavrakakis | stale.append(server_id) |
154 | 1464a17b | Christos Stavrakakis | elif (db_server.operstate == "ERROR" and |
155 | 1464a17b | Christos Stavrakakis | db_server.action != "DESTROY"):
|
156 | 1464a17b | Christos Stavrakakis | # Servers at building ERROR are stale only if the user has
|
157 | 1464a17b | Christos Stavrakakis | # asked to destroy them.
|
158 | 1464a17b | Christos Stavrakakis | pass
|
159 | 75dc539e | Christos Stavrakakis | else:
|
160 | 75dc539e | Christos Stavrakakis | stale.append(server_id) |
161 | 75dc539e | Christos Stavrakakis | |
162 | 75dc539e | Christos Stavrakakis | # Report them
|
163 | 75dc539e | Christos Stavrakakis | if stale:
|
164 | 75dc539e | Christos Stavrakakis | self.log.info("Found stale servers %s at backend %s", |
165 | 75dc539e | Christos Stavrakakis | ", ".join(map(str, stale)), self.backend) |
166 | 75dc539e | Christos Stavrakakis | else:
|
167 | 75dc539e | Christos Stavrakakis | self.log.debug("No stale servers at backend %s", self.backend) |
168 | 75dc539e | Christos Stavrakakis | |
169 | 75dc539e | Christos Stavrakakis | # Fix them
|
170 | 75dc539e | Christos Stavrakakis | if stale and self.options["fix_stale"]: |
171 | 75dc539e | Christos Stavrakakis | for server_id in stale: |
172 | 45ead074 | Christos Stavrakakis | vm = get_locked_server(server_id) |
173 | 75dc539e | Christos Stavrakakis | backend_mod.process_op_status( |
174 | 45ead074 | Christos Stavrakakis | vm=vm, |
175 | 75dc539e | Christos Stavrakakis | etime=self.event_time,
|
176 | 75dc539e | Christos Stavrakakis | jobid=-0,
|
177 | 75dc539e | Christos Stavrakakis | opcode='OP_INSTANCE_REMOVE', status='success', |
178 | 75dc539e | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
179 | 75dc539e | Christos Stavrakakis | self.log.debug("Simulated Ganeti removal for stale servers.") |
180 | 75dc539e | Christos Stavrakakis | |
181 | 75dc539e | Christos Stavrakakis | def reconcile_orphan_servers(self): |
182 | 75dc539e | Christos Stavrakakis | orphans = self.gnt_servers_keys - self.db_servers_keys |
183 | 75dc539e | Christos Stavrakakis | if orphans:
|
184 | 75dc539e | Christos Stavrakakis | self.log.info("Found orphan servers %s at backend %s", |
185 | 75dc539e | Christos Stavrakakis | ", ".join(map(str, orphans)), self.backend) |
186 | 75dc539e | Christos Stavrakakis | else:
|
187 | 75dc539e | Christos Stavrakakis | self.log.debug("No orphan servers at backend %s", self.backend) |
188 | 75dc539e | Christos Stavrakakis | |
189 | 75dc539e | Christos Stavrakakis | if orphans and self.options["fix_orphans"]: |
190 | 75dc539e | Christos Stavrakakis | for server_id in orphans: |
191 | 75dc539e | Christos Stavrakakis | server_name = utils.id_to_instance_name(server_id) |
192 | 75dc539e | Christos Stavrakakis | self.client.DeleteInstance(server_name)
|
193 | 75dc539e | Christos Stavrakakis | self.log.debug("Issued OP_INSTANCE_REMOVE for orphan servers.") |
194 | 75dc539e | Christos Stavrakakis | |
195 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_servers(self): |
196 | 75dc539e | Christos Stavrakakis | #log = self.log
|
197 | 75dc539e | Christos Stavrakakis | for server_id in self.db_servers_keys & self.gnt_servers_keys: |
198 | 75dc539e | Christos Stavrakakis | db_server = self.db_servers[server_id]
|
199 | 75dc539e | Christos Stavrakakis | gnt_server = self.gnt_servers[server_id]
|
200 | 75dc539e | Christos Stavrakakis | if db_server.operstate == "BUILD": |
201 | 7ef05bd4 | Christos Stavrakakis | build_status, end_timestamp = self.get_build_status(db_server)
|
202 | 75dc539e | Christos Stavrakakis | if build_status == "RUNNING": |
203 | 75dc539e | Christos Stavrakakis | # Do not reconcile building VMs
|
204 | 75dc539e | Christos Stavrakakis | continue
|
205 | 75dc539e | Christos Stavrakakis | elif build_status == "ERROR": |
206 | 75dc539e | Christos Stavrakakis | # Special handling of build errors
|
207 | 75dc539e | Christos Stavrakakis | self.reconcile_building_server(db_server)
|
208 | 75dc539e | Christos Stavrakakis | continue
|
209 | 7ef05bd4 | Christos Stavrakakis | elif end_timestamp >= self.event_time: |
210 | 7ef05bd4 | Christos Stavrakakis | # Do not continue reconciliation for building server that
|
211 | 7ef05bd4 | Christos Stavrakakis | # the build job completed after quering the state of
|
212 | 7ef05bd4 | Christos Stavrakakis | # Ganeti servers.
|
213 | 7ef05bd4 | Christos Stavrakakis | continue
|
214 | 75dc539e | Christos Stavrakakis | |
215 | 75dc539e | Christos Stavrakakis | self.reconcile_unsynced_operstate(server_id, db_server,
|
216 | 75dc539e | Christos Stavrakakis | gnt_server) |
217 | 75dc539e | Christos Stavrakakis | self.reconcile_unsynced_flavor(server_id, db_server,
|
218 | 75dc539e | Christos Stavrakakis | gnt_server) |
219 | 75dc539e | Christos Stavrakakis | self.reconcile_unsynced_nics(server_id, db_server, gnt_server)
|
220 | 75dc539e | Christos Stavrakakis | self.reconcile_unsynced_disks(server_id, db_server, gnt_server)
|
221 | 63f9eb8e | Christos Stavrakakis | if db_server.task is not None: |
222 | 63f9eb8e | Christos Stavrakakis | self.reconcile_pending_task(server_id, db_server)
|
223 | 75dc539e | Christos Stavrakakis | |
224 | 75dc539e | Christos Stavrakakis | def reconcile_building_server(self, db_server): |
225 | 75dc539e | Christos Stavrakakis | self.log.info("Server '%s' is BUILD in DB, but 'ERROR' in Ganeti.", |
226 | 75dc539e | Christos Stavrakakis | db_server.id) |
227 | 75dc539e | Christos Stavrakakis | if self.options["fix_unsynced"]: |
228 | 75dc539e | Christos Stavrakakis | fix_opcode = "OP_INSTANCE_CREATE"
|
229 | 45ead074 | Christos Stavrakakis | vm = get_locked_server(db_server.id) |
230 | 75dc539e | Christos Stavrakakis | backend_mod.process_op_status( |
231 | 45ead074 | Christos Stavrakakis | vm=vm, |
232 | 75dc539e | Christos Stavrakakis | etime=self.event_time,
|
233 | 75dc539e | Christos Stavrakakis | jobid=-0,
|
234 | 75dc539e | Christos Stavrakakis | opcode=fix_opcode, status='error',
|
235 | 75dc539e | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
236 | 75dc539e | Christos Stavrakakis | self.log.debug("Simulated Ganeti error build event for" |
237 | 75dc539e | Christos Stavrakakis | " server '%s'", db_server.id)
|
238 | 75dc539e | Christos Stavrakakis | |
239 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_operstate(self, server_id, db_server, gnt_server): |
240 | 75dc539e | Christos Stavrakakis | if db_server.operstate != gnt_server["state"]: |
241 | 75dc539e | Christos Stavrakakis | self.log.info("Server '%s' is '%s' in DB and '%s' in Ganeti.", |
242 | 75dc539e | Christos Stavrakakis | server_id, db_server.operstate, gnt_server["state"])
|
243 | 75dc539e | Christos Stavrakakis | if self.options["fix_unsynced"]: |
244 | 45ead074 | Christos Stavrakakis | vm = get_locked_server(server_id) |
245 | 5d3e597a | Christos Stavrakakis | # If server is in building state, you will have first to
|
246 | 5d3e597a | Christos Stavrakakis | # reconcile it's creation, to avoid wrong quotas
|
247 | 5d3e597a | Christos Stavrakakis | if db_server.operstate == "BUILD": |
248 | 5d3e597a | Christos Stavrakakis | backend_mod.process_op_status( |
249 | 45ead074 | Christos Stavrakakis | vm=vm, etime=self.event_time, jobid=-0, |
250 | 5d3e597a | Christos Stavrakakis | opcode="OP_INSTANCE_CREATE", status='success', |
251 | 5d3e597a | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
252 | 5d3e597a | Christos Stavrakakis | fix_opcode = "OP_INSTANCE_STARTUP"\
|
253 | 5d3e597a | Christos Stavrakakis | if gnt_server["state"] == "STARTED"\ |
254 | 75dc539e | Christos Stavrakakis | else "OP_INSTANCE_SHUTDOWN" |
255 | 75dc539e | Christos Stavrakakis | backend_mod.process_op_status( |
256 | 45ead074 | Christos Stavrakakis | vm=vm, etime=self.event_time, jobid=-0, |
257 | 75dc539e | Christos Stavrakakis | opcode=fix_opcode, status='success',
|
258 | 75dc539e | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
259 | 75dc539e | Christos Stavrakakis | self.log.debug("Simulated Ganeti state event for server '%s'", |
260 | 75dc539e | Christos Stavrakakis | server_id) |
261 | 75dc539e | Christos Stavrakakis | |
262 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_flavor(self, server_id, db_server, gnt_server): |
263 | 75dc539e | Christos Stavrakakis | db_flavor = db_server.flavor |
264 | 75dc539e | Christos Stavrakakis | gnt_flavor = gnt_server["flavor"]
|
265 | 75dc539e | Christos Stavrakakis | if (db_flavor.ram != gnt_flavor["ram"] or |
266 | 5d3e597a | Christos Stavrakakis | db_flavor.cpu != gnt_flavor["vcpus"]):
|
267 | a67419d8 | Christos Stavrakakis | try:
|
268 | a67419d8 | Christos Stavrakakis | gnt_flavor = Flavor.objects.get( |
269 | 75dc539e | Christos Stavrakakis | ram=gnt_flavor["ram"],
|
270 | 75dc539e | Christos Stavrakakis | cpu=gnt_flavor["vcpus"],
|
271 | 75dc539e | Christos Stavrakakis | disk=db_flavor.disk, |
272 | 75dc539e | Christos Stavrakakis | disk_template=db_flavor.disk_template) |
273 | a67419d8 | Christos Stavrakakis | except Flavor.DoesNotExist:
|
274 | 75dc539e | Christos Stavrakakis | self.log.warning("Server '%s' has unknown flavor.", server_id) |
275 | 75dc539e | Christos Stavrakakis | return
|
276 | 75dc539e | Christos Stavrakakis | |
277 | 4111c53e | Christos Stavrakakis | self.log.info("Server '%s' has flavor '%s' in DB and '%s' in" |
278 | 75dc539e | Christos Stavrakakis | " Ganeti", server_id, db_flavor, gnt_flavor)
|
279 | 75dc539e | Christos Stavrakakis | if self.options["fix_unsynced_flavors"]: |
280 | 45ead074 | Christos Stavrakakis | vm = get_locked_server(server_id) |
281 | 45ead074 | Christos Stavrakakis | old_state = vm.operstate |
282 | 75dc539e | Christos Stavrakakis | opcode = "OP_INSTANCE_SET_PARAMS"
|
283 | 75dc539e | Christos Stavrakakis | beparams = {"vcpus": gnt_flavor.cpu,
|
284 | 75dc539e | Christos Stavrakakis | "minmem": gnt_flavor.ram,
|
285 | 75dc539e | Christos Stavrakakis | "maxmem": gnt_flavor.ram}
|
286 | 75dc539e | Christos Stavrakakis | backend_mod.process_op_status( |
287 | 45ead074 | Christos Stavrakakis | vm=vm, etime=self.event_time, jobid=-0, |
288 | 75dc539e | Christos Stavrakakis | opcode=opcode, status='success',
|
289 | e6fbada1 | Christos Stavrakakis | job_fields={"beparams": beparams},
|
290 | 75dc539e | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
291 | 75dc539e | Christos Stavrakakis | # process_op_status with beparams will set the vmstate to
|
292 | 75dc539e | Christos Stavrakakis | # shutdown. Fix this be returning it to old state
|
293 | 75dc539e | Christos Stavrakakis | vm = VirtualMachine.objects.get(pk=server_id) |
294 | 75dc539e | Christos Stavrakakis | vm.operstate = old_state |
295 | 75dc539e | Christos Stavrakakis | vm.save() |
296 | 75dc539e | Christos Stavrakakis | self.log.debug("Simulated Ganeti flavor event for server '%s'", |
297 | 75dc539e | Christos Stavrakakis | server_id) |
298 | 75dc539e | Christos Stavrakakis | |
299 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_nics(self, server_id, db_server, gnt_server): |
300 | 1cb7846c | Christos Stavrakakis | building_time = self.event_time - BUILDING_NIC_TIMEOUT
|
301 | 3a6be177 | Christos Stavrakakis | db_nics = db_server.nics.exclude(state="BUILD",
|
302 | d7ff7f5a | Christos Stavrakakis | created__lte=building_time) \ |
303 | 0d069390 | Christos Stavrakakis | .order_by("id")
|
304 | 75dc539e | Christos Stavrakakis | gnt_nics = gnt_server["nics"]
|
305 | 75dc539e | Christos Stavrakakis | gnt_nics_parsed = backend_mod.process_ganeti_nics(gnt_nics) |
306 | a1baa42b | Christos Stavrakakis | nics_changed = len(db_nics) != len(gnt_nics) |
307 | a1baa42b | Christos Stavrakakis | for db_nic, gnt_nic in zip(db_nics, sorted(gnt_nics_parsed.items())): |
308 | a1baa42b | Christos Stavrakakis | gnt_nic_id, gnt_nic = gnt_nic |
309 | a1baa42b | Christos Stavrakakis | if (db_nic.id == gnt_nic_id) and\ |
310 | a1baa42b | Christos Stavrakakis | backend_mod.nics_are_equal(db_nic, gnt_nic): |
311 | a1baa42b | Christos Stavrakakis | continue
|
312 | a1baa42b | Christos Stavrakakis | else:
|
313 | a1baa42b | Christos Stavrakakis | nics_changed = True
|
314 | a1baa42b | Christos Stavrakakis | break
|
315 | a1baa42b | Christos Stavrakakis | if nics_changed:
|
316 | a1baa42b | Christos Stavrakakis | msg = "Found unsynced NICs for server '%s'.\n"\
|
317 | a1baa42b | Christos Stavrakakis | "\tDB:\n\t\t%s\n\tGaneti:\n\t\t%s"
|
318 | a1baa42b | Christos Stavrakakis | db_nics_str = "\n\t\t".join(map(format_db_nic, db_nics)) |
319 | a1baa42b | Christos Stavrakakis | gnt_nics_str = "\n\t\t".join(map(format_gnt_nic, |
320 | 0d069390 | Christos Stavrakakis | sorted(gnt_nics_parsed.items())))
|
321 | 75dc539e | Christos Stavrakakis | self.log.info(msg, server_id, db_nics_str, gnt_nics_str)
|
322 | 75dc539e | Christos Stavrakakis | if self.options["fix_unsynced_nics"]: |
323 | 45ead074 | Christos Stavrakakis | vm = get_locked_server(server_id) |
324 | 45ead074 | Christos Stavrakakis | backend_mod.process_net_status(vm=vm, |
325 | 75dc539e | Christos Stavrakakis | etime=self.event_time,
|
326 | 75dc539e | Christos Stavrakakis | nics=gnt_nics) |
327 | 75dc539e | Christos Stavrakakis | |
328 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_disks(self, server_id, db_server, gnt_server): |
329 | 75dc539e | Christos Stavrakakis | pass
|
330 | 75dc539e | Christos Stavrakakis | |
331 | 63f9eb8e | Christos Stavrakakis | def reconcile_pending_task(self, server_id, db_server): |
332 | 63f9eb8e | Christos Stavrakakis | job_id = db_server.task_job_id |
333 | 63f9eb8e | Christos Stavrakakis | pending_task = False
|
334 | 63f9eb8e | Christos Stavrakakis | if job_id not in self.gnt_jobs: |
335 | 63f9eb8e | Christos Stavrakakis | pending_task = True
|
336 | 63f9eb8e | Christos Stavrakakis | else:
|
337 | ce55f724 | Christos Stavrakakis | gnt_job_status = self.gnt_jobs[job_id]["status"] |
338 | edbc1d5a | Christos Stavrakakis | if gnt_job_status in rapi.JOB_STATUS_FINALIZED: |
339 | 63f9eb8e | Christos Stavrakakis | pending_task = True
|
340 | 63f9eb8e | Christos Stavrakakis | |
341 | 63f9eb8e | Christos Stavrakakis | if pending_task:
|
342 | 45ead074 | Christos Stavrakakis | db_server = get_locked_server(server_id) |
343 | 45ead074 | Christos Stavrakakis | if db_server.task_job_id != job_id:
|
344 | 45ead074 | Christos Stavrakakis | # task has changed!
|
345 | 45ead074 | Christos Stavrakakis | return
|
346 | 63f9eb8e | Christos Stavrakakis | self.log.info("Found server '%s' with pending task: '%s'", |
347 | 63f9eb8e | Christos Stavrakakis | server_id, db_server.task) |
348 | 160c81a1 | Christos Stavrakakis | if self.options["fix_pending_tasks"]: |
349 | 63f9eb8e | Christos Stavrakakis | db_server.task = None
|
350 | 63f9eb8e | Christos Stavrakakis | db_server.task_job_id = None
|
351 | 63f9eb8e | Christos Stavrakakis | db_server.save() |
352 | 63f9eb8e | Christos Stavrakakis | self.log.info("Cleared pending task for server '%s", server_id) |
353 | 63f9eb8e | Christos Stavrakakis | |
354 | 75dc539e | Christos Stavrakakis | |
355 | bfd04b01 | Christos Stavrakakis | NIC_MSG = ": %s\t".join(["ID", "State", "IP", "Network", "MAC", "Index", |
356 | bfd04b01 | Christos Stavrakakis | "Firewall"]) + ": %s" |
357 | a1baa42b | Christos Stavrakakis | |
358 | a1baa42b | Christos Stavrakakis | |
359 | 75dc539e | Christos Stavrakakis | def format_db_nic(nic): |
360 | 8764d304 | Christos Stavrakakis | return NIC_MSG % (nic.id, nic.state, nic.ipv4_address, nic.network_id,
|
361 | 8764d304 | Christos Stavrakakis | nic.mac, nic.index, nic.firewall_profile) |
362 | 75dc539e | Christos Stavrakakis | |
363 | 75dc539e | Christos Stavrakakis | |
364 | 75dc539e | Christos Stavrakakis | def format_gnt_nic(nic): |
365 | a1baa42b | Christos Stavrakakis | nic_name, nic = nic |
366 | 8764d304 | Christos Stavrakakis | return NIC_MSG % (nic_name, nic["state"], nic["ipv4_address"], |
367 | 8764d304 | Christos Stavrakakis | nic["network"].id, nic["mac"], nic["index"], |
368 | 8764d304 | Christos Stavrakakis | nic["firewall_profile"])
|
369 | 9fea53cc | Vangelis Koukis | |
370 | cc92b70f | Christos Stavrakakis | |
371 | 0e9a423f | Christos Stavrakakis | #
|
372 | 0e9a423f | Christos Stavrakakis | # Networks
|
373 | 0e9a423f | Christos Stavrakakis | #
|
374 | 3524241a | Christos Stavrakakis | |
375 | 3524241a | Christos Stavrakakis | |
376 | 0e9a423f | Christos Stavrakakis | def get_networks_from_ganeti(backend): |
377 | 44e2c577 | Christos Stavrakakis | prefix = settings.BACKEND_PREFIX_ID + 'net-'
|
378 | 0e9a423f | Christos Stavrakakis | |
379 | 0e9a423f | Christos Stavrakakis | networks = {} |
380 | 3524241a | Christos Stavrakakis | with pooled_rapi_client(backend) as c: |
381 | 3524241a | Christos Stavrakakis | for net in c.GetNetworks(bulk=True): |
382 | 3524241a | Christos Stavrakakis | if net['name'].startswith(prefix): |
383 | 3524241a | Christos Stavrakakis | id = utils.id_from_network_name(net['name'])
|
384 | 3524241a | Christos Stavrakakis | networks[id] = net
|
385 | 0e9a423f | Christos Stavrakakis | |
386 | 0e9a423f | Christos Stavrakakis | return networks
|
387 | 0e9a423f | Christos Stavrakakis | |
388 | 0e9a423f | Christos Stavrakakis | |
389 | 0e9a423f | Christos Stavrakakis | def hanging_networks(backend, GNets): |
390 | 0e9a423f | Christos Stavrakakis | """Get networks that are not connected to all Nodegroups.
|
391 | 0e9a423f | Christos Stavrakakis |
|
392 | 0e9a423f | Christos Stavrakakis | """
|
393 | 0e9a423f | Christos Stavrakakis | def get_network_groups(group_list): |
394 | 0e9a423f | Christos Stavrakakis | groups = set()
|
395 | a9bb2a1a | Christos Stavrakakis | for (name, mode, link) in group_list: |
396 | a9bb2a1a | Christos Stavrakakis | groups.add(name) |
397 | 0e9a423f | Christos Stavrakakis | return groups
|
398 | 0e9a423f | Christos Stavrakakis | |
399 | 3524241a | Christos Stavrakakis | with pooled_rapi_client(backend) as c: |
400 | 3524241a | Christos Stavrakakis | groups = set(c.GetGroups())
|
401 | 0e9a423f | Christos Stavrakakis | |
402 | 0e9a423f | Christos Stavrakakis | hanging = {} |
403 | 0e9a423f | Christos Stavrakakis | for id, info in GNets.items(): |
404 | 0e9a423f | Christos Stavrakakis | group_list = get_network_groups(info['group_list'])
|
405 | 0e9a423f | Christos Stavrakakis | if group_list != groups:
|
406 | 0e9a423f | Christos Stavrakakis | hanging[id] = groups - group_list
|
407 | 0e9a423f | Christos Stavrakakis | return hanging
|
408 | 0e9a423f | Christos Stavrakakis | |
409 | 9fea53cc | Vangelis Koukis | |
410 | 75dc539e | Christos Stavrakakis | def get_online_backends(): |
411 | 75dc539e | Christos Stavrakakis | return Backend.objects.filter(offline=False) |
412 | 75dc539e | Christos Stavrakakis | |
413 | 75dc539e | Christos Stavrakakis | |
414 | 75dc539e | Christos Stavrakakis | def get_database_servers(backend): |
415 | 3278725f | Christos Stavrakakis | servers = backend.virtual_machines.select_related("flavor")\
|
416 | 3278725f | Christos Stavrakakis | .prefetch_related("nics__ips__subnet")\
|
417 | 75dc539e | Christos Stavrakakis | .filter(deleted=False)
|
418 | 75dc539e | Christos Stavrakakis | return dict([(s.id, s) for s in servers]) |
419 | 75dc539e | Christos Stavrakakis | |
420 | 75dc539e | Christos Stavrakakis | |
421 | 75dc539e | Christos Stavrakakis | def get_ganeti_servers(backend): |
422 | 75dc539e | Christos Stavrakakis | gnt_instances = backend_mod.get_instances(backend) |
423 | 75dc539e | Christos Stavrakakis | # Filter out non-synnefo instances
|
424 | 75dc539e | Christos Stavrakakis | snf_backend_prefix = settings.BACKEND_PREFIX_ID |
425 | 75dc539e | Christos Stavrakakis | gnt_instances = filter(lambda i: i["name"].startswith(snf_backend_prefix), |
426 | 75dc539e | Christos Stavrakakis | gnt_instances) |
427 | 75dc539e | Christos Stavrakakis | gnt_instances = map(parse_gnt_instance, gnt_instances)
|
428 | 75dc539e | Christos Stavrakakis | return dict([(i["id"], i) for i in gnt_instances if i["id"] is not None]) |
429 | 75dc539e | Christos Stavrakakis | |
430 | 75dc539e | Christos Stavrakakis | |
431 | 75dc539e | Christos Stavrakakis | def parse_gnt_instance(instance): |
432 | 75dc539e | Christos Stavrakakis | try:
|
433 | 75dc539e | Christos Stavrakakis | instance_id = utils.id_from_instance_name(instance['name'])
|
434 | 75dc539e | Christos Stavrakakis | except Exception: |
435 | 75dc539e | Christos Stavrakakis | logger.error("Ignoring instance with malformed name %s",
|
436 | 75dc539e | Christos Stavrakakis | instance['name'])
|
437 | 75dc539e | Christos Stavrakakis | return (None, None) |
438 | 75dc539e | Christos Stavrakakis | |
439 | 75dc539e | Christos Stavrakakis | beparams = instance["beparams"]
|
440 | 75dc539e | Christos Stavrakakis | |
441 | 75dc539e | Christos Stavrakakis | vcpus = beparams["vcpus"]
|
442 | 75dc539e | Christos Stavrakakis | ram = beparams["maxmem"]
|
443 | 75dc539e | Christos Stavrakakis | state = instance["oper_state"] and "STARTED" or "STOPPED" |
444 | 75dc539e | Christos Stavrakakis | |
445 | 75dc539e | Christos Stavrakakis | return {
|
446 | 75dc539e | Christos Stavrakakis | "id": instance_id,
|
447 | 75dc539e | Christos Stavrakakis | "state": state, # FIX |
448 | 75dc539e | Christos Stavrakakis | "updated": datetime.fromtimestamp(instance["mtime"]), |
449 | 75dc539e | Christos Stavrakakis | "disks": disks_from_instance(instance),
|
450 | 75dc539e | Christos Stavrakakis | "nics": nics_from_instance(instance),
|
451 | 75dc539e | Christos Stavrakakis | "flavor": {"vcpus": vcpus, |
452 | 75dc539e | Christos Stavrakakis | "ram": ram},
|
453 | 75dc539e | Christos Stavrakakis | "tags": instance["tags"] |
454 | 75dc539e | Christos Stavrakakis | } |
455 | 75dc539e | Christos Stavrakakis | |
456 | 75dc539e | Christos Stavrakakis | |
457 | 75dc539e | Christos Stavrakakis | def nics_from_instance(i): |
458 | 75dc539e | Christos Stavrakakis | ips = zip(itertools.repeat('ip'), i['nic.ips']) |
459 | a1baa42b | Christos Stavrakakis | names = zip(itertools.repeat('name'), i['nic.names']) |
460 | 75dc539e | Christos Stavrakakis | macs = zip(itertools.repeat('mac'), i['nic.macs']) |
461 | a1baa42b | Christos Stavrakakis | networks = zip(itertools.repeat('network'), i['nic.networks.names']) |
462 | 75dc539e | Christos Stavrakakis | # modes = zip(itertools.repeat('mode'), i['nic.modes'])
|
463 | 75dc539e | Christos Stavrakakis | # links = zip(itertools.repeat('link'), i['nic.links'])
|
464 | 75dc539e | Christos Stavrakakis | # nics = zip(ips,macs,modes,networks,links)
|
465 | a1baa42b | Christos Stavrakakis | nics = zip(ips, names, macs, networks)
|
466 | 75dc539e | Christos Stavrakakis | nics = map(lambda x: dict(x), nics) |
467 | 75dc539e | Christos Stavrakakis | #nics = dict(enumerate(nics))
|
468 | 75dc539e | Christos Stavrakakis | tags = i["tags"]
|
469 | 75dc539e | Christos Stavrakakis | for tag in tags: |
470 | 75dc539e | Christos Stavrakakis | t = tag.split(":")
|
471 | 75dc539e | Christos Stavrakakis | if t[0:2] == ["synnefo", "network"]: |
472 | 75dc539e | Christos Stavrakakis | if len(t) != 4: |
473 | 75dc539e | Christos Stavrakakis | logger.error("Malformed synefo tag %s", tag)
|
474 | 75dc539e | Christos Stavrakakis | continue
|
475 | d0545590 | Christos Stavrakakis | nic_name = t[2]
|
476 | d0545590 | Christos Stavrakakis | firewall = t[3]
|
477 | d0545590 | Christos Stavrakakis | [nic.setdefault("firewall", firewall)
|
478 | d0545590 | Christos Stavrakakis | for nic in nics if nic["name"] == nic_name] |
479 | 75dc539e | Christos Stavrakakis | return nics
|
480 | 9fea53cc | Vangelis Koukis | |
481 | 9fea53cc | Vangelis Koukis | |
482 | 63f9eb8e | Christos Stavrakakis | def get_ganeti_jobs(backend): |
483 | 63f9eb8e | Christos Stavrakakis | gnt_jobs = backend_mod.get_jobs(backend) |
484 | 63f9eb8e | Christos Stavrakakis | return dict([(int(j["id"]), j) for j in gnt_jobs]) |
485 | 63f9eb8e | Christos Stavrakakis | |
486 | 63f9eb8e | Christos Stavrakakis | |
487 | 75dc539e | Christos Stavrakakis | def disks_from_instance(i): |
488 | 75dc539e | Christos Stavrakakis | return dict([(index, {"size": size}) |
489 | 75dc539e | Christos Stavrakakis | for index, size in enumerate(i["disk.sizes"])]) |
490 | 89b2b908 | Christos Stavrakakis | |
491 | 89b2b908 | Christos Stavrakakis | |
492 | 89b2b908 | Christos Stavrakakis | class NetworkReconciler(object): |
493 | 0ccb6461 | Christos Stavrakakis | def __init__(self, logger, fix=False): |
494 | 89b2b908 | Christos Stavrakakis | self.log = logger
|
495 | 89b2b908 | Christos Stavrakakis | self.fix = fix
|
496 | 89b2b908 | Christos Stavrakakis | |
497 | 89b2b908 | Christos Stavrakakis | @transaction.commit_on_success
|
498 | 89b2b908 | Christos Stavrakakis | def reconcile_networks(self): |
499 | 89b2b908 | Christos Stavrakakis | # Get models from DB
|
500 | 0ccb6461 | Christos Stavrakakis | self.backends = Backend.objects.exclude(offline=True) |
501 | 0ccb6461 | Christos Stavrakakis | self.networks = Network.objects.filter(deleted=False) |
502 | 89b2b908 | Christos Stavrakakis | |
503 | 89b2b908 | Christos Stavrakakis | self.event_time = datetime.now()
|
504 | 89b2b908 | Christos Stavrakakis | |
505 | 89b2b908 | Christos Stavrakakis | # Get info from all ganeti backends
|
506 | 0ccb6461 | Christos Stavrakakis | self.ganeti_networks = {}
|
507 | 0ccb6461 | Christos Stavrakakis | self.ganeti_hanging_networks = {}
|
508 | 0ccb6461 | Christos Stavrakakis | for b in self.backends: |
509 | 89b2b908 | Christos Stavrakakis | g_nets = get_networks_from_ganeti(b) |
510 | 0ccb6461 | Christos Stavrakakis | self.ganeti_networks[b] = g_nets
|
511 | 89b2b908 | Christos Stavrakakis | g_hanging_nets = hanging_networks(b, g_nets) |
512 | 0ccb6461 | Christos Stavrakakis | self.ganeti_hanging_networks[b] = g_hanging_nets
|
513 | 89b2b908 | Christos Stavrakakis | |
514 | 0ccb6461 | Christos Stavrakakis | self._reconcile_orphan_networks()
|
515 | 89b2b908 | Christos Stavrakakis | |
516 | 0ccb6461 | Christos Stavrakakis | for network in self.networks: |
517 | 0ccb6461 | Christos Stavrakakis | self._reconcile_network(network)
|
518 | 89b2b908 | Christos Stavrakakis | |
519 | 0ccb6461 | Christos Stavrakakis | @transaction.commit_on_success
|
520 | 0ccb6461 | Christos Stavrakakis | def _reconcile_network(self, network): |
521 | 0ccb6461 | Christos Stavrakakis | """Reconcile a network with corresponging Ganeti networks.
|
522 | 0ccb6461 | Christos Stavrakakis |
|
523 | 0ccb6461 | Christos Stavrakakis | Reconcile a Network and the associated BackendNetworks with the
|
524 | 0ccb6461 | Christos Stavrakakis | corresponding Ganeti networks in all Ganeti backends.
|
525 | 0ccb6461 | Christos Stavrakakis |
|
526 | 0ccb6461 | Christos Stavrakakis | """
|
527 | 4cbd934b | Christos Stavrakakis | if network.subnets.filter(ipversion=4, dhcp=True).exists(): |
528 | 4cbd934b | Christos Stavrakakis | ip_pools = network.get_ip_pools() # X-Lock on IP pools
|
529 | 4cbd934b | Christos Stavrakakis | else:
|
530 | 4cbd934b | Christos Stavrakakis | ip_pools = None
|
531 | 0ccb6461 | Christos Stavrakakis | for bend in self.backends: |
532 | 0ccb6461 | Christos Stavrakakis | bnet = get_backend_network(network, bend) |
533 | 0ccb6461 | Christos Stavrakakis | gnet = self.ganeti_networks[bend].get(network.id)
|
534 | 84a0469b | Christos Stavrakakis | if bnet is None and gnet is not None: |
535 | 84a0469b | Christos Stavrakakis | # Network exists in backend but not in DB for this backend
|
536 | 84a0469b | Christos Stavrakakis | bnet = self.reconcile_parted_network(network, bend)
|
537 | 84a0469b | Christos Stavrakakis | |
538 | 84a0469b | Christos Stavrakakis | if bnet is None: |
539 | 84a0469b | Christos Stavrakakis | continue
|
540 | 84a0469b | Christos Stavrakakis | |
541 | 84a0469b | Christos Stavrakakis | if gnet is None: |
542 | 0ccb6461 | Christos Stavrakakis | # Network does not exist in Ganeti. If the network action
|
543 | 0ccb6461 | Christos Stavrakakis | # is DESTROY, we have to mark as deleted in DB, else we
|
544 | 0ccb6461 | Christos Stavrakakis | # have to create it in Ganeti.
|
545 | 0ccb6461 | Christos Stavrakakis | if network.action == "DESTROY": |
546 | 0ccb6461 | Christos Stavrakakis | if bnet.operstate != "DELETED": |
547 | 0ccb6461 | Christos Stavrakakis | self.reconcile_stale_network(bnet)
|
548 | 0ccb6461 | Christos Stavrakakis | else:
|
549 | 0ccb6461 | Christos Stavrakakis | self.reconcile_missing_network(network, bend)
|
550 | 0ccb6461 | Christos Stavrakakis | # Skip rest reconciliation!
|
551 | 0ccb6461 | Christos Stavrakakis | continue
|
552 | 89b2b908 | Christos Stavrakakis | |
553 | 0ccb6461 | Christos Stavrakakis | try:
|
554 | 0ccb6461 | Christos Stavrakakis | hanging_groups = self.ganeti_hanging_networks[bend][network.id]
|
555 | 0ccb6461 | Christos Stavrakakis | except KeyError: |
556 | 0ccb6461 | Christos Stavrakakis | # Network is connected to all nodegroups
|
557 | 0ccb6461 | Christos Stavrakakis | hanging_groups = [] |
558 | 0ccb6461 | Christos Stavrakakis | |
559 | 0ccb6461 | Christos Stavrakakis | if hanging_groups:
|
560 | 0ccb6461 | Christos Stavrakakis | # CASE-3: Ganeti networks not connected to all nodegroups
|
561 | 0ccb6461 | Christos Stavrakakis | self.reconcile_hanging_groups(network, bend,
|
562 | 0ccb6461 | Christos Stavrakakis | hanging_groups) |
563 | 0ccb6461 | Christos Stavrakakis | continue
|
564 | 89b2b908 | Christos Stavrakakis | |
565 | 0ccb6461 | Christos Stavrakakis | if bnet.operstate != 'ACTIVE': |
566 | 0ccb6461 | Christos Stavrakakis | # CASE-4: Unsynced network state. At this point the network
|
567 | 0ccb6461 | Christos Stavrakakis | # exists and is connected to all nodes so is must be
|
568 | 0ccb6461 | Christos Stavrakakis | # active!
|
569 | 0ccb6461 | Christos Stavrakakis | self.reconcile_unsynced_network(network, bend, bnet)
|
570 | 0ccb6461 | Christos Stavrakakis | |
571 | 0ccb6461 | Christos Stavrakakis | # Check that externally reserved IPs of the network in Ganeti are
|
572 | 0ccb6461 | Christos Stavrakakis | # also externally reserved to the IP pool
|
573 | 0ccb6461 | Christos Stavrakakis | externally_reserved = gnet['external_reservations']
|
574 | 4cbd934b | Christos Stavrakakis | if externally_reserved and ip_pools is not None: |
575 | 98a01362 | Christos Stavrakakis | for ip in externally_reserved.split(","): |
576 | 98a01362 | Christos Stavrakakis | ip = ip.strip() |
577 | 0e0a6ef9 | Christos Stavrakakis | for ip_pool in ip_pools: |
578 | 0e0a6ef9 | Christos Stavrakakis | if ip_pool.contains(ip):
|
579 | 0e0a6ef9 | Christos Stavrakakis | if not ip_pool.is_reserved(ip): |
580 | 0e0a6ef9 | Christos Stavrakakis | msg = ("D: IP '%s' is reserved for network"
|
581 | 0e0a6ef9 | Christos Stavrakakis | " '%s' in backend '%s' but not in DB.")
|
582 | 0e0a6ef9 | Christos Stavrakakis | self.log.info(msg, ip, network, bend)
|
583 | 0e0a6ef9 | Christos Stavrakakis | if self.fix: |
584 | 0e0a6ef9 | Christos Stavrakakis | ip_pool.reserve(ip, external=True)
|
585 | 0e0a6ef9 | Christos Stavrakakis | ip_pool.save() |
586 | 0e0a6ef9 | Christos Stavrakakis | self.log.info("F: Reserved IP '%s'", ip) |
587 | ec4e0acc | Christos Stavrakakis | if network.state != "ACTIVE": |
588 | ec4e0acc | Christos Stavrakakis | network = Network.objects.select_for_update().get(id=network.id) |
589 | ec4e0acc | Christos Stavrakakis | backend_mod.update_network_state(network) |
590 | 89b2b908 | Christos Stavrakakis | |
591 | 89b2b908 | Christos Stavrakakis | def reconcile_parted_network(self, network, backend): |
592 | 89b2b908 | Christos Stavrakakis | self.log.info("D: Missing DB entry for network %s in backend %s", |
593 | 89b2b908 | Christos Stavrakakis | network, backend) |
594 | 89b2b908 | Christos Stavrakakis | if self.fix: |
595 | 89b2b908 | Christos Stavrakakis | network.create_backend_network(backend) |
596 | 89b2b908 | Christos Stavrakakis | self.log.info("F: Created DB entry") |
597 | 89b2b908 | Christos Stavrakakis | bnet = get_backend_network(network, backend) |
598 | 89b2b908 | Christos Stavrakakis | return bnet
|
599 | 89b2b908 | Christos Stavrakakis | |
600 | 89b2b908 | Christos Stavrakakis | def reconcile_stale_network(self, backend_network): |
601 | 89b2b908 | Christos Stavrakakis | self.log.info("D: Stale DB entry for network %s in backend %s", |
602 | 89b2b908 | Christos Stavrakakis | backend_network.network, backend_network.backend) |
603 | 89b2b908 | Christos Stavrakakis | if self.fix: |
604 | 45ead074 | Christos Stavrakakis | backend_network = BackendNetwork.objects.select_for_update()\ |
605 | 45ead074 | Christos Stavrakakis | .get(id=backend_network.id) |
606 | 89b2b908 | Christos Stavrakakis | backend_mod.process_network_status( |
607 | 89b2b908 | Christos Stavrakakis | backend_network, self.event_time, 0, |
608 | 89b2b908 | Christos Stavrakakis | "OP_NETWORK_REMOVE",
|
609 | 89b2b908 | Christos Stavrakakis | "success",
|
610 | 89b2b908 | Christos Stavrakakis | "Reconciliation simulated event")
|
611 | 89b2b908 | Christos Stavrakakis | self.log.info("F: Reconciled event: OP_NETWORK_REMOVE") |
612 | 89b2b908 | Christos Stavrakakis | |
613 | 89b2b908 | Christos Stavrakakis | def reconcile_missing_network(self, network, backend): |
614 | 89b2b908 | Christos Stavrakakis | self.log.info("D: Missing Ganeti network %s in backend %s", |
615 | 89b2b908 | Christos Stavrakakis | network, backend) |
616 | 89b2b908 | Christos Stavrakakis | if self.fix: |
617 | 89b2b908 | Christos Stavrakakis | backend_mod.create_network(network, backend) |
618 | 89b2b908 | Christos Stavrakakis | self.log.info("F: Issued OP_NETWORK_CONNECT") |
619 | 89b2b908 | Christos Stavrakakis | |
620 | 89b2b908 | Christos Stavrakakis | def reconcile_hanging_groups(self, network, backend, hanging_groups): |
621 | 89b2b908 | Christos Stavrakakis | self.log.info('D: Network %s in backend %s is not connected to ' |
622 | 89b2b908 | Christos Stavrakakis | 'the following groups:', network, backend)
|
623 | 89b2b908 | Christos Stavrakakis | self.log.info('- ' + '\n- '.join(hanging_groups)) |
624 | 89b2b908 | Christos Stavrakakis | if self.fix: |
625 | 89b2b908 | Christos Stavrakakis | for group in hanging_groups: |
626 | 89b2b908 | Christos Stavrakakis | self.log.info('F: Connecting network %s to nodegroup %s', |
627 | 89b2b908 | Christos Stavrakakis | network, group) |
628 | 89b2b908 | Christos Stavrakakis | backend_mod.connect_network(network, backend, depends=[], |
629 | 89b2b908 | Christos Stavrakakis | group=group) |
630 | 89b2b908 | Christos Stavrakakis | |
631 | 89b2b908 | Christos Stavrakakis | def reconcile_unsynced_network(self, network, backend, backend_network): |
632 | 89b2b908 | Christos Stavrakakis | self.log.info("D: Unsynced network %s in backend %s", network, backend) |
633 | 89b2b908 | Christos Stavrakakis | if self.fix: |
634 | 89b2b908 | Christos Stavrakakis | self.log.info("F: Issuing OP_NETWORK_CONNECT") |
635 | 45ead074 | Christos Stavrakakis | backend_network = BackendNetwork.objects.select_for_update()\ |
636 | 45ead074 | Christos Stavrakakis | .get(id=backend_network.id) |
637 | 89b2b908 | Christos Stavrakakis | backend_mod.process_network_status( |
638 | 89b2b908 | Christos Stavrakakis | backend_network, self.event_time, 0, |
639 | 89b2b908 | Christos Stavrakakis | "OP_NETWORK_CONNECT",
|
640 | 89b2b908 | Christos Stavrakakis | "success",
|
641 | 89b2b908 | Christos Stavrakakis | "Reconciliation simulated eventd")
|
642 | 89b2b908 | Christos Stavrakakis | |
643 | 0ccb6461 | Christos Stavrakakis | def _reconcile_orphan_networks(self): |
644 | 0ccb6461 | Christos Stavrakakis | db_networks = self.networks
|
645 | 0ccb6461 | Christos Stavrakakis | ganeti_networks = self.ganeti_networks
|
646 | 89b2b908 | Christos Stavrakakis | # Detect Orphan Networks in Ganeti
|
647 | 89b2b908 | Christos Stavrakakis | db_network_ids = set([net.id for net in db_networks]) |
648 | 89b2b908 | Christos Stavrakakis | for back_end, ganeti_networks in ganeti_networks.items(): |
649 | 89b2b908 | Christos Stavrakakis | ganeti_network_ids = set(ganeti_networks.keys())
|
650 | 89b2b908 | Christos Stavrakakis | orphans = ganeti_network_ids - db_network_ids |
651 | 89b2b908 | Christos Stavrakakis | |
652 | 89b2b908 | Christos Stavrakakis | if len(orphans) > 0: |
653 | 89b2b908 | Christos Stavrakakis | self.log.info('D: Orphan Networks in backend %s:', |
654 | 89b2b908 | Christos Stavrakakis | back_end.clustername) |
655 | 89b2b908 | Christos Stavrakakis | self.log.info('- ' + '\n- '.join([str(o) for o in orphans])) |
656 | 89b2b908 | Christos Stavrakakis | if self.fix: |
657 | 89b2b908 | Christos Stavrakakis | for net_id in orphans: |
658 | 89b2b908 | Christos Stavrakakis | self.log.info('Disconnecting and deleting network %d', |
659 | 89b2b908 | Christos Stavrakakis | net_id) |
660 | 89b2b908 | Christos Stavrakakis | try:
|
661 | 89b2b908 | Christos Stavrakakis | network = Network.objects.get(id=net_id) |
662 | 89b2b908 | Christos Stavrakakis | backend_mod.delete_network(network, |
663 | 89b2b908 | Christos Stavrakakis | backend=back_end) |
664 | 89b2b908 | Christos Stavrakakis | except Network.DoesNotExist:
|
665 | 89b2b908 | Christos Stavrakakis | self.log.info("Not entry for network %s in DB !!", |
666 | 89b2b908 | Christos Stavrakakis | net_id) |
667 | 89b2b908 | Christos Stavrakakis | |
668 | 89b2b908 | Christos Stavrakakis | |
669 | 89b2b908 | Christos Stavrakakis | def get_backend_network(network, backend): |
670 | 89b2b908 | Christos Stavrakakis | try:
|
671 | 89b2b908 | Christos Stavrakakis | return BackendNetwork.objects.get(network=network, backend=backend)
|
672 | 89b2b908 | Christos Stavrakakis | except BackendNetwork.DoesNotExist:
|
673 | 89b2b908 | Christos Stavrakakis | return None |
674 | 89b2b908 | Christos Stavrakakis | |
675 | 89b2b908 | Christos Stavrakakis | |
676 | 0ccb6461 | Christos Stavrakakis | class PoolReconciler(object): |
677 | 0ccb6461 | Christos Stavrakakis | def __init__(self, logger, fix=False): |
678 | 0ccb6461 | Christos Stavrakakis | self.log = logger
|
679 | 0ccb6461 | Christos Stavrakakis | self.fix = fix
|
680 | 0ccb6461 | Christos Stavrakakis | |
681 | 0ccb6461 | Christos Stavrakakis | def reconcile(self): |
682 | 0ccb6461 | Christos Stavrakakis | self.reconcile_bridges()
|
683 | 0ccb6461 | Christos Stavrakakis | self.reconcile_mac_prefixes()
|
684 | 4cbd934b | Christos Stavrakakis | |
685 | 4cbd934b | Christos Stavrakakis | networks = Network.objects.prefetch_related("subnets")\
|
686 | 4cbd934b | Christos Stavrakakis | .filter(deleted=False)
|
687 | 4cbd934b | Christos Stavrakakis | for network in networks: |
688 | 4cbd934b | Christos Stavrakakis | for subnet in network.subnets.all(): |
689 | 4cbd934b | Christos Stavrakakis | if subnet.ipversion == 4 and subnet.dhcp: |
690 | 4cbd934b | Christos Stavrakakis | self.reconcile_ip_pool(network)
|
691 | 0ccb6461 | Christos Stavrakakis | |
692 | 0ccb6461 | Christos Stavrakakis | @transaction.commit_on_success
|
693 | 0ccb6461 | Christos Stavrakakis | def reconcile_bridges(self): |
694 | 0ccb6461 | Christos Stavrakakis | networks = Network.objects.filter(deleted=False,
|
695 | 0ccb6461 | Christos Stavrakakis | flavor="PHYSICAL_VLAN")
|
696 | 0ccb6461 | Christos Stavrakakis | check_unique_values(objects=networks, field='link', logger=self.log) |
697 | 0ccb6461 | Christos Stavrakakis | try:
|
698 | 0ccb6461 | Christos Stavrakakis | pool = BridgePoolTable.get_pool() |
699 | 0ccb6461 | Christos Stavrakakis | except pools.EmptyPool:
|
700 | 0ccb6461 | Christos Stavrakakis | self.log.info("There is no available pool for bridges.") |
701 | 0ccb6461 | Christos Stavrakakis | return
|
702 | 0ccb6461 | Christos Stavrakakis | |
703 | ec4e0acc | Christos Stavrakakis | # Since pool is locked, no new networks may be created
|
704 | 0ccb6461 | Christos Stavrakakis | used_bridges = set(networks.values_list('link', flat=True)) |
705 | 0ccb6461 | Christos Stavrakakis | check_pool_consistent(pool=pool, pool_class=pools.BridgePool, |
706 | 0ccb6461 | Christos Stavrakakis | used_values=used_bridges, fix=self.fix,
|
707 | 0ccb6461 | Christos Stavrakakis | logger=self.log)
|
708 | 89b2b908 | Christos Stavrakakis | |
709 | 0ccb6461 | Christos Stavrakakis | @transaction.commit_on_success
|
710 | 0ccb6461 | Christos Stavrakakis | def reconcile_mac_prefixes(self): |
711 | 0ccb6461 | Christos Stavrakakis | networks = Network.objects.filter(deleted=False, flavor="MAC_FILTERED") |
712 | 0ccb6461 | Christos Stavrakakis | check_unique_values(objects=networks, field='mac_prefix',
|
713 | 0ccb6461 | Christos Stavrakakis | logger=self.log)
|
714 | 0ccb6461 | Christos Stavrakakis | try:
|
715 | 0ccb6461 | Christos Stavrakakis | pool = MacPrefixPoolTable.get_pool() |
716 | 0ccb6461 | Christos Stavrakakis | except pools.EmptyPool:
|
717 | 0ccb6461 | Christos Stavrakakis | self.log.info("There is no available pool for MAC prefixes.") |
718 | 0ccb6461 | Christos Stavrakakis | return
|
719 | 0ccb6461 | Christos Stavrakakis | |
720 | ec4e0acc | Christos Stavrakakis | # Since pool is locked, no new network may be created
|
721 | 0ccb6461 | Christos Stavrakakis | used_mac_prefixes = set(networks.values_list('mac_prefix', flat=True)) |
722 | 0ccb6461 | Christos Stavrakakis | check_pool_consistent(pool=pool, pool_class=pools.MacPrefixPool, |
723 | 0ccb6461 | Christos Stavrakakis | used_values=used_mac_prefixes, fix=self.fix,
|
724 | 0ccb6461 | Christos Stavrakakis | logger=self.log)
|
725 | 89b2b908 | Christos Stavrakakis | |
726 | 0ccb6461 | Christos Stavrakakis | @transaction.commit_on_success
|
727 | 0ccb6461 | Christos Stavrakakis | def reconcile_ip_pool(self, network): |
728 | 0ccb6461 | Christos Stavrakakis | # Check that all NICs have unique IPv4 address
|
729 | 4cbd934b | Christos Stavrakakis | nics = network.ips.exclude(address__isnull=True).all()
|
730 | 0e0a6ef9 | Christos Stavrakakis | check_unique_values(objects=nics, field="address", logger=self.log) |
731 | 0e0a6ef9 | Christos Stavrakakis | |
732 | 0e0a6ef9 | Christos Stavrakakis | for ip_pool in network.get_ip_pools(): |
733 | ec4e0acc | Christos Stavrakakis | # IP pool is now locked, so no new IPs may be created
|
734 | 4cbd934b | Christos Stavrakakis | used_ips = ip_pool.pool_table.subnet\ |
735 | 4cbd934b | Christos Stavrakakis | .ips.exclude(address__isnull=True)\
|
736 | c82f57ad | Christos Stavrakakis | .exclude(deleted=True)\
|
737 | 4cbd934b | Christos Stavrakakis | .values_list("address", flat=True) |
738 | 0e0a6ef9 | Christos Stavrakakis | used_ips = filter(lambda x: ip_pool.contains(x), used_ips) |
739 | 0e0a6ef9 | Christos Stavrakakis | check_pool_consistent(pool=ip_pool, |
740 | 0e0a6ef9 | Christos Stavrakakis | pool_class=pools.IPPool, |
741 | 0e0a6ef9 | Christos Stavrakakis | used_values=used_ips, |
742 | 0e0a6ef9 | Christos Stavrakakis | fix=self.fix, logger=self.log) |
743 | 0ccb6461 | Christos Stavrakakis | |
744 | 0ccb6461 | Christos Stavrakakis | |
745 | 0ccb6461 | Christos Stavrakakis | def check_unique_values(objects, field, logger): |
746 | 0ccb6461 | Christos Stavrakakis | used_values = list(objects.values_list(field, flat=True)) |
747 | 0ccb6461 | Christos Stavrakakis | if len(used_values) != len(set(used_values)): |
748 | 0ccb6461 | Christos Stavrakakis | duplicate_values = [v for v in used_values if used_values.count(v) > 1] |
749 | 0ccb6461 | Christos Stavrakakis | for value in duplicate_values: |
750 | 0ccb6461 | Christos Stavrakakis | filter_args = {field: value} |
751 | 0ccb6461 | Christos Stavrakakis | using_objects = objects.filter(**filter_args) |
752 | 0ccb6461 | Christos Stavrakakis | msg = "Value '%s' is used as %s for more than one objects: %s"
|
753 | 0ccb6461 | Christos Stavrakakis | logger.error(msg, value, field, ",".join(map(str, using_objects))) |
754 | 0ccb6461 | Christos Stavrakakis | return False |
755 | 0ccb6461 | Christos Stavrakakis | logger.debug("Values for field '%s' are unique.", field)
|
756 | 0ccb6461 | Christos Stavrakakis | return True |
757 | 0ccb6461 | Christos Stavrakakis | |
758 | 0ccb6461 | Christos Stavrakakis | |
759 | 0ccb6461 | Christos Stavrakakis | def check_pool_consistent(pool, pool_class, used_values, fix, logger): |
760 | 0ccb6461 | Christos Stavrakakis | dummy_pool = create_empty_pool(pool, pool_class) |
761 | 0ccb6461 | Christos Stavrakakis | [dummy_pool.reserve(value) for value in used_values] |
762 | 0ccb6461 | Christos Stavrakakis | if dummy_pool.available != pool.available:
|
763 | 0ccb6461 | Christos Stavrakakis | msg = "'%s' is not consistent!\nPool: %s\nUsed: %s"
|
764 | 0ccb6461 | Christos Stavrakakis | pool_diff = dummy_pool.available ^ pool.available |
765 | 0ccb6461 | Christos Stavrakakis | for index in pool_diff.itersearch(bitarray.bitarray("1")): |
766 | 0ccb6461 | Christos Stavrakakis | value = pool.index_to_value(int(index))
|
767 | 0ccb6461 | Christos Stavrakakis | msg = "%s is incosistent! Value '%s' is %s but should be %s."
|
768 | 0ccb6461 | Christos Stavrakakis | value1 = pool.is_available(value) and "available" or "unavailable" |
769 | 0ccb6461 | Christos Stavrakakis | value2 = dummy_pool.is_available(value) and "available"\ |
770 | 0ccb6461 | Christos Stavrakakis | or "unavailable" |
771 | 0ccb6461 | Christos Stavrakakis | logger.error(msg, pool, value, value1, value2) |
772 | 0ccb6461 | Christos Stavrakakis | if fix:
|
773 | 0ccb6461 | Christos Stavrakakis | pool.available = dummy_pool.available |
774 | 0ccb6461 | Christos Stavrakakis | pool.save() |
775 | 0ccb6461 | Christos Stavrakakis | logger.info("Fixed available map of pool '%s'", pool)
|
776 | 0ccb6461 | Christos Stavrakakis | |
777 | 0ccb6461 | Christos Stavrakakis | |
778 | 0ccb6461 | Christos Stavrakakis | def create_empty_pool(pool, pool_class): |
779 | 0ccb6461 | Christos Stavrakakis | pool_row = pool.pool_table |
780 | 0ccb6461 | Christos Stavrakakis | pool_row.available_map = ""
|
781 | 0ccb6461 | Christos Stavrakakis | pool_row.reserved_map = ""
|
782 | 0ccb6461 | Christos Stavrakakis | return pool_class(pool_row)
|
783 | 45ead074 | Christos Stavrakakis | |
784 | 45ead074 | Christos Stavrakakis | |
785 | 45ead074 | Christos Stavrakakis | def get_locked_server(server_id): |
786 | 45ead074 | Christos Stavrakakis | return VirtualMachine.objects.select_for_update().get(id=server_id) |