root / snf-cyclades-app / synnefo / logic / reconciliation.py @ 75dc539e
History | View | Annotate | Download (16.4 kB)
1 | 9fea53cc | Vangelis Koukis | # -*- coding: utf-8 -*-
|
---|---|---|---|
2 | 9fea53cc | Vangelis Koukis | #
|
3 | 75dc539e | Christos Stavrakakis | # Copyright 2011-2013 GRNET S.A. All rights reserved.
|
4 | 9fea53cc | Vangelis Koukis | #
|
5 | 9fea53cc | Vangelis Koukis | # Redistribution and use in source and binary forms, with or
|
6 | 9fea53cc | Vangelis Koukis | # without modification, are permitted provided that the following
|
7 | 9fea53cc | Vangelis Koukis | # conditions are met:
|
8 | 9fea53cc | Vangelis Koukis | #
|
9 | 9fea53cc | Vangelis Koukis | # 1. Redistributions of source code must retain the above
|
10 | 9fea53cc | Vangelis Koukis | # copyright notice, this list of conditions and the following
|
11 | 9fea53cc | Vangelis Koukis | # disclaimer.
|
12 | 9fea53cc | Vangelis Koukis | #
|
13 | 9fea53cc | Vangelis Koukis | # 2. Redistributions in binary form must reproduce the above
|
14 | 9fea53cc | Vangelis Koukis | # copyright notice, this list of conditions and the following
|
15 | 9fea53cc | Vangelis Koukis | # disclaimer in the documentation and/or other materials
|
16 | 9fea53cc | Vangelis Koukis | # provided with the distribution.
|
17 | 9fea53cc | Vangelis Koukis | #
|
18 | 9fea53cc | Vangelis Koukis | # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
|
19 | 9fea53cc | Vangelis Koukis | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
20 | 9fea53cc | Vangelis Koukis | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
21 | 9fea53cc | Vangelis Koukis | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
|
22 | 9fea53cc | Vangelis Koukis | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
23 | 9fea53cc | Vangelis Koukis | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
24 | 9fea53cc | Vangelis Koukis | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
25 | 9fea53cc | Vangelis Koukis | # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
26 | 9fea53cc | Vangelis Koukis | # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
27 | 9fea53cc | Vangelis Koukis | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
28 | 9fea53cc | Vangelis Koukis | # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
29 | 9fea53cc | Vangelis Koukis | # POSSIBILITY OF SUCH DAMAGE.
|
30 | 9fea53cc | Vangelis Koukis | #
|
31 | 9fea53cc | Vangelis Koukis | # The views and conclusions contained in the software and
|
32 | 9fea53cc | Vangelis Koukis | # documentation are those of the authors and should not be
|
33 | 9fea53cc | Vangelis Koukis | # interpreted as representing official policies, either expressed
|
34 | 9fea53cc | Vangelis Koukis | # or implied, of GRNET S.A.
|
35 | 9fea53cc | Vangelis Koukis | #
|
36 | 9fea53cc | Vangelis Koukis | """Business logic for reconciliation
|
37 | 9fea53cc | Vangelis Koukis |
|
38 | 9fea53cc | Vangelis Koukis | Reconcile the contents of the DB with the actual state of the
|
39 | 9fea53cc | Vangelis Koukis | Ganeti backend.
|
40 | 9fea53cc | Vangelis Koukis |
|
41 | 9fea53cc | Vangelis Koukis | Let D be the set of VMs in the DB, G the set of VMs in Ganeti.
|
42 | 9fea53cc | Vangelis Koukis | RULES:
|
43 | 9fea53cc | Vangelis Koukis | R1. Stale servers in DB:
|
44 | 9fea53cc | Vangelis Koukis | For any v in D but not in G:
|
45 | 9fea53cc | Vangelis Koukis | Set deleted=True.
|
46 | 9fea53cc | Vangelis Koukis | R2. Orphan instances in Ganet:
|
47 | 9fea53cc | Vangelis Koukis | For any v in G with deleted=True in D:
|
48 | 9fea53cc | Vangelis Koukis | Issue OP_INSTANCE_DESTROY.
|
49 | 9fea53cc | Vangelis Koukis | R3. Unsynced operstate:
|
50 | 9fea53cc | Vangelis Koukis | For any v whose operating state differs between G and V:
|
51 | 9fea53cc | Vangelis Koukis | Set the operating state in D based on the state in G.
|
52 | 9fea53cc | Vangelis Koukis | In the code, D, G are Python dicts mapping instance ids to operating state.
|
53 | 9fea53cc | Vangelis Koukis | For D, the operating state is chosen from VirtualMachine.OPER_STATES.
|
54 | 9fea53cc | Vangelis Koukis | For G, the operating state is True if the machine is up, False otherwise.
|
55 | 9fea53cc | Vangelis Koukis |
|
56 | 9fea53cc | Vangelis Koukis | """
|
57 | 9fea53cc | Vangelis Koukis | |
58 | 9fea53cc | Vangelis Koukis | |
59 | 9fea53cc | Vangelis Koukis | from django.core.management import setup_environ |
60 | 9fea53cc | Vangelis Koukis | try:
|
61 | 9fea53cc | Vangelis Koukis | from synnefo import settings |
62 | 9fea53cc | Vangelis Koukis | except ImportError: |
63 | 9fea53cc | Vangelis Koukis | raise Exception("Cannot import settings, make sure PYTHONPATH contains " |
64 | 9fea53cc | Vangelis Koukis | "the parent directory of the Synnefo Django project.")
|
65 | 9fea53cc | Vangelis Koukis | setup_environ(settings) |
66 | 9fea53cc | Vangelis Koukis | |
67 | 4161cb41 | Christos Stavrakakis | |
68 | 75dc539e | Christos Stavrakakis | import logging |
69 | 75dc539e | Christos Stavrakakis | import itertools |
70 | 4161cb41 | Christos Stavrakakis | from datetime import datetime, timedelta |
71 | 4161cb41 | Christos Stavrakakis | |
72 | 75dc539e | Christos Stavrakakis | from django.db import transaction |
73 | 75dc539e | Christos Stavrakakis | from synnefo.db.models import (Backend, VirtualMachine, Flavor, |
74 | 7ab30015 | Christos Stavrakakis | pooled_rapi_client) |
75 | 75dc539e | Christos Stavrakakis | from synnefo.logic import utils, backend as backend_mod |
76 | 3524241a | Christos Stavrakakis | from synnefo.logic.rapi import GanetiApiError |
77 | 9fea53cc | Vangelis Koukis | |
78 | 75dc539e | Christos Stavrakakis | logger = logging.getLogger() |
79 | 75dc539e | Christos Stavrakakis | logging.basicConfig() |
80 | 9e98ba3c | Giorgos Verigakis | |
81 | e63050ca | Christos Stavrakakis | try:
|
82 | e63050ca | Christos Stavrakakis | CHECK_INTERVAL = settings.RECONCILIATION_CHECK_INTERVAL |
83 | e63050ca | Christos Stavrakakis | except AttributeError: |
84 | e63050ca | Christos Stavrakakis | CHECK_INTERVAL = 60
|
85 | e63050ca | Christos Stavrakakis | |
86 | e63050ca | Christos Stavrakakis | |
87 | 75dc539e | Christos Stavrakakis | class BackendReconciler(object): |
88 | 75dc539e | Christos Stavrakakis | def __init__(self, backend, logger, options=None): |
89 | 75dc539e | Christos Stavrakakis | self.backend = backend
|
90 | 75dc539e | Christos Stavrakakis | self.log = logger
|
91 | 75dc539e | Christos Stavrakakis | self.client = backend.get_client()
|
92 | 75dc539e | Christos Stavrakakis | if options is None: |
93 | 75dc539e | Christos Stavrakakis | self.options = {}
|
94 | 4161cb41 | Christos Stavrakakis | else:
|
95 | 75dc539e | Christos Stavrakakis | self.options = options
|
96 | 75dc539e | Christos Stavrakakis | |
97 | 75dc539e | Christos Stavrakakis | def close(self): |
98 | 75dc539e | Christos Stavrakakis | self.backend.put_client(self.client) |
99 | 75dc539e | Christos Stavrakakis | |
100 | 75dc539e | Christos Stavrakakis | @transaction.commit_on_success
|
101 | 75dc539e | Christos Stavrakakis | def reconcile(self): |
102 | 75dc539e | Christos Stavrakakis | log = self.log
|
103 | 75dc539e | Christos Stavrakakis | backend = self.backend
|
104 | 75dc539e | Christos Stavrakakis | log.debug("Reconciling backend %s", backend)
|
105 | 75dc539e | Christos Stavrakakis | |
106 | 75dc539e | Christos Stavrakakis | self.db_servers = get_database_servers(backend)
|
107 | 75dc539e | Christos Stavrakakis | self.db_servers_keys = set(self.db_servers.keys()) |
108 | 75dc539e | Christos Stavrakakis | log.debug("Got servers info from database.")
|
109 | 75dc539e | Christos Stavrakakis | |
110 | 75dc539e | Christos Stavrakakis | self.gnt_servers = get_ganeti_servers(backend)
|
111 | 75dc539e | Christos Stavrakakis | self.gnt_servers_keys = set(self.gnt_servers.keys()) |
112 | 75dc539e | Christos Stavrakakis | log.debug("Got servers info from Ganeti backend.")
|
113 | 75dc539e | Christos Stavrakakis | |
114 | 75dc539e | Christos Stavrakakis | self.event_time = datetime.now()
|
115 | 75dc539e | Christos Stavrakakis | |
116 | 75dc539e | Christos Stavrakakis | self.stale_servers = self.reconcile_stale_servers() |
117 | 75dc539e | Christos Stavrakakis | self.orphan_servers = self.reconcile_orphan_servers() |
118 | 75dc539e | Christos Stavrakakis | self.unsynced_servers = self.reconcile_unsynced_servers() |
119 | 75dc539e | Christos Stavrakakis | self.close()
|
120 | 75dc539e | Christos Stavrakakis | |
121 | 75dc539e | Christos Stavrakakis | def get_build_status(self, db_server): |
122 | 75dc539e | Christos Stavrakakis | job = db_server.backendjobid |
123 | 75dc539e | Christos Stavrakakis | if job is None: |
124 | 75dc539e | Christos Stavrakakis | created = db_server.created |
125 | 75dc539e | Christos Stavrakakis | # Job has not yet been enqueued.
|
126 | 75dc539e | Christos Stavrakakis | if self.event_time < created + timedelta(seconds=60): |
127 | 75dc539e | Christos Stavrakakis | return "RUNNING" |
128 | 75dc539e | Christos Stavrakakis | else:
|
129 | 75dc539e | Christos Stavrakakis | return "ERROR" |
130 | 75dc539e | Christos Stavrakakis | else:
|
131 | 75dc539e | Christos Stavrakakis | updated = db_server.backendtime |
132 | 75dc539e | Christos Stavrakakis | if self.event_time >= updated + timedelta(seconds=60): |
133 | 75dc539e | Christos Stavrakakis | try:
|
134 | 75dc539e | Christos Stavrakakis | job_info = self.client.GetJobStatus(job_id=job)
|
135 | 75dc539e | Christos Stavrakakis | finalized = ["success", "error", "cancelled"] |
136 | 75dc539e | Christos Stavrakakis | if job_info["status"] == "error": |
137 | 75dc539e | Christos Stavrakakis | return "ERROR" |
138 | 75dc539e | Christos Stavrakakis | elif job_info["status"] not in finalized: |
139 | 75dc539e | Christos Stavrakakis | return "RUNNING" |
140 | 75dc539e | Christos Stavrakakis | else:
|
141 | 75dc539e | Christos Stavrakakis | return "FINALIZED" |
142 | 75dc539e | Christos Stavrakakis | except GanetiApiError:
|
143 | 75dc539e | Christos Stavrakakis | return "ERROR" |
144 | 75dc539e | Christos Stavrakakis | else:
|
145 | 75dc539e | Christos Stavrakakis | self.log.debug("Pending build for server '%s'", db_server.id) |
146 | 75dc539e | Christos Stavrakakis | return "RUNNING" |
147 | 75dc539e | Christos Stavrakakis | |
148 | 75dc539e | Christos Stavrakakis | def reconcile_stale_servers(self): |
149 | 75dc539e | Christos Stavrakakis | # Detect stale servers
|
150 | 75dc539e | Christos Stavrakakis | stale = [] |
151 | 75dc539e | Christos Stavrakakis | stale_keys = self.db_servers_keys - self.gnt_servers_keys |
152 | 75dc539e | Christos Stavrakakis | for server_id in stale_keys: |
153 | 75dc539e | Christos Stavrakakis | db_server = self.db_servers[server_id]
|
154 | 75dc539e | Christos Stavrakakis | if db_server.operstate == "BUILD": |
155 | 75dc539e | Christos Stavrakakis | build_status = self.get_build_status(db_server)
|
156 | 75dc539e | Christos Stavrakakis | if build_status == "ERROR": |
157 | 75dc539e | Christos Stavrakakis | # Special handling of BUILD eerrors
|
158 | 75dc539e | Christos Stavrakakis | self.reconcile_building_server(db_server)
|
159 | 75dc539e | Christos Stavrakakis | elif build_status != "RUNNING": |
160 | 75dc539e | Christos Stavrakakis | stale.append(server_id) |
161 | 75dc539e | Christos Stavrakakis | else:
|
162 | 75dc539e | Christos Stavrakakis | stale.append(server_id) |
163 | 75dc539e | Christos Stavrakakis | |
164 | 75dc539e | Christos Stavrakakis | # Report them
|
165 | 75dc539e | Christos Stavrakakis | if stale:
|
166 | 75dc539e | Christos Stavrakakis | self.log.info("Found stale servers %s at backend %s", |
167 | 75dc539e | Christos Stavrakakis | ", ".join(map(str, stale)), self.backend) |
168 | 75dc539e | Christos Stavrakakis | else:
|
169 | 75dc539e | Christos Stavrakakis | self.log.debug("No stale servers at backend %s", self.backend) |
170 | 75dc539e | Christos Stavrakakis | |
171 | 75dc539e | Christos Stavrakakis | # Fix them
|
172 | 75dc539e | Christos Stavrakakis | if stale and self.options["fix_stale"]: |
173 | 75dc539e | Christos Stavrakakis | for server_id in stale: |
174 | 75dc539e | Christos Stavrakakis | db_server = self.db_servers[server_id]
|
175 | 75dc539e | Christos Stavrakakis | backend_mod.process_op_status( |
176 | 75dc539e | Christos Stavrakakis | vm=db_server, |
177 | 75dc539e | Christos Stavrakakis | etime=self.event_time,
|
178 | 75dc539e | Christos Stavrakakis | jobid=-0,
|
179 | 75dc539e | Christos Stavrakakis | opcode='OP_INSTANCE_REMOVE', status='success', |
180 | 75dc539e | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
181 | 75dc539e | Christos Stavrakakis | self.log.debug("Simulated Ganeti removal for stale servers.") |
182 | 75dc539e | Christos Stavrakakis | |
183 | 75dc539e | Christos Stavrakakis | def reconcile_orphan_servers(self): |
184 | 75dc539e | Christos Stavrakakis | orphans = self.gnt_servers_keys - self.db_servers_keys |
185 | 75dc539e | Christos Stavrakakis | if orphans:
|
186 | 75dc539e | Christos Stavrakakis | self.log.info("Found orphan servers %s at backend %s", |
187 | 75dc539e | Christos Stavrakakis | ", ".join(map(str, orphans)), self.backend) |
188 | 75dc539e | Christos Stavrakakis | else:
|
189 | 75dc539e | Christos Stavrakakis | self.log.debug("No orphan servers at backend %s", self.backend) |
190 | 75dc539e | Christos Stavrakakis | |
191 | 75dc539e | Christos Stavrakakis | if orphans and self.options["fix_orphans"]: |
192 | 75dc539e | Christos Stavrakakis | for server_id in orphans: |
193 | 75dc539e | Christos Stavrakakis | server_name = utils.id_to_instance_name(server_id) |
194 | 75dc539e | Christos Stavrakakis | self.client.DeleteInstance(server_name)
|
195 | 75dc539e | Christos Stavrakakis | self.log.debug("Issued OP_INSTANCE_REMOVE for orphan servers.") |
196 | 75dc539e | Christos Stavrakakis | |
197 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_servers(self): |
198 | 75dc539e | Christos Stavrakakis | #log = self.log
|
199 | 75dc539e | Christos Stavrakakis | for server_id in self.db_servers_keys & self.gnt_servers_keys: |
200 | 75dc539e | Christos Stavrakakis | db_server = self.db_servers[server_id]
|
201 | 75dc539e | Christos Stavrakakis | gnt_server = self.gnt_servers[server_id]
|
202 | 75dc539e | Christos Stavrakakis | if db_server.operstate == "BUILD": |
203 | 75dc539e | Christos Stavrakakis | build_status = self.get_build_status(db_server)
|
204 | 75dc539e | Christos Stavrakakis | if build_status == "RUNNING": |
205 | 75dc539e | Christos Stavrakakis | # Do not reconcile building VMs
|
206 | 75dc539e | Christos Stavrakakis | continue
|
207 | 75dc539e | Christos Stavrakakis | elif build_status == "ERROR": |
208 | 75dc539e | Christos Stavrakakis | # Special handling of build errors
|
209 | 75dc539e | Christos Stavrakakis | self.reconcile_building_server(db_server)
|
210 | 75dc539e | Christos Stavrakakis | continue
|
211 | 75dc539e | Christos Stavrakakis | |
212 | 75dc539e | Christos Stavrakakis | self.reconcile_unsynced_operstate(server_id, db_server,
|
213 | 75dc539e | Christos Stavrakakis | gnt_server) |
214 | 75dc539e | Christos Stavrakakis | self.reconcile_unsynced_flavor(server_id, db_server,
|
215 | 75dc539e | Christos Stavrakakis | gnt_server) |
216 | 75dc539e | Christos Stavrakakis | self.reconcile_unsynced_nics(server_id, db_server, gnt_server)
|
217 | 75dc539e | Christos Stavrakakis | self.reconcile_unsynced_disks(server_id, db_server, gnt_server)
|
218 | 75dc539e | Christos Stavrakakis | |
219 | 75dc539e | Christos Stavrakakis | def reconcile_building_server(self, db_server): |
220 | 75dc539e | Christos Stavrakakis | self.log.info("Server '%s' is BUILD in DB, but 'ERROR' in Ganeti.", |
221 | 75dc539e | Christos Stavrakakis | db_server.id) |
222 | 75dc539e | Christos Stavrakakis | if self.options["fix_unsynced"]: |
223 | 75dc539e | Christos Stavrakakis | fix_opcode = "OP_INSTANCE_CREATE"
|
224 | 75dc539e | Christos Stavrakakis | backend_mod.process_op_status( |
225 | 75dc539e | Christos Stavrakakis | vm=db_server, |
226 | 75dc539e | Christos Stavrakakis | etime=self.event_time,
|
227 | 75dc539e | Christos Stavrakakis | jobid=-0,
|
228 | 75dc539e | Christos Stavrakakis | opcode=fix_opcode, status='error',
|
229 | 75dc539e | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
230 | 75dc539e | Christos Stavrakakis | self.log.debug("Simulated Ganeti error build event for" |
231 | 75dc539e | Christos Stavrakakis | " server '%s'", db_server.id)
|
232 | 75dc539e | Christos Stavrakakis | |
233 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_operstate(self, server_id, db_server, gnt_server): |
234 | 75dc539e | Christos Stavrakakis | if db_server.operstate != gnt_server["state"]: |
235 | 75dc539e | Christos Stavrakakis | self.log.info("Server '%s' is '%s' in DB and '%s' in Ganeti.", |
236 | 75dc539e | Christos Stavrakakis | server_id, db_server.operstate, gnt_server["state"])
|
237 | 75dc539e | Christos Stavrakakis | if self.options["fix_unsynced"]: |
238 | 75dc539e | Christos Stavrakakis | fix_opcode = \ |
239 | 75dc539e | Christos Stavrakakis | "OP_INSTANCE_STARTUP" if gnt_server["state"] == "STARTED"\ |
240 | 75dc539e | Christos Stavrakakis | else "OP_INSTANCE_SHUTDOWN" |
241 | 75dc539e | Christos Stavrakakis | backend_mod.process_op_status( |
242 | 75dc539e | Christos Stavrakakis | vm=db_server, |
243 | 75dc539e | Christos Stavrakakis | etime=self.event_time,
|
244 | 75dc539e | Christos Stavrakakis | jobid=-0,
|
245 | 75dc539e | Christos Stavrakakis | opcode=fix_opcode, status='success',
|
246 | 75dc539e | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
247 | 75dc539e | Christos Stavrakakis | self.log.debug("Simulated Ganeti state event for server '%s'", |
248 | 75dc539e | Christos Stavrakakis | server_id) |
249 | 75dc539e | Christos Stavrakakis | |
250 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_flavor(self, server_id, db_server, gnt_server): |
251 | 75dc539e | Christos Stavrakakis | db_flavor = db_server.flavor |
252 | 75dc539e | Christos Stavrakakis | gnt_flavor = gnt_server["flavor"]
|
253 | 75dc539e | Christos Stavrakakis | if (db_flavor.ram != gnt_flavor["ram"] or |
254 | 75dc539e | Christos Stavrakakis | db_flavor.cpu != gnt_flavor["vcpus"]):
|
255 | a67419d8 | Christos Stavrakakis | try:
|
256 | a67419d8 | Christos Stavrakakis | gnt_flavor = Flavor.objects.get( |
257 | 75dc539e | Christos Stavrakakis | ram=gnt_flavor["ram"],
|
258 | 75dc539e | Christos Stavrakakis | cpu=gnt_flavor["vcpus"],
|
259 | 75dc539e | Christos Stavrakakis | disk=db_flavor.disk, |
260 | 75dc539e | Christos Stavrakakis | disk_template=db_flavor.disk_template) |
261 | a67419d8 | Christos Stavrakakis | except Flavor.DoesNotExist:
|
262 | 75dc539e | Christos Stavrakakis | self.log.warning("Server '%s' has unknown flavor.", server_id) |
263 | 75dc539e | Christos Stavrakakis | return
|
264 | 75dc539e | Christos Stavrakakis | |
265 | 75dc539e | Christos Stavrakakis | self.log.info("Server '%s' has flavor '%' in DB and '%s' in" |
266 | 75dc539e | Christos Stavrakakis | " Ganeti", server_id, db_flavor, gnt_flavor)
|
267 | 75dc539e | Christos Stavrakakis | if self.options["fix_unsynced_flavors"]: |
268 | 75dc539e | Christos Stavrakakis | old_state = db_server.operstate |
269 | 75dc539e | Christos Stavrakakis | opcode = "OP_INSTANCE_SET_PARAMS"
|
270 | 75dc539e | Christos Stavrakakis | beparams = {"vcpus": gnt_flavor.cpu,
|
271 | 75dc539e | Christos Stavrakakis | "minmem": gnt_flavor.ram,
|
272 | 75dc539e | Christos Stavrakakis | "maxmem": gnt_flavor.ram}
|
273 | 75dc539e | Christos Stavrakakis | backend_mod.process_op_status( |
274 | 75dc539e | Christos Stavrakakis | vm=db_server, etime=self.event_time, jobid=-0, |
275 | 75dc539e | Christos Stavrakakis | opcode=opcode, status='success',
|
276 | 75dc539e | Christos Stavrakakis | beparams=beparams, |
277 | 75dc539e | Christos Stavrakakis | logmsg='Reconciliation: simulated Ganeti event')
|
278 | 75dc539e | Christos Stavrakakis | # process_op_status with beparams will set the vmstate to
|
279 | 75dc539e | Christos Stavrakakis | # shutdown. Fix this be returning it to old state
|
280 | 75dc539e | Christos Stavrakakis | vm = VirtualMachine.objects.get(pk=server_id) |
281 | 75dc539e | Christos Stavrakakis | vm.operstate = old_state |
282 | 75dc539e | Christos Stavrakakis | vm.save() |
283 | 75dc539e | Christos Stavrakakis | self.log.debug("Simulated Ganeti flavor event for server '%s'", |
284 | 75dc539e | Christos Stavrakakis | server_id) |
285 | 75dc539e | Christos Stavrakakis | |
286 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_nics(self, server_id, db_server, gnt_server): |
287 | 75dc539e | Christos Stavrakakis | db_nics = db_server.nics.order_by("index")
|
288 | 75dc539e | Christos Stavrakakis | gnt_nics = gnt_server["nics"]
|
289 | 75dc539e | Christos Stavrakakis | gnt_nics_parsed = backend_mod.process_ganeti_nics(gnt_nics) |
290 | 75dc539e | Christos Stavrakakis | if backend_mod.nics_changed(db_nics, gnt_nics_parsed):
|
291 | 75dc539e | Christos Stavrakakis | msg = "Found unsynced NICs for server '%s'.\n\t"\
|
292 | 75dc539e | Christos Stavrakakis | "DB: %s\n\tGaneti: %s"
|
293 | 75dc539e | Christos Stavrakakis | db_nics_str = ", ".join(map(format_db_nic, db_nics)) |
294 | 75dc539e | Christos Stavrakakis | gnt_nics_str = ", ".join(map(format_gnt_nic, gnt_nics_parsed)) |
295 | 75dc539e | Christos Stavrakakis | self.log.info(msg, server_id, db_nics_str, gnt_nics_str)
|
296 | 75dc539e | Christos Stavrakakis | if self.options["fix_unsynced_nics"]: |
297 | 75dc539e | Christos Stavrakakis | backend_mod.process_net_status(vm=db_server, |
298 | 75dc539e | Christos Stavrakakis | etime=self.event_time,
|
299 | 75dc539e | Christos Stavrakakis | nics=gnt_nics) |
300 | 75dc539e | Christos Stavrakakis | |
301 | 75dc539e | Christos Stavrakakis | def reconcile_unsynced_disks(self, server_id, db_server, gnt_server): |
302 | 75dc539e | Christos Stavrakakis | pass
|
303 | 75dc539e | Christos Stavrakakis | |
304 | 75dc539e | Christos Stavrakakis | |
305 | 75dc539e | Christos Stavrakakis | def format_db_nic(nic): |
306 | 75dc539e | Christos Stavrakakis | return "Index: %s IP: %s Network: %s MAC: %s Firewall: %s" % (nic.index, |
307 | 75dc539e | Christos Stavrakakis | nic.ipv4, nic.network_id, nic.mac, nic.firewall_profile) |
308 | 75dc539e | Christos Stavrakakis | |
309 | 75dc539e | Christos Stavrakakis | |
310 | 75dc539e | Christos Stavrakakis | def format_gnt_nic(nic): |
311 | 75dc539e | Christos Stavrakakis | return "Index: %s IP: %s Network: %s MAC: %s Firewall: %s" %\ |
312 | 75dc539e | Christos Stavrakakis | (nic["index"], nic["ipv4"], nic["network"], nic["mac"], |
313 | 75dc539e | Christos Stavrakakis | nic["firewall_profile"])
|
314 | 9fea53cc | Vangelis Koukis | |
315 | cc92b70f | Christos Stavrakakis | |
316 | 0e9a423f | Christos Stavrakakis | #
|
317 | 0e9a423f | Christos Stavrakakis | # Networks
|
318 | 0e9a423f | Christos Stavrakakis | #
|
319 | 3524241a | Christos Stavrakakis | |
320 | 3524241a | Christos Stavrakakis | |
321 | 0e9a423f | Christos Stavrakakis | def get_networks_from_ganeti(backend): |
322 | 44e2c577 | Christos Stavrakakis | prefix = settings.BACKEND_PREFIX_ID + 'net-'
|
323 | 0e9a423f | Christos Stavrakakis | |
324 | 0e9a423f | Christos Stavrakakis | networks = {} |
325 | 3524241a | Christos Stavrakakis | with pooled_rapi_client(backend) as c: |
326 | 3524241a | Christos Stavrakakis | for net in c.GetNetworks(bulk=True): |
327 | 3524241a | Christos Stavrakakis | if net['name'].startswith(prefix): |
328 | 3524241a | Christos Stavrakakis | id = utils.id_from_network_name(net['name'])
|
329 | 3524241a | Christos Stavrakakis | networks[id] = net
|
330 | 0e9a423f | Christos Stavrakakis | |
331 | 0e9a423f | Christos Stavrakakis | return networks
|
332 | 0e9a423f | Christos Stavrakakis | |
333 | 0e9a423f | Christos Stavrakakis | |
334 | 0e9a423f | Christos Stavrakakis | def hanging_networks(backend, GNets): |
335 | 0e9a423f | Christos Stavrakakis | """Get networks that are not connected to all Nodegroups.
|
336 | 0e9a423f | Christos Stavrakakis |
|
337 | 0e9a423f | Christos Stavrakakis | """
|
338 | 0e9a423f | Christos Stavrakakis | def get_network_groups(group_list): |
339 | 0e9a423f | Christos Stavrakakis | groups = set()
|
340 | 0e9a423f | Christos Stavrakakis | for g in group_list: |
341 | 0e9a423f | Christos Stavrakakis | g_name = g.split('(')[0] |
342 | 0e9a423f | Christos Stavrakakis | groups.add(g_name) |
343 | 0e9a423f | Christos Stavrakakis | return groups
|
344 | 0e9a423f | Christos Stavrakakis | |
345 | 3524241a | Christos Stavrakakis | with pooled_rapi_client(backend) as c: |
346 | 3524241a | Christos Stavrakakis | groups = set(c.GetGroups())
|
347 | 0e9a423f | Christos Stavrakakis | |
348 | 0e9a423f | Christos Stavrakakis | hanging = {} |
349 | 0e9a423f | Christos Stavrakakis | for id, info in GNets.items(): |
350 | 0e9a423f | Christos Stavrakakis | group_list = get_network_groups(info['group_list'])
|
351 | 0e9a423f | Christos Stavrakakis | if group_list != groups:
|
352 | 0e9a423f | Christos Stavrakakis | hanging[id] = groups - group_list
|
353 | 0e9a423f | Christos Stavrakakis | return hanging
|
354 | 0e9a423f | Christos Stavrakakis | |
355 | 9fea53cc | Vangelis Koukis | |
356 | 75dc539e | Christos Stavrakakis | def get_online_backends(): |
357 | 75dc539e | Christos Stavrakakis | return Backend.objects.filter(offline=False) |
358 | 75dc539e | Christos Stavrakakis | |
359 | 75dc539e | Christos Stavrakakis | |
360 | 75dc539e | Christos Stavrakakis | def get_database_servers(backend): |
361 | 75dc539e | Christos Stavrakakis | servers = backend.virtual_machines.select_related("nics", "flavor")\ |
362 | 75dc539e | Christos Stavrakakis | .filter(deleted=False)
|
363 | 75dc539e | Christos Stavrakakis | return dict([(s.id, s) for s in servers]) |
364 | 75dc539e | Christos Stavrakakis | |
365 | 75dc539e | Christos Stavrakakis | |
366 | 75dc539e | Christos Stavrakakis | def get_ganeti_servers(backend): |
367 | 75dc539e | Christos Stavrakakis | gnt_instances = backend_mod.get_instances(backend) |
368 | 75dc539e | Christos Stavrakakis | # Filter out non-synnefo instances
|
369 | 75dc539e | Christos Stavrakakis | snf_backend_prefix = settings.BACKEND_PREFIX_ID |
370 | 75dc539e | Christos Stavrakakis | gnt_instances = filter(lambda i: i["name"].startswith(snf_backend_prefix), |
371 | 75dc539e | Christos Stavrakakis | gnt_instances) |
372 | 75dc539e | Christos Stavrakakis | gnt_instances = map(parse_gnt_instance, gnt_instances)
|
373 | 75dc539e | Christos Stavrakakis | return dict([(i["id"], i) for i in gnt_instances if i["id"] is not None]) |
374 | 75dc539e | Christos Stavrakakis | |
375 | 75dc539e | Christos Stavrakakis | |
376 | 75dc539e | Christos Stavrakakis | def parse_gnt_instance(instance): |
377 | 75dc539e | Christos Stavrakakis | try:
|
378 | 75dc539e | Christos Stavrakakis | instance_id = utils.id_from_instance_name(instance['name'])
|
379 | 75dc539e | Christos Stavrakakis | except Exception: |
380 | 75dc539e | Christos Stavrakakis | logger.error("Ignoring instance with malformed name %s",
|
381 | 75dc539e | Christos Stavrakakis | instance['name'])
|
382 | 75dc539e | Christos Stavrakakis | return (None, None) |
383 | 75dc539e | Christos Stavrakakis | |
384 | 75dc539e | Christos Stavrakakis | beparams = instance["beparams"]
|
385 | 75dc539e | Christos Stavrakakis | |
386 | 75dc539e | Christos Stavrakakis | vcpus = beparams["vcpus"]
|
387 | 75dc539e | Christos Stavrakakis | ram = beparams["maxmem"]
|
388 | 75dc539e | Christos Stavrakakis | state = instance["oper_state"] and "STARTED" or "STOPPED" |
389 | 75dc539e | Christos Stavrakakis | |
390 | 75dc539e | Christos Stavrakakis | return {
|
391 | 75dc539e | Christos Stavrakakis | "id": instance_id,
|
392 | 75dc539e | Christos Stavrakakis | "state": state, # FIX |
393 | 75dc539e | Christos Stavrakakis | "updated": datetime.fromtimestamp(instance["mtime"]), |
394 | 75dc539e | Christos Stavrakakis | "disks": disks_from_instance(instance),
|
395 | 75dc539e | Christos Stavrakakis | "nics": nics_from_instance(instance),
|
396 | 75dc539e | Christos Stavrakakis | "flavor": {"vcpus": vcpus, |
397 | 75dc539e | Christos Stavrakakis | "ram": ram},
|
398 | 75dc539e | Christos Stavrakakis | "tags": instance["tags"] |
399 | 75dc539e | Christos Stavrakakis | } |
400 | 75dc539e | Christos Stavrakakis | |
401 | 75dc539e | Christos Stavrakakis | |
402 | 75dc539e | Christos Stavrakakis | def nics_from_instance(i): |
403 | 75dc539e | Christos Stavrakakis | ips = zip(itertools.repeat('ip'), i['nic.ips']) |
404 | 75dc539e | Christos Stavrakakis | macs = zip(itertools.repeat('mac'), i['nic.macs']) |
405 | 75dc539e | Christos Stavrakakis | networks = zip(itertools.repeat('network'), i['nic.networks']) |
406 | 75dc539e | Christos Stavrakakis | # modes = zip(itertools.repeat('mode'), i['nic.modes'])
|
407 | 75dc539e | Christos Stavrakakis | # links = zip(itertools.repeat('link'), i['nic.links'])
|
408 | 75dc539e | Christos Stavrakakis | # nics = zip(ips,macs,modes,networks,links)
|
409 | 75dc539e | Christos Stavrakakis | nics = zip(ips, macs, networks)
|
410 | 75dc539e | Christos Stavrakakis | nics = map(lambda x: dict(x), nics) |
411 | 75dc539e | Christos Stavrakakis | #nics = dict(enumerate(nics))
|
412 | 75dc539e | Christos Stavrakakis | tags = i["tags"]
|
413 | 75dc539e | Christos Stavrakakis | for tag in tags: |
414 | 75dc539e | Christos Stavrakakis | t = tag.split(":")
|
415 | 75dc539e | Christos Stavrakakis | if t[0:2] == ["synnefo", "network"]: |
416 | 75dc539e | Christos Stavrakakis | if len(t) != 4: |
417 | 75dc539e | Christos Stavrakakis | logger.error("Malformed synefo tag %s", tag)
|
418 | 75dc539e | Christos Stavrakakis | continue
|
419 | 75dc539e | Christos Stavrakakis | try:
|
420 | 75dc539e | Christos Stavrakakis | index = int(t[2]) |
421 | 75dc539e | Christos Stavrakakis | nics[index]['firewall'] = t[3] |
422 | 75dc539e | Christos Stavrakakis | except ValueError: |
423 | 75dc539e | Christos Stavrakakis | logger.error("Malformed synnefo tag %s", tag)
|
424 | 75dc539e | Christos Stavrakakis | except IndexError: |
425 | 75dc539e | Christos Stavrakakis | logger.error("Found tag %s for non-existent NIC %d",
|
426 | 75dc539e | Christos Stavrakakis | tag, index) |
427 | 75dc539e | Christos Stavrakakis | return nics
|
428 | 9fea53cc | Vangelis Koukis | |
429 | 9fea53cc | Vangelis Koukis | |
430 | 75dc539e | Christos Stavrakakis | def disks_from_instance(i): |
431 | 75dc539e | Christos Stavrakakis | return dict([(index, {"size": size}) |
432 | 75dc539e | Christos Stavrakakis | for index, size in enumerate(i["disk.sizes"])]) |