Revision 7ef05bd4
b/snf-cyclades-app/synnefo/logic/reconciliation.py | ||
---|---|---|
70 | 70 |
MacPrefixPoolTable) |
71 | 71 |
from synnefo.db import pools |
72 | 72 |
from synnefo.logic import utils, rapi, backend as backend_mod |
73 |
from synnefo.lib.utils import merge_time |
|
73 | 74 |
|
74 | 75 |
logger = logging.getLogger() |
75 | 76 |
logging.basicConfig() |
... | ... | |
96 | 97 |
backend = self.backend |
97 | 98 |
log.debug("Reconciling backend %s", backend) |
98 | 99 |
|
100 |
self.event_time = datetime.now() |
|
101 |
|
|
99 | 102 |
self.db_servers = get_database_servers(backend) |
100 | 103 |
self.db_servers_keys = set(self.db_servers.keys()) |
101 | 104 |
log.debug("Got servers info from database.") |
... | ... | |
107 | 110 |
self.gnt_jobs = get_ganeti_jobs(backend) |
108 | 111 |
log.debug("Got jobs from Ganeti backend") |
109 | 112 |
|
110 |
self.event_time = datetime.now() |
|
111 |
|
|
112 | 113 |
self.stale_servers = self.reconcile_stale_servers() |
113 | 114 |
self.orphan_servers = self.reconcile_orphan_servers() |
114 | 115 |
self.unsynced_servers = self.reconcile_unsynced_servers() |
115 | 116 |
self.close() |
116 | 117 |
|
117 | 118 |
def get_build_status(self, db_server): |
119 |
"""Return the status of the build job. |
|
120 |
|
|
121 |
Return whether the job is RUNNING, FINALIZED or ERROR, together |
|
122 |
with the timestamp that the job finished (if any). |
|
123 |
|
|
124 |
""" |
|
118 | 125 |
job_id = db_server.backendjobid |
119 | 126 |
if job_id in self.gnt_jobs: |
120 |
gnt_job_status = self.gnt_jobs[job_id]["status"] |
|
127 |
job = self.gnt_jobs[job_id] |
|
128 |
gnt_job_status = job["status"] |
|
129 |
end_timestamp = merge_time(job["end_ts"]) |
|
121 | 130 |
if gnt_job_status == rapi.JOB_STATUS_ERROR: |
122 |
return "ERROR" |
|
131 |
return "ERROR", end_timestamp
|
|
123 | 132 |
elif gnt_job_status not in rapi.JOB_STATUS_FINALIZED: |
124 |
return "RUNNING" |
|
133 |
return "RUNNING", None
|
|
125 | 134 |
else: |
126 |
return "FINALIZED" |
|
135 |
return "FINALIZED", end_timestamp
|
|
127 | 136 |
else: |
128 |
return "ERROR" |
|
137 |
return "ERROR", None
|
|
129 | 138 |
|
130 | 139 |
def reconcile_stale_servers(self): |
131 | 140 |
# Detect stale servers |
... | ... | |
134 | 143 |
for server_id in stale_keys: |
135 | 144 |
db_server = self.db_servers[server_id] |
136 | 145 |
if db_server.operstate == "BUILD": |
137 |
build_status = self.get_build_status(db_server) |
|
146 |
build_status, end_timestamp = self.get_build_status(db_server)
|
|
138 | 147 |
if build_status == "ERROR": |
139 | 148 |
# Special handling of BUILD eerrors |
140 | 149 |
self.reconcile_building_server(db_server) |
... | ... | |
187 | 196 |
db_server = self.db_servers[server_id] |
188 | 197 |
gnt_server = self.gnt_servers[server_id] |
189 | 198 |
if db_server.operstate == "BUILD": |
190 |
build_status = self.get_build_status(db_server) |
|
199 |
build_status, end_timestamp = self.get_build_status(db_server)
|
|
191 | 200 |
if build_status == "RUNNING": |
192 | 201 |
# Do not reconcile building VMs |
193 | 202 |
continue |
... | ... | |
195 | 204 |
# Special handling of build errors |
196 | 205 |
self.reconcile_building_server(db_server) |
197 | 206 |
continue |
207 |
elif end_timestamp >= self.event_time: |
|
208 |
# Do not continue reconciliation for building server that |
|
209 |
# the build job completed after quering the state of |
|
210 |
# Ganeti servers. |
|
211 |
continue |
|
198 | 212 |
|
199 | 213 |
self.reconcile_unsynced_operstate(server_id, db_server, |
200 | 214 |
gnt_server) |
Also available in: Unified diff