Revision 7ef05bd4 snf-cyclades-app/synnefo/logic/reconciliation.py
| b/snf-cyclades-app/synnefo/logic/reconciliation.py | ||
|---|---|---|
| 70 | 70 |
MacPrefixPoolTable) |
| 71 | 71 |
from synnefo.db import pools |
| 72 | 72 |
from synnefo.logic import utils, rapi, backend as backend_mod |
| 73 |
from synnefo.lib.utils import merge_time |
|
| 73 | 74 |
|
| 74 | 75 |
logger = logging.getLogger() |
| 75 | 76 |
logging.basicConfig() |
| ... | ... | |
| 96 | 97 |
backend = self.backend |
| 97 | 98 |
log.debug("Reconciling backend %s", backend)
|
| 98 | 99 |
|
| 100 |
self.event_time = datetime.now() |
|
| 101 |
|
|
| 99 | 102 |
self.db_servers = get_database_servers(backend) |
| 100 | 103 |
self.db_servers_keys = set(self.db_servers.keys()) |
| 101 | 104 |
log.debug("Got servers info from database.")
|
| ... | ... | |
| 107 | 110 |
self.gnt_jobs = get_ganeti_jobs(backend) |
| 108 | 111 |
log.debug("Got jobs from Ganeti backend")
|
| 109 | 112 |
|
| 110 |
self.event_time = datetime.now() |
|
| 111 |
|
|
| 112 | 113 |
self.stale_servers = self.reconcile_stale_servers() |
| 113 | 114 |
self.orphan_servers = self.reconcile_orphan_servers() |
| 114 | 115 |
self.unsynced_servers = self.reconcile_unsynced_servers() |
| 115 | 116 |
self.close() |
| 116 | 117 |
|
| 117 | 118 |
def get_build_status(self, db_server): |
| 119 |
"""Return the status of the build job. |
|
| 120 |
|
|
| 121 |
Return whether the job is RUNNING, FINALIZED or ERROR, together |
|
| 122 |
with the timestamp that the job finished (if any). |
|
| 123 |
|
|
| 124 |
""" |
|
| 118 | 125 |
job_id = db_server.backendjobid |
| 119 | 126 |
if job_id in self.gnt_jobs: |
| 120 |
gnt_job_status = self.gnt_jobs[job_id]["status"] |
|
| 127 |
job = self.gnt_jobs[job_id] |
|
| 128 |
gnt_job_status = job["status"] |
|
| 129 |
end_timestamp = merge_time(job["end_ts"]) |
|
| 121 | 130 |
if gnt_job_status == rapi.JOB_STATUS_ERROR: |
| 122 |
return "ERROR" |
|
| 131 |
return "ERROR", end_timestamp
|
|
| 123 | 132 |
elif gnt_job_status not in rapi.JOB_STATUS_FINALIZED: |
| 124 |
return "RUNNING" |
|
| 133 |
return "RUNNING", None
|
|
| 125 | 134 |
else: |
| 126 |
return "FINALIZED" |
|
| 135 |
return "FINALIZED", end_timestamp
|
|
| 127 | 136 |
else: |
| 128 |
return "ERROR" |
|
| 137 |
return "ERROR", None
|
|
| 129 | 138 |
|
| 130 | 139 |
def reconcile_stale_servers(self): |
| 131 | 140 |
# Detect stale servers |
| ... | ... | |
| 134 | 143 |
for server_id in stale_keys: |
| 135 | 144 |
db_server = self.db_servers[server_id] |
| 136 | 145 |
if db_server.operstate == "BUILD": |
| 137 |
build_status = self.get_build_status(db_server) |
|
| 146 |
build_status, end_timestamp = self.get_build_status(db_server)
|
|
| 138 | 147 |
if build_status == "ERROR": |
| 139 | 148 |
# Special handling of BUILD eerrors |
| 140 | 149 |
self.reconcile_building_server(db_server) |
| ... | ... | |
| 187 | 196 |
db_server = self.db_servers[server_id] |
| 188 | 197 |
gnt_server = self.gnt_servers[server_id] |
| 189 | 198 |
if db_server.operstate == "BUILD": |
| 190 |
build_status = self.get_build_status(db_server) |
|
| 199 |
build_status, end_timestamp = self.get_build_status(db_server)
|
|
| 191 | 200 |
if build_status == "RUNNING": |
| 192 | 201 |
# Do not reconcile building VMs |
| 193 | 202 |
continue |
| ... | ... | |
| 195 | 204 |
# Special handling of build errors |
| 196 | 205 |
self.reconcile_building_server(db_server) |
| 197 | 206 |
continue |
| 207 |
elif end_timestamp >= self.event_time: |
|
| 208 |
# Do not continue reconciliation for building server that |
|
| 209 |
# the build job completed after quering the state of |
|
| 210 |
# Ganeti servers. |
|
| 211 |
continue |
|
| 198 | 212 |
|
| 199 | 213 |
self.reconcile_unsynced_operstate(server_id, db_server, |
| 200 | 214 |
gnt_server) |
Also available in: Unified diff