Revision 4161cb41
| b/snf-cyclades-app/synnefo/api/servers.py | ||
|---|---|---|
| 269 | 269 |
flavor=flavor) |
| 270 | 270 |
|
| 271 | 271 |
try: |
| 272 |
create_instance(vm, flavor, image, password, personality) |
|
| 272 |
jobID = create_instance(vm, flavor, image, password, personality)
|
|
| 273 | 273 |
except GanetiApiError: |
| 274 | 274 |
vm.delete() |
| 275 | 275 |
raise |
| 276 | 276 |
|
| 277 |
vm.backendjobid = jobID |
|
| 278 |
vm.save() |
|
| 279 |
|
|
| 277 | 280 |
for key, val in metadata.items(): |
| 278 | 281 |
VirtualMachineMetadata.objects.create( |
| 279 | 282 |
meta_key=key, |
| b/snf-cyclades-app/synnefo/logic/management/commands/reconcile.py | ||
|---|---|---|
| 62 | 62 |
dest='detect_unsynced', |
| 63 | 63 |
default=False, help='Detect unsynced operstate between ' + |
| 64 | 64 |
'DB and Ganeti'), |
| 65 |
make_option('--detect-build-errors', action='store_true',
|
|
| 66 |
dest='detect_build_errors', default=False, |
|
| 67 |
help='Detect instances with build error'), |
|
| 65 | 68 |
make_option('--detect-all', action='store_true',
|
| 66 | 69 |
dest='detect_all', |
| 67 | 70 |
default=False, help='Enable all --detect-* arguments'), |
| ... | ... | |
| 72 | 75 |
make_option('--fix-unsynced', action='store_true', dest='fix_unsynced',
|
| 73 | 76 |
default=False, help='Fix server operstate in DB, set ' + |
| 74 | 77 |
'from Ganeti'), |
| 78 |
make_option('--fix-build-errors', action='store_true',
|
|
| 79 |
dest='fix_build_errors', default=False, |
|
| 80 |
help='Fix (remove) instances with build errors'), |
|
| 75 | 81 |
make_option('--fix-all', action='store_true', dest='fix_all',
|
| 76 | 82 |
default=False, help='Enable all --fix-* arguments')) |
| 77 | 83 |
|
| ... | ... | |
| 136 | 142 |
elif verbosity == 2: |
| 137 | 143 |
print >> sys.stderr, "The operstate of all servers is in sync." |
| 138 | 144 |
|
| 145 |
if options['detect_build_errors']: |
|
| 146 |
build_errors = reconciliation.instances_with_build_errors(D, G) |
|
| 147 |
if len(build_errors) > 0: |
|
| 148 |
print >> sys.stderr, "The os for the following server IDs was "\ |
|
| 149 |
"not build successfully:" |
|
| 150 |
print " " + "\n ".join( |
|
| 151 |
["%d" % x for x in build_errors]) |
|
| 152 |
elif verbosity == 2: |
|
| 153 |
print >> sys.stderr, "Found no instances with build errors." |
|
| 154 |
|
|
| 139 | 155 |
# |
| 140 | 156 |
# Then fix them |
| 141 | 157 |
# |
| ... | ... | |
| 172 | 188 |
opcode=opcode, status='success', |
| 173 | 189 |
logmsg='Reconciliation: simulated Ganeti event') |
| 174 | 190 |
print >> sys.stderr, " ...done" |
| 191 |
|
|
| 192 |
if options['fix_build_errors'] and len(build_errors) > 0: |
|
| 193 |
print >> sys.stderr, "Setting the state of %d build-errors VMs:" % \ |
|
| 194 |
len(build_errors) |
|
| 195 |
for id in build_errors: |
|
| 196 |
vm = VirtualMachine.objects.get(pk=id) |
|
| 197 |
event_time = datetime.datetime.now() |
|
| 198 |
backend.process_op_status(vm=vm, etime=event_time ,jobid=-0, |
|
| 199 |
opcode="OP_INSTANCE_CREATE", status='error', |
|
| 200 |
logmsg='Reconciliation: simulated Ganeti event') |
|
| 201 |
print >> sys.stderr, " ...done" |
|
| 202 |
|
|
| b/snf-cyclades-app/synnefo/logic/reconciliation.py | ||
|---|---|---|
| 67 | 67 |
"the parent directory of the Synnefo Django project.") |
| 68 | 68 |
setup_environ(settings) |
| 69 | 69 |
|
| 70 |
|
|
| 71 |
from datetime import datetime, timedelta |
|
| 72 |
|
|
| 70 | 73 |
from synnefo.db.models import VirtualMachine |
| 71 | 74 |
from synnefo.util.dictconfig import dictConfig |
| 75 |
from synnefo.util.rapi import GanetiApiError |
|
| 72 | 76 |
from synnefo.logic.backend import get_ganeti_instances |
| 73 | 77 |
|
| 74 | 78 |
|
| ... | ... | |
| 79 | 83 |
idD = set(D.keys()) |
| 80 | 84 |
idG = set(G.keys()) |
| 81 | 85 |
|
| 82 |
return idD - idG |
|
| 86 |
stale = set() |
|
| 87 |
for i in idD - idG: |
|
| 88 |
if D[i] == 'BUILD': |
|
| 89 |
vm = VirtualMachine.objects.get(id=i) |
|
| 90 |
# Check time to avoid many rapi calls |
|
| 91 |
if datetime.now() > vm.backendtime + timedelta(seconds=5): |
|
| 92 |
try: |
|
| 93 |
job_status = vm.client.GetJobStatus(vm.backendjobid)['status'] |
|
| 94 |
if job_status in ('queued', 'waiting', 'running'):
|
|
| 95 |
# Server is still building in Ganeti |
|
| 96 |
continue |
|
| 97 |
else: |
|
| 98 |
new_vm = vm.client.GetInstance('%s%d' %
|
|
| 99 |
(settings.BACKEND_PREFIX_ID, i)) |
|
| 100 |
# Server has just been created in Ganeti |
|
| 101 |
continue |
|
| 102 |
except GanetiApiError: |
|
| 103 |
stale.add(i) |
|
| 104 |
else: |
|
| 105 |
stale.add(i) |
|
| 106 |
|
|
| 107 |
return stale |
|
| 83 | 108 |
|
| 84 | 109 |
|
| 85 | 110 |
def orphan_instances_in_ganeti(D, G): |
| ... | ... | |
| 98 | 123 |
if (G[i] and D[i] != 'STARTED' or |
| 99 | 124 |
not G[i] and D[i] not in ('BUILD', 'ERROR', 'STOPPED')):
|
| 100 | 125 |
unsynced.add((i, D[i], G[i])) |
| 126 |
if not G[i] and D[i] == 'BUILD': |
|
| 127 |
vm = VirtualMachine.objects.get(id=i) |
|
| 128 |
# Check time to avoid many rapi calls |
|
| 129 |
if datetime.now() > vm.backendtime + timedelta(seconds=5): |
|
| 130 |
try: |
|
| 131 |
job_info = vm.client.GetJobStatus(job_id = vm.backendjobid) |
|
| 132 |
if job_info['status'] == 'success': |
|
| 133 |
unsynced.add((i, D[i], G[i])) |
|
| 134 |
except GanetiApiError: |
|
| 135 |
pass |
|
| 101 | 136 |
|
| 102 | 137 |
return unsynced |
| 103 | 138 |
|
| 104 | 139 |
|
| 140 |
def instances_with_build_errors(D, G): |
|
| 141 |
failed = set() |
|
| 142 |
idD = set(D.keys()) |
|
| 143 |
idG = set(G.keys()) |
|
| 144 |
|
|
| 145 |
for i in idD & idG: |
|
| 146 |
if not G[i] and D[i] == 'BUILD': |
|
| 147 |
vm = VirtualMachine.objects.get(id=i) |
|
| 148 |
# Check time to avoid many rapi calls |
|
| 149 |
if datetime.now() > vm.backendtime + timedelta(seconds=5): |
|
| 150 |
try: |
|
| 151 |
job_info = vm.client.GetJobStatus(job_id = vm.backendjobid) |
|
| 152 |
if job_info['status'] == 'error': |
|
| 153 |
failed.add(i) |
|
| 154 |
except GanetiApiError: |
|
| 155 |
failed.add(i) |
|
| 156 |
|
|
| 157 |
return failed |
|
| 158 |
|
|
| 159 |
|
|
| 160 |
|
|
| 105 | 161 |
def get_servers_from_db(): |
| 106 | 162 |
vms = VirtualMachine.objects.filter(deleted=False) |
| 107 | 163 |
return dict(map(lambda x: (x.id, x.operstate), vms)) |
Also available in: Unified diff