Revision d2036274 snf-cyclades-app/synnefo/logic/backend.py
b/snf-cyclades-app/synnefo/logic/backend.py | ||
---|---|---|
42 | 42 |
from synnefo import quotas |
43 | 43 |
from synnefo.api.util import release_resource |
44 | 44 |
from synnefo.util.mac2eui64 import mac2eui64 |
45 |
from synnefo.logic.rapi import GanetiApiError
|
|
45 |
from synnefo.logic import rapi
|
|
46 | 46 |
|
47 | 47 |
from logging import getLogger |
48 | 48 |
log = getLogger(__name__) |
... | ... | |
76 | 76 |
rejected, since they reflect a previous state of the VM. |
77 | 77 |
|
78 | 78 |
""" |
79 |
if job_status not in ["success", "error", "canceled"]:
|
|
79 |
if job_status not in rapi.JOB_STATUS_FINALIZED:
|
|
80 | 80 |
return vm |
81 | 81 |
|
82 | 82 |
# Check successful completion of a job will trigger any quotable change in |
... | ... | |
94 | 94 |
# if fails, must be accepted, as the user must manually remove the |
95 | 95 |
# failed server |
96 | 96 |
serial = vm.serial |
97 |
if job_status == "success":
|
|
97 |
if job_status == rapi.JOB_STATUS_SUCCESS:
|
|
98 | 98 |
quotas.accept_serial(serial) |
99 |
elif job_status in ["error", "canceled"]:
|
|
99 |
elif job_status in [rapi.JOB_STATUS_ERROR, rapi.JOB_STATUS_CANCELED]:
|
|
100 | 100 |
log.debug("Job %s failed. Rejecting related serial %s", job_id, |
101 | 101 |
serial) |
102 | 102 |
quotas.reject_serial(serial) |
103 | 103 |
vm.serial = None |
104 |
elif job_status == "success" and commission_info is not None:
|
|
104 |
elif job_status == rapi.JOB_STATUS_SUCCESS and commission_info is not None:
|
|
105 | 105 |
log.debug("Expected job was %s. Processing job %s. Commission for" |
106 | 106 |
" this job: %s", vm.task_job_id, job_id, commission_info) |
107 | 107 |
# Commission for this change has not been issued, or the issued |
... | ... | |
139 | 139 |
vm.backendopcode = opcode |
140 | 140 |
vm.backendlogmsg = logmsg |
141 | 141 |
|
142 |
if status in ["queued", "waiting", "running"]:
|
|
142 |
if status not in rapi.JOB_STATUS_FINALIZED:
|
|
143 | 143 |
vm.save() |
144 | 144 |
return |
145 | 145 |
|
... | ... | |
148 | 148 |
state_for_success = VirtualMachine.OPER_STATE_FROM_OPCODE.get(opcode) |
149 | 149 |
|
150 | 150 |
# Notifications of success change the operating state |
151 |
if status == "success":
|
|
151 |
if status == rapi.JOB_STATUS_SUCCESS:
|
|
152 | 152 |
if state_for_success is not None: |
153 | 153 |
vm.operstate = state_for_success |
154 | 154 |
beparams = job_fields.get("beparams", None) |
... | ... | |
162 | 162 |
# in reversed order. |
163 | 163 |
vm.backendtime = etime |
164 | 164 |
|
165 |
if status in ["success", "error", "canceled"] and nics is not None:
|
|
165 |
if status in rapi.JOB_STATUS_FINALIZED and nics is not None:
|
|
166 | 166 |
# Update the NICs of the VM |
167 | 167 |
_process_net_status(vm, etime, nics) |
168 | 168 |
|
169 | 169 |
# Special case: if OP_INSTANCE_CREATE fails --> ERROR |
170 |
if opcode == 'OP_INSTANCE_CREATE' and status in ('canceled', 'error'): |
|
170 |
if opcode == 'OP_INSTANCE_CREATE' and status in (rapi.JOB_STATUS_CANCELED, |
|
171 |
rapi.JOB_STATUS_ERROR): |
|
171 | 172 |
vm.operstate = 'ERROR' |
172 | 173 |
vm.backendtime = etime |
173 | 174 |
# Update state of associated NICs |
... | ... | |
176 | 177 |
# Special case: OP_INSTANCE_REMOVE fails for machines in ERROR, |
177 | 178 |
# when no instance exists at the Ganeti backend. |
178 | 179 |
# See ticket #799 for all the details. |
179 |
if status == 'success' or (status == 'error' and
|
|
180 |
not vm_exists_in_backend(vm)):
|
|
180 |
if (status == rapi.JOB_STATUS_SUCCESS or
|
|
181 |
(status == rapi.JOB_STATUS_ERROR and not vm_exists_in_backend(vm))):
|
|
181 | 182 |
# VM has been deleted |
182 | 183 |
for nic in vm.nics.all(): |
183 | 184 |
# Release the IP |
... | ... | |
187 | 188 |
vm.deleted = True |
188 | 189 |
vm.operstate = state_for_success |
189 | 190 |
vm.backendtime = etime |
190 |
status = "success"
|
|
191 |
status = rapi.JOB_STATUS_SUCCESS
|
|
191 | 192 |
|
192 |
if status in ["success", "error", "canceled"]:
|
|
193 |
if status in rapi.JOB_STATUS_FINALIZED:
|
|
193 | 194 |
# Job is finalized: Handle quotas/commissioning |
194 | 195 |
vm = handle_vm_quotas(vm, job_id=jobid, job_opcode=opcode, |
195 | 196 |
job_status=status, job_fields=job_fields) |
... | ... | |
436 | 437 |
|
437 | 438 |
# Notifications of success change the operating state |
438 | 439 |
state_for_success = BackendNetwork.OPER_STATE_FROM_OPCODE.get(opcode, None) |
439 |
if status == 'success' and state_for_success is not None:
|
|
440 |
if status == rapi.JOB_STATUS_SUCCESS and state_for_success is not None:
|
|
440 | 441 |
back_network.operstate = state_for_success |
441 | 442 |
|
442 |
if status in ('canceled', 'error') and opcode == 'OP_NETWORK_ADD': |
|
443 |
if (status in (rapi.JOB_STATUS_CANCELED, rapi.JOB_STATUS_ERROR) |
|
444 |
and opcode == 'OP_NETWORK_ADD'): |
|
443 | 445 |
back_network.operstate = 'ERROR' |
444 | 446 |
back_network.backendtime = etime |
445 | 447 |
|
446 | 448 |
if opcode == 'OP_NETWORK_REMOVE': |
447 |
network_is_deleted = (status == "success")
|
|
448 |
if network_is_deleted or (status == "error" and not
|
|
449 |
network_is_deleted = (status == rapi.JOB_STATUS_SUCCESS)
|
|
450 |
if network_is_deleted or (status == rapi.JOB_STATUS_ERROR and not
|
|
449 | 451 |
network_exists_in_backend(back_network)): |
450 | 452 |
back_network.operstate = state_for_success |
451 | 453 |
back_network.deleted = True |
452 | 454 |
back_network.backendtime = etime |
453 | 455 |
|
454 |
if status == 'success':
|
|
456 |
if status == rapi.JOB_STATUS_SUCCESS:
|
|
455 | 457 |
back_network.backendtime = etime |
456 | 458 |
back_network.save() |
457 | 459 |
# Also you must update the state of the Network!! |
... | ... | |
547 | 549 |
for ip in add_reserved_ips: |
548 | 550 |
network.reserve_address(ip, external=True) |
549 | 551 |
|
550 |
if status == 'success':
|
|
552 |
if status == rapi.JOB_STATUS_SUCCESS:
|
|
551 | 553 |
back_network.backendtime = etime |
552 | 554 |
back_network.save() |
553 | 555 |
|
... | ... | |
631 | 633 |
"network": nic.network.backend_id, |
632 | 634 |
"ip": nic.ipv4_address} |
633 | 635 |
for nic in nics] |
636 |
|
|
634 | 637 |
backend = vm.backend |
635 | 638 |
depend_jobs = [] |
636 | 639 |
for nic in nics: |
637 |
network = Network.objects.select_for_update().get(id=nic.network_id) |
|
638 |
bnet, created = BackendNetwork.objects.get_or_create(backend=backend, |
|
639 |
network=network) |
|
640 |
if bnet.operstate != "ACTIVE": |
|
641 |
depend_jobs = create_network(network, backend, connect=True) |
|
642 |
kw["depends"] = [[job, ["success", "error", "canceled"]] |
|
643 |
for job in depend_jobs] |
|
640 |
bnet, job_ids = ensure_network_is_active(backend, nic.network_id) |
|
641 |
depend_jobs.extend(job_ids) |
|
642 |
|
|
643 |
kw["depends"] = create_job_dependencies(depend_jobs) |
|
644 | 644 |
|
645 | 645 |
# Defined in settings.GANETI_CREATEINSTANCE_KWARGS |
646 | 646 |
# kw['os'] = settings.GANETI_OS_PROVIDER |
... | ... | |
753 | 753 |
try: |
754 | 754 |
get_instance_info(vm) |
755 | 755 |
return True |
756 |
except GanetiApiError as e: |
|
756 |
except rapi.GanetiApiError as e:
|
|
757 | 757 |
if e.code == 404: |
758 | 758 |
return False |
759 | 759 |
raise e |
... | ... | |
768 | 768 |
try: |
769 | 769 |
get_network_info(backend_network) |
770 | 770 |
return True |
771 |
except GanetiApiError as e: |
|
771 |
except rapi.GanetiApiError as e:
|
|
772 | 772 |
if e.code == 404: |
773 | 773 |
return False |
774 | 774 |
|
775 | 775 |
|
776 |
def ensure_network_is_active(backend, network_id): |
|
777 |
"""Ensure that a network is active in the specified backend |
|
778 |
|
|
779 |
Check that a network exists and is active in the specified backend. If not |
|
780 |
(re-)create the network. Return the corresponding BackendNetwork object |
|
781 |
and the IDs of the Ganeti job to create the network. |
|
782 |
|
|
783 |
""" |
|
784 |
network = Network.objects.select_for_update().get(id=network_id) |
|
785 |
bnet, created = BackendNetwork.objects.get_or_create(backend=backend, |
|
786 |
network=network) |
|
787 |
job_ids = [] |
|
788 |
if bnet.operstate != "ACTIVE": |
|
789 |
job_ids = create_network(network, backend, connect=True) |
|
790 |
|
|
791 |
return bnet, job_ids |
|
792 |
|
|
793 |
|
|
776 | 794 |
def create_network(network, backend, connect=True): |
777 | 795 |
"""Create a network in a Ganeti backend""" |
778 | 796 |
log.debug("Creating network %s in backend %s", network, backend) |
... | ... | |
845 | 863 |
else: |
846 | 864 |
conflicts_check = False |
847 | 865 |
|
848 |
depends = [[job, ["success", "error", "canceled"]] for job in depends]
|
|
866 |
depends = create_job_dependencies(depends)
|
|
849 | 867 |
with pooled_rapi_client(backend) as client: |
850 | 868 |
groups = [group] if group is not None else client.GetGroups() |
851 | 869 |
job_ids = [] |
... | ... | |
868 | 886 |
|
869 | 887 |
|
870 | 888 |
def _delete_network(network, backend, depends=[]): |
871 |
depends = [[job, ["success", "error", "canceled"]] for job in depends]
|
|
889 |
depends = create_job_dependencies(depends)
|
|
872 | 890 |
with pooled_rapi_client(backend) as client: |
873 | 891 |
return client.DeleteNetwork(network.backend_id, depends) |
874 | 892 |
|
... | ... | |
888 | 906 |
def connect_to_network(vm, nic): |
889 | 907 |
network = nic.network |
890 | 908 |
backend = vm.backend |
891 |
network = Network.objects.select_for_update().get(id=network.id) |
|
892 |
bnet, created = BackendNetwork.objects.get_or_create(backend=backend, |
|
893 |
network=network) |
|
894 |
depend_jobs = [] |
|
895 |
if bnet.operstate != "ACTIVE": |
|
896 |
depend_jobs = create_network(network, backend, connect=True) |
|
909 |
bnet, depend_jobs = ensure_network_is_active(backend, network.id) |
|
897 | 910 |
|
898 |
depends = [[job, ["success", "error", "canceled"]] for job in depend_jobs]
|
|
911 |
depends = create_job_dependencies(depend_jobs)
|
|
899 | 912 |
|
900 | 913 |
nic = {'name': nic.backend_uuid, |
901 | 914 |
'network': network.backend_id, |
... | ... | |
1071 | 1084 |
|
1072 | 1085 |
def create_network_synced(network, backend): |
1073 | 1086 |
result = _create_network_synced(network, backend) |
1074 |
if result[0] != 'success':
|
|
1087 |
if result[0] != rapi.JOB_STATUS_SUCCESS:
|
|
1075 | 1088 |
return result |
1076 | 1089 |
result = connect_network_synced(network, backend) |
1077 | 1090 |
return result |
... | ... | |
1090 | 1103 |
job = client.ConnectNetwork(network.backend_id, group, |
1091 | 1104 |
network.mode, network.link) |
1092 | 1105 |
result = wait_for_job(client, job) |
1093 |
if result[0] != 'success':
|
|
1106 |
if result[0] != rapi.JOB_STATUS_SUCCESS:
|
|
1094 | 1107 |
return result |
1095 | 1108 |
|
1096 | 1109 |
return result |
... | ... | |
1099 | 1112 |
def wait_for_job(client, jobid): |
1100 | 1113 |
result = client.WaitForJobChange(jobid, ['status', 'opresult'], None, None) |
1101 | 1114 |
status = result['job_info'][0] |
1102 |
while status not in ['success', 'error', 'cancel']:
|
|
1115 |
while status not in rapi.JOB_STATUS_FINALIZED:
|
|
1103 | 1116 |
result = client.WaitForJobChange(jobid, ['status', 'opresult'], |
1104 | 1117 |
[result], None) |
1105 | 1118 |
status = result['job_info'][0] |
1106 | 1119 |
|
1107 |
if status == 'success':
|
|
1120 |
if status == rapi.JOB_STATUS_SUCCESS:
|
|
1108 | 1121 |
return (status, None) |
1109 | 1122 |
else: |
1110 | 1123 |
error = result['job_info'][1] |
1111 | 1124 |
return (status, error) |
1125 |
|
|
1126 |
|
|
1127 |
def create_job_dependencies(job_ids=[], job_states=None): |
|
1128 |
"""Transform a list of job IDs to Ganeti 'depends' attribute.""" |
|
1129 |
if job_states is None: |
|
1130 |
job_states = list(rapi.JOB_STATUS_FINALIZED) |
|
1131 |
assert(type(job_states) == list) |
|
1132 |
return [[job_id, job_states] for job_id in job_ids] |
Also available in: Unified diff