42 |
42 |
from synnefo import quotas
|
43 |
43 |
from synnefo.api.util import release_resource
|
44 |
44 |
from synnefo.util.mac2eui64 import mac2eui64
|
45 |
|
from synnefo.logic.rapi import GanetiApiError
|
|
45 |
from synnefo.logic import rapi
|
46 |
46 |
|
47 |
47 |
from logging import getLogger
|
48 |
48 |
log = getLogger(__name__)
|
... | ... | |
76 |
76 |
rejected, since they reflect a previous state of the VM.
|
77 |
77 |
|
78 |
78 |
"""
|
79 |
|
if job_status not in ["success", "error", "canceled"]:
|
|
79 |
if job_status not in rapi.JOB_STATUS_FINALIZED:
|
80 |
80 |
return vm
|
81 |
81 |
|
82 |
82 |
# Check successful completion of a job will trigger any quotable change in
|
... | ... | |
94 |
94 |
# if fails, must be accepted, as the user must manually remove the
|
95 |
95 |
# failed server
|
96 |
96 |
serial = vm.serial
|
97 |
|
if job_status == "success":
|
|
97 |
if job_status == rapi.JOB_STATUS_SUCCESS:
|
98 |
98 |
quotas.accept_serial(serial)
|
99 |
|
elif job_status in ["error", "canceled"]:
|
|
99 |
elif job_status in [rapi.JOB_STATUS_ERROR, rapi.JOB_STATUS_CANCELED]:
|
100 |
100 |
log.debug("Job %s failed. Rejecting related serial %s", job_id,
|
101 |
101 |
serial)
|
102 |
102 |
quotas.reject_serial(serial)
|
103 |
103 |
vm.serial = None
|
104 |
|
elif job_status == "success" and commission_info is not None:
|
|
104 |
elif job_status == rapi.JOB_STATUS_SUCCESS and commission_info is not None:
|
105 |
105 |
log.debug("Expected job was %s. Processing job %s. Commission for"
|
106 |
106 |
" this job: %s", vm.task_job_id, job_id, commission_info)
|
107 |
107 |
# Commission for this change has not been issued, or the issued
|
... | ... | |
139 |
139 |
vm.backendopcode = opcode
|
140 |
140 |
vm.backendlogmsg = logmsg
|
141 |
141 |
|
142 |
|
if status in ["queued", "waiting", "running"]:
|
|
142 |
if status not in rapi.JOB_STATUS_FINALIZED:
|
143 |
143 |
vm.save()
|
144 |
144 |
return
|
145 |
145 |
|
... | ... | |
148 |
148 |
state_for_success = VirtualMachine.OPER_STATE_FROM_OPCODE.get(opcode)
|
149 |
149 |
|
150 |
150 |
# Notifications of success change the operating state
|
151 |
|
if status == "success":
|
|
151 |
if status == rapi.JOB_STATUS_SUCCESS:
|
152 |
152 |
if state_for_success is not None:
|
153 |
153 |
vm.operstate = state_for_success
|
154 |
154 |
beparams = job_fields.get("beparams", None)
|
... | ... | |
162 |
162 |
# in reversed order.
|
163 |
163 |
vm.backendtime = etime
|
164 |
164 |
|
165 |
|
if status in ["success", "error", "canceled"] and nics is not None:
|
|
165 |
if status in rapi.JOB_STATUS_FINALIZED and nics is not None:
|
166 |
166 |
# Update the NICs of the VM
|
167 |
167 |
_process_net_status(vm, etime, nics)
|
168 |
168 |
|
169 |
169 |
# Special case: if OP_INSTANCE_CREATE fails --> ERROR
|
170 |
|
if opcode == 'OP_INSTANCE_CREATE' and status in ('canceled', 'error'):
|
|
170 |
if opcode == 'OP_INSTANCE_CREATE' and status in (rapi.JOB_STATUS_CANCELED,
|
|
171 |
rapi.JOB_STATUS_ERROR):
|
171 |
172 |
vm.operstate = 'ERROR'
|
172 |
173 |
vm.backendtime = etime
|
173 |
174 |
# Update state of associated NICs
|
... | ... | |
176 |
177 |
# Special case: OP_INSTANCE_REMOVE fails for machines in ERROR,
|
177 |
178 |
# when no instance exists at the Ganeti backend.
|
178 |
179 |
# See ticket #799 for all the details.
|
179 |
|
if status == 'success' or (status == 'error' and
|
180 |
|
not vm_exists_in_backend(vm)):
|
|
180 |
if (status == rapi.JOB_STATUS_SUCCESS or
|
|
181 |
(status == rapi.JOB_STATUS_ERROR and not vm_exists_in_backend(vm))):
|
181 |
182 |
# VM has been deleted
|
182 |
183 |
for nic in vm.nics.all():
|
183 |
184 |
# Release the IP
|
... | ... | |
187 |
188 |
vm.deleted = True
|
188 |
189 |
vm.operstate = state_for_success
|
189 |
190 |
vm.backendtime = etime
|
190 |
|
status = "success"
|
|
191 |
status = rapi.JOB_STATUS_SUCCESS
|
191 |
192 |
|
192 |
|
if status in ["success", "error", "canceled"]:
|
|
193 |
if status in rapi.JOB_STATUS_FINALIZED:
|
193 |
194 |
# Job is finalized: Handle quotas/commissioning
|
194 |
195 |
vm = handle_vm_quotas(vm, job_id=jobid, job_opcode=opcode,
|
195 |
196 |
job_status=status, job_fields=job_fields)
|
... | ... | |
436 |
437 |
|
437 |
438 |
# Notifications of success change the operating state
|
438 |
439 |
state_for_success = BackendNetwork.OPER_STATE_FROM_OPCODE.get(opcode, None)
|
439 |
|
if status == 'success' and state_for_success is not None:
|
|
440 |
if status == rapi.JOB_STATUS_SUCCESS and state_for_success is not None:
|
440 |
441 |
back_network.operstate = state_for_success
|
441 |
442 |
|
442 |
|
if status in ('canceled', 'error') and opcode == 'OP_NETWORK_ADD':
|
|
443 |
if (status in (rapi.JOB_STATUS_CANCELED, rapi.JOB_STATUS_ERROR)
|
|
444 |
and opcode == 'OP_NETWORK_ADD'):
|
443 |
445 |
back_network.operstate = 'ERROR'
|
444 |
446 |
back_network.backendtime = etime
|
445 |
447 |
|
446 |
448 |
if opcode == 'OP_NETWORK_REMOVE':
|
447 |
|
network_is_deleted = (status == "success")
|
448 |
|
if network_is_deleted or (status == "error" and not
|
|
449 |
network_is_deleted = (status == rapi.JOB_STATUS_SUCCESS)
|
|
450 |
if network_is_deleted or (status == rapi.JOB_STATUS_ERROR and not
|
449 |
451 |
network_exists_in_backend(back_network)):
|
450 |
452 |
back_network.operstate = state_for_success
|
451 |
453 |
back_network.deleted = True
|
452 |
454 |
back_network.backendtime = etime
|
453 |
455 |
|
454 |
|
if status == 'success':
|
|
456 |
if status == rapi.JOB_STATUS_SUCCESS:
|
455 |
457 |
back_network.backendtime = etime
|
456 |
458 |
back_network.save()
|
457 |
459 |
# Also you must update the state of the Network!!
|
... | ... | |
547 |
549 |
for ip in add_reserved_ips:
|
548 |
550 |
network.reserve_address(ip, external=True)
|
549 |
551 |
|
550 |
|
if status == 'success':
|
|
552 |
if status == rapi.JOB_STATUS_SUCCESS:
|
551 |
553 |
back_network.backendtime = etime
|
552 |
554 |
back_network.save()
|
553 |
555 |
|
... | ... | |
631 |
633 |
"network": nic.network.backend_id,
|
632 |
634 |
"ip": nic.ipv4_address}
|
633 |
635 |
for nic in nics]
|
|
636 |
|
634 |
637 |
backend = vm.backend
|
635 |
638 |
depend_jobs = []
|
636 |
639 |
for nic in nics:
|
637 |
|
network = Network.objects.select_for_update().get(id=nic.network_id)
|
638 |
|
bnet, created = BackendNetwork.objects.get_or_create(backend=backend,
|
639 |
|
network=network)
|
640 |
|
if bnet.operstate != "ACTIVE":
|
641 |
|
depend_jobs = create_network(network, backend, connect=True)
|
642 |
|
kw["depends"] = [[job, ["success", "error", "canceled"]]
|
643 |
|
for job in depend_jobs]
|
|
640 |
bnet, job_ids = ensure_network_is_active(backend, nic.network_id)
|
|
641 |
depend_jobs.extend(job_ids)
|
|
642 |
|
|
643 |
kw["depends"] = create_job_dependencies(depend_jobs)
|
644 |
644 |
|
645 |
645 |
# Defined in settings.GANETI_CREATEINSTANCE_KWARGS
|
646 |
646 |
# kw['os'] = settings.GANETI_OS_PROVIDER
|
... | ... | |
753 |
753 |
try:
|
754 |
754 |
get_instance_info(vm)
|
755 |
755 |
return True
|
756 |
|
except GanetiApiError as e:
|
|
756 |
except rapi.GanetiApiError as e:
|
757 |
757 |
if e.code == 404:
|
758 |
758 |
return False
|
759 |
759 |
raise e
|
... | ... | |
768 |
768 |
try:
|
769 |
769 |
get_network_info(backend_network)
|
770 |
770 |
return True
|
771 |
|
except GanetiApiError as e:
|
|
771 |
except rapi.GanetiApiError as e:
|
772 |
772 |
if e.code == 404:
|
773 |
773 |
return False
|
774 |
774 |
|
775 |
775 |
|
|
776 |
def ensure_network_is_active(backend, network_id):
|
|
777 |
"""Ensure that a network is active in the specified backend
|
|
778 |
|
|
779 |
Check that a network exists and is active in the specified backend. If not
|
|
780 |
(re-)create the network. Return the corresponding BackendNetwork object
|
|
781 |
and the IDs of the Ganeti job to create the network.
|
|
782 |
|
|
783 |
"""
|
|
784 |
network = Network.objects.select_for_update().get(id=network_id)
|
|
785 |
bnet, created = BackendNetwork.objects.get_or_create(backend=backend,
|
|
786 |
network=network)
|
|
787 |
job_ids = []
|
|
788 |
if bnet.operstate != "ACTIVE":
|
|
789 |
job_ids = create_network(network, backend, connect=True)
|
|
790 |
|
|
791 |
return bnet, job_ids
|
|
792 |
|
|
793 |
|
776 |
794 |
def create_network(network, backend, connect=True):
|
777 |
795 |
"""Create a network in a Ganeti backend"""
|
778 |
796 |
log.debug("Creating network %s in backend %s", network, backend)
|
... | ... | |
845 |
863 |
else:
|
846 |
864 |
conflicts_check = False
|
847 |
865 |
|
848 |
|
depends = [[job, ["success", "error", "canceled"]] for job in depends]
|
|
866 |
depends = create_job_dependencies(depends)
|
849 |
867 |
with pooled_rapi_client(backend) as client:
|
850 |
868 |
groups = [group] if group is not None else client.GetGroups()
|
851 |
869 |
job_ids = []
|
... | ... | |
868 |
886 |
|
869 |
887 |
|
870 |
888 |
def _delete_network(network, backend, depends=[]):
|
871 |
|
depends = [[job, ["success", "error", "canceled"]] for job in depends]
|
|
889 |
depends = create_job_dependencies(depends)
|
872 |
890 |
with pooled_rapi_client(backend) as client:
|
873 |
891 |
return client.DeleteNetwork(network.backend_id, depends)
|
874 |
892 |
|
... | ... | |
888 |
906 |
def connect_to_network(vm, nic):
|
889 |
907 |
network = nic.network
|
890 |
908 |
backend = vm.backend
|
891 |
|
network = Network.objects.select_for_update().get(id=network.id)
|
892 |
|
bnet, created = BackendNetwork.objects.get_or_create(backend=backend,
|
893 |
|
network=network)
|
894 |
|
depend_jobs = []
|
895 |
|
if bnet.operstate != "ACTIVE":
|
896 |
|
depend_jobs = create_network(network, backend, connect=True)
|
|
909 |
bnet, depend_jobs = ensure_network_is_active(backend, network.id)
|
897 |
910 |
|
898 |
|
depends = [[job, ["success", "error", "canceled"]] for job in depend_jobs]
|
|
911 |
depends = create_job_dependencies(depend_jobs)
|
899 |
912 |
|
900 |
913 |
nic = {'name': nic.backend_uuid,
|
901 |
914 |
'network': network.backend_id,
|
... | ... | |
1071 |
1084 |
|
1072 |
1085 |
def create_network_synced(network, backend):
|
1073 |
1086 |
result = _create_network_synced(network, backend)
|
1074 |
|
if result[0] != 'success':
|
|
1087 |
if result[0] != rapi.JOB_STATUS_SUCCESS:
|
1075 |
1088 |
return result
|
1076 |
1089 |
result = connect_network_synced(network, backend)
|
1077 |
1090 |
return result
|
... | ... | |
1090 |
1103 |
job = client.ConnectNetwork(network.backend_id, group,
|
1091 |
1104 |
network.mode, network.link)
|
1092 |
1105 |
result = wait_for_job(client, job)
|
1093 |
|
if result[0] != 'success':
|
|
1106 |
if result[0] != rapi.JOB_STATUS_SUCCESS:
|
1094 |
1107 |
return result
|
1095 |
1108 |
|
1096 |
1109 |
return result
|
... | ... | |
1099 |
1112 |
def wait_for_job(client, jobid):
|
1100 |
1113 |
result = client.WaitForJobChange(jobid, ['status', 'opresult'], None, None)
|
1101 |
1114 |
status = result['job_info'][0]
|
1102 |
|
while status not in ['success', 'error', 'cancel']:
|
|
1115 |
while status not in rapi.JOB_STATUS_FINALIZED:
|
1103 |
1116 |
result = client.WaitForJobChange(jobid, ['status', 'opresult'],
|
1104 |
1117 |
[result], None)
|
1105 |
1118 |
status = result['job_info'][0]
|
1106 |
1119 |
|
1107 |
|
if status == 'success':
|
|
1120 |
if status == rapi.JOB_STATUS_SUCCESS:
|
1108 |
1121 |
return (status, None)
|
1109 |
1122 |
else:
|
1110 |
1123 |
error = result['job_info'][1]
|
1111 |
1124 |
return (status, error)
|
|
1125 |
|
|
1126 |
|
|
1127 |
def create_job_dependencies(job_ids=[], job_states=None):
|
|
1128 |
"""Transform a list of job IDs to Ganeti 'depends' attribute."""
|
|
1129 |
if job_states is None:
|
|
1130 |
job_states = list(rapi.JOB_STATUS_FINALIZED)
|
|
1131 |
assert(type(job_states) == list)
|
|
1132 |
return [[job_id, job_states] for job_id in job_ids]
|