58 |
58 |
SIMPLE_NIC_FIELDS = ["state", "mac", "network", "firewall_profile", "index"]
|
59 |
59 |
COMPLEX_NIC_FIELDS = ["ipv4_address", "ipv6_address"]
|
60 |
60 |
NIC_FIELDS = SIMPLE_NIC_FIELDS + COMPLEX_NIC_FIELDS
|
|
61 |
DISK_FIELDS = ["status", "size", "index"]
|
61 |
62 |
UNKNOWN_NIC_PREFIX = "unknown-"
|
|
63 |
UNKNOWN_DISK_PREFIX = "unknown-"
|
62 |
64 |
|
63 |
65 |
|
64 |
66 |
def handle_vm_quotas(vm, job_id, job_opcode, job_status, job_fields):
|
... | ... | |
119 |
121 |
|
120 |
122 |
@transaction.commit_on_success
|
121 |
123 |
def process_op_status(vm, etime, jobid, opcode, status, logmsg, nics=None,
|
122 |
|
job_fields=None):
|
|
124 |
disks=None, job_fields=None):
|
123 |
125 |
"""Process a job progress notification from the backend
|
124 |
126 |
|
125 |
127 |
Process an incoming message from the backend (currently Ganeti).
|
... | ... | |
165 |
167 |
# in reversed order.
|
166 |
168 |
vm.backendtime = etime
|
167 |
169 |
|
168 |
|
if status in rapi.JOB_STATUS_FINALIZED and nics is not None:
|
169 |
|
# Update the NICs of the VM
|
170 |
|
_process_net_status(vm, etime, nics)
|
|
170 |
if status in rapi.JOB_STATUS_FINALIZED:
|
|
171 |
if nics is not None: # Update the NICs of the VM
|
|
172 |
_process_net_status(vm, etime, nics)
|
|
173 |
if disks is not None: # Update the disks of the VM
|
|
174 |
_process_disks_status(vm, etime, disks)
|
171 |
175 |
|
172 |
176 |
# Special case: if OP_INSTANCE_CREATE fails --> ERROR
|
173 |
177 |
if opcode == 'OP_INSTANCE_CREATE' and status in (rapi.JOB_STATUS_CANCELED,
|
174 |
178 |
rapi.JOB_STATUS_ERROR):
|
175 |
179 |
new_operstate = "ERROR"
|
176 |
180 |
vm.backendtime = etime
|
177 |
|
# Update state of associated NICs
|
|
181 |
# Update state of associated attachments
|
178 |
182 |
vm.nics.all().update(state="ERROR")
|
|
183 |
vm.volumes.all().update(status="ERROR")
|
179 |
184 |
elif opcode == 'OP_INSTANCE_REMOVE':
|
180 |
185 |
# Special case: OP_INSTANCE_REMOVE fails for machines in ERROR,
|
181 |
186 |
# when no instance exists at the Ganeti backend.
|
182 |
187 |
# See ticket #799 for all the details.
|
183 |
188 |
if (status == rapi.JOB_STATUS_SUCCESS or
|
184 |
189 |
(status == rapi.JOB_STATUS_ERROR and not vm_exists_in_backend(vm))):
|
185 |
|
# VM has been deleted
|
|
190 |
# server has been deleted, so delete the server's attachments
|
|
191 |
vm.volumes.all().update(deleted=True, machine=None)
|
186 |
192 |
for nic in vm.nics.all():
|
187 |
|
# Release the IP
|
|
193 |
# but first release the IP
|
188 |
194 |
remove_nic_ips(nic)
|
189 |
|
# And delete the NIC.
|
190 |
195 |
nic.delete()
|
191 |
196 |
vm.deleted = True
|
192 |
197 |
new_operstate = state_for_success
|
193 |
198 |
vm.backendtime = etime
|
194 |
199 |
status = rapi.JOB_STATUS_SUCCESS
|
195 |
|
#status = "success"
|
196 |
|
vm.volumes.all().update(deleted=True, machine=None)
|
197 |
200 |
|
198 |
201 |
if status in rapi.JOB_STATUS_FINALIZED:
|
199 |
202 |
# Job is finalized: Handle quotas/commissioning
|
... | ... | |
427 |
430 |
|
428 |
431 |
|
429 |
432 |
@transaction.commit_on_success
|
|
433 |
def process_disks_status(vm, etime, disks):
|
|
434 |
"""Wrap _process_disks_status inside transaction."""
|
|
435 |
_process_disks_status(vm, etime, disks)
|
|
436 |
|
|
437 |
|
|
438 |
def _process_disks_status(vm, etime, disks):
|
|
439 |
"""Process a disks status notification from the backend
|
|
440 |
|
|
441 |
Process an incoming message from the Ganeti backend,
|
|
442 |
detailing the disk configuration of a VM instance.
|
|
443 |
|
|
444 |
Update the state of the VM in the DB accordingly.
|
|
445 |
|
|
446 |
"""
|
|
447 |
ganeti_disks = process_ganeti_disks(disks)
|
|
448 |
db_disks = dict([(disk.id, disk)
|
|
449 |
for disk in vm.volumes.filter(deleted=False)])
|
|
450 |
|
|
451 |
for disk_name in set(db_disks.keys()) | set(ganeti_disks.keys()):
|
|
452 |
db_disk = db_disks.get(disk_name)
|
|
453 |
ganeti_disk = ganeti_disks.get(disk_name)
|
|
454 |
if ganeti_disk is None:
|
|
455 |
if disk_is_stale(vm, disk):
|
|
456 |
log.debug("Removing stale disk '%s'" % db_disk)
|
|
457 |
# TODO: Handle disk deletion
|
|
458 |
db_disk.deleted = True
|
|
459 |
db_disk.save()
|
|
460 |
else:
|
|
461 |
log.info("disk '%s' is still being created" % db_disk)
|
|
462 |
elif db_disk is None:
|
|
463 |
msg = ("disk/%s of VM %s does not exist in DB! Cannot"
|
|
464 |
" automatically fix this issue!" % (disk_name, vm))
|
|
465 |
log.error(msg)
|
|
466 |
continue
|
|
467 |
elif not disks_are_equal(db_disk, ganeti_disk):
|
|
468 |
for f in DISK_FIELDS:
|
|
469 |
# Update the disk in DB with the values from Ganeti disk
|
|
470 |
setattr(db_disk, f, ganeti_disk[f])
|
|
471 |
db_disk.save()
|
|
472 |
|
|
473 |
# TODO: Special case where the size of the disk has changed!!
|
|
474 |
assert(ganeti_disk["size"] == db_disk.size)
|
|
475 |
|
|
476 |
vm.backendtime = etime
|
|
477 |
vm.save()
|
|
478 |
|
|
479 |
|
|
480 |
def disks_are_equal(db_disk, gnt_disk):
|
|
481 |
for field in DISK_FIELDS:
|
|
482 |
if getattr(db_disk, field) != gnt_disk[field]:
|
|
483 |
return False
|
|
484 |
return True
|
|
485 |
|
|
486 |
|
|
487 |
def process_ganeti_disks(ganeti_disks):
|
|
488 |
"""Process disk dict from ganeti"""
|
|
489 |
new_disks = []
|
|
490 |
for index, gdisk in enumerate(ganeti_disks):
|
|
491 |
disk_name = gdisk.get("name", None)
|
|
492 |
if disk_name is not None:
|
|
493 |
disk_id = utils.id_from_disk_name(disk_name)
|
|
494 |
else:
|
|
495 |
# Put as default value the index. If it is an unknown disk to
|
|
496 |
# synnefo it will be created automaticaly.
|
|
497 |
disk_id = UNKNOWN_DISK_PREFIX + str(index)
|
|
498 |
|
|
499 |
# Get disk size in GB
|
|
500 |
size = gdisk.get("size") >> 10
|
|
501 |
|
|
502 |
disk_info = {
|
|
503 |
'index': index,
|
|
504 |
'size': size,
|
|
505 |
'status': "IN_USE"}
|
|
506 |
|
|
507 |
new_disks.append((disk_id, disk_info))
|
|
508 |
return dict(new_disks)
|
|
509 |
|
|
510 |
|
|
511 |
@transaction.commit_on_success
|
430 |
512 |
def process_network_status(back_network, etime, jobid, opcode, status, logmsg):
|
431 |
513 |
if status not in [x[0] for x in BACKEND_STATUSES]:
|
432 |
514 |
raise Network.InvalidBackendMsgError(opcode, status)
|
... | ... | |
632 |
714 |
kw['disk_template'] = flavor.disk_template
|
633 |
715 |
disks = []
|
634 |
716 |
for volume in volumes:
|
635 |
|
disk = {}
|
636 |
|
disk["size"] = volume.size * 1024
|
|
717 |
disk = {"name": volume.backend_volume_uuid,
|
|
718 |
"size": volume.size * 1024}
|
637 |
719 |
provider = flavor.disk_provider
|
638 |
720 |
if provider is not None:
|
639 |
721 |
disk["provider"] = provider
|
... | ... | |
807 |
889 |
return False
|
808 |
890 |
|
809 |
891 |
|
|
892 |
def disk_is_stale(vm, disk, timeout=60):
|
|
893 |
"""Check if a disk is stale or exists in the Ganeti backend."""
|
|
894 |
# First check the state of the disk
|
|
895 |
if disk.status == "CREATING":
|
|
896 |
if datetime.now() < disk.created + timedelta(seconds=timeout):
|
|
897 |
# Do not check for too recent disks to avoid the time overhead
|
|
898 |
return False
|
|
899 |
if job_is_still_running(vm, job_id=disk.backendjobid):
|
|
900 |
return False
|
|
901 |
else:
|
|
902 |
# If job has finished, check that the disk exists, because the
|
|
903 |
# message may have been lost or stuck in the queue.
|
|
904 |
vm_info = get_instance_info(vm)
|
|
905 |
if disk.backend_volume_uuid in vm_info["disk.names"]:
|
|
906 |
return False
|
|
907 |
return True
|
|
908 |
|
|
909 |
|
810 |
910 |
def nic_is_stale(vm, nic, timeout=60):
|
811 |
911 |
"""Check if a NIC is stale or exists in the Ganeti backend."""
|
812 |
912 |
# First check the state of the NIC and if there is a pending CONNECT
|