Revision 6a1434d7
b/lib/backend.py | ||
---|---|---|
1280 | 1280 |
_Fail("Failed to accept instance: %s", err, exc=True) |
1281 | 1281 |
|
1282 | 1282 |
|
1283 |
def FinalizeMigration(instance, info, success): |
|
1283 |
def FinalizeMigrationDst(instance, info, success):
|
|
1284 | 1284 |
"""Finalize any preparation to accept an instance. |
1285 | 1285 |
|
1286 | 1286 |
@type instance: L{objects.Instance} |
... | ... | |
1293 | 1293 |
""" |
1294 | 1294 |
hyper = hypervisor.GetHypervisor(instance.hypervisor) |
1295 | 1295 |
try: |
1296 |
hyper.FinalizeMigration(instance, info, success) |
|
1296 |
hyper.FinalizeMigrationDst(instance, info, success)
|
|
1297 | 1297 |
except errors.HypervisorError, err: |
1298 |
_Fail("Failed to finalize migration: %s", err, exc=True) |
|
1298 |
_Fail("Failed to finalize migration on the target node: %s", err, exc=True)
|
|
1299 | 1299 |
|
1300 | 1300 |
|
1301 | 1301 |
def MigrateInstance(instance, target, live): |
... | ... | |
1319 | 1319 |
_Fail("Failed to migrate instance: %s", err, exc=True) |
1320 | 1320 |
|
1321 | 1321 |
|
1322 |
def FinalizeMigrationSource(instance, success, live): |
|
1323 |
"""Finalize the instance migration on the source node. |
|
1324 |
|
|
1325 |
@type instance: L{objects.Instance} |
|
1326 |
@param instance: the instance definition of the migrated instance |
|
1327 |
@type success: bool |
|
1328 |
@param success: whether the migration succeeded or not |
|
1329 |
@type live: bool |
|
1330 |
@param live: whether the user requested a live migration or not |
|
1331 |
@raise RPCFail: If the execution fails for some reason |
|
1332 |
|
|
1333 |
""" |
|
1334 |
hyper = hypervisor.GetHypervisor(instance.hypervisor) |
|
1335 |
|
|
1336 |
try: |
|
1337 |
hyper.FinalizeMigrationSource(instance, success, live) |
|
1338 |
except Exception, err: # pylint: disable=W0703 |
|
1339 |
_Fail("Failed to finalize the migration on the source node: %s", err, |
|
1340 |
exc=True) |
|
1341 |
|
|
1342 |
|
|
1343 |
def GetMigrationStatus(instance): |
|
1344 |
"""Get the migration status |
|
1345 |
|
|
1346 |
@type instance: L{objects.Instance} |
|
1347 |
@param instance: the instance that is being migrated |
|
1348 |
@rtype: L{objects.MigrationStatus} |
|
1349 |
@return: the status of the current migration (one of |
|
1350 |
L{constants.HV_MIGRATION_VALID_STATUSES}), plus any additional |
|
1351 |
progress info that can be retrieved from the hypervisor |
|
1352 |
@raise RPCFail: If the migration status cannot be retrieved |
|
1353 |
|
|
1354 |
""" |
|
1355 |
hyper = hypervisor.GetHypervisor(instance.hypervisor) |
|
1356 |
try: |
|
1357 |
return hyper.GetMigrationStatus(instance) |
|
1358 |
except Exception, err: # pylint: disable=W0703 |
|
1359 |
_Fail("Failed to get migration status: %s", err, exc=True) |
|
1360 |
|
|
1361 |
|
|
1322 | 1362 |
def BlockdevCreate(disk, size, owner, on_primary, info): |
1323 | 1363 |
"""Creates a block device for an instance. |
1324 | 1364 |
|
b/lib/cmdlib.py | ||
---|---|---|
7025 | 7025 |
@ivar shutdown_timeout: In case of failover timeout of the shutdown |
7026 | 7026 |
|
7027 | 7027 |
""" |
7028 |
|
|
7029 |
# Constants |
|
7030 |
_MIGRATION_POLL_INTERVAL = 0.5 |
|
7031 |
|
|
7028 | 7032 |
def __init__(self, lu, instance_name, cleanup=False, |
7029 | 7033 |
failover=False, fallback=False, |
7030 | 7034 |
ignore_consistency=False, |
... | ... | |
7348 | 7352 |
""" |
7349 | 7353 |
instance = self.instance |
7350 | 7354 |
target_node = self.target_node |
7355 |
source_node = self.source_node |
|
7351 | 7356 |
migration_info = self.migration_info |
7352 | 7357 |
|
7353 |
abort_result = self.rpc.call_finalize_migration(target_node,
|
|
7354 |
instance, |
|
7355 |
migration_info, |
|
7356 |
False) |
|
7358 |
abort_result = self.rpc.call_instance_finalize_migration_dst(target_node,
|
|
7359 |
instance,
|
|
7360 |
migration_info,
|
|
7361 |
False)
|
|
7357 | 7362 |
abort_msg = abort_result.fail_msg |
7358 | 7363 |
if abort_msg: |
7359 | 7364 |
logging.error("Aborting migration failed on target node %s: %s", |
... | ... | |
7361 | 7366 |
# Don't raise an exception here, as we stil have to try to revert the |
7362 | 7367 |
# disk status, even if this step failed. |
7363 | 7368 |
|
7369 |
abort_result = self.rpc.call_instance_finalize_migration_src(source_node, |
|
7370 |
instance, False, self.live) |
|
7371 |
abort_msg = abort_result.fail_msg |
|
7372 |
if abort_msg: |
|
7373 |
logging.error("Aborting migration failed on source node %s: %s", |
|
7374 |
source_node, abort_msg) |
|
7375 |
|
|
7364 | 7376 |
def _ExecMigration(self): |
7365 | 7377 |
"""Migrate an instance. |
7366 | 7378 |
|
... | ... | |
7432 | 7444 |
raise errors.OpExecError("Could not migrate instance %s: %s" % |
7433 | 7445 |
(instance.name, msg)) |
7434 | 7446 |
|
7447 |
self.feedback_fn("* starting memory transfer") |
|
7448 |
while True: |
|
7449 |
result = self.rpc.call_instance_get_migration_status(source_node, |
|
7450 |
instance) |
|
7451 |
msg = result.fail_msg |
|
7452 |
ms = result.payload # MigrationStatus instance |
|
7453 |
if msg or (ms.status in constants.HV_MIGRATION_FAILED_STATUSES): |
|
7454 |
logging.error("Instance migration failed, trying to revert" |
|
7455 |
" disk status: %s", msg) |
|
7456 |
self.feedback_fn("Migration failed, aborting") |
|
7457 |
self._AbortMigration() |
|
7458 |
self._RevertDiskStatus() |
|
7459 |
raise errors.OpExecError("Could not migrate instance %s: %s" % |
|
7460 |
(instance.name, msg)) |
|
7461 |
|
|
7462 |
if result.payload.status != constants.HV_MIGRATION_ACTIVE: |
|
7463 |
self.feedback_fn("* memory transfer complete") |
|
7464 |
break |
|
7465 |
|
|
7466 |
time.sleep(self._MIGRATION_POLL_INTERVAL) |
|
7467 |
|
|
7468 |
result = self.rpc.call_instance_finalize_migration_src(source_node, |
|
7469 |
instance, |
|
7470 |
True, |
|
7471 |
self.live) |
|
7472 |
msg = result.fail_msg |
|
7473 |
if msg: |
|
7474 |
logging.error("Instance migration succeeded, but finalization failed" |
|
7475 |
" on the source node: %s", msg) |
|
7476 |
raise errors.OpExecError("Could not finalize instance migration: %s" % |
|
7477 |
msg) |
|
7478 |
|
|
7435 | 7479 |
instance.primary_node = target_node |
7480 |
|
|
7436 | 7481 |
# distribute new instance config to the other nodes |
7437 | 7482 |
self.cfg.Update(instance, self.feedback_fn) |
7438 | 7483 |
|
7439 |
result = self.rpc.call_finalize_migration(target_node,
|
|
7440 |
instance, |
|
7441 |
migration_info, |
|
7442 |
True) |
|
7484 |
result = self.rpc.call_instance_finalize_migration_dst(target_node,
|
|
7485 |
instance,
|
|
7486 |
migration_info,
|
|
7487 |
True)
|
|
7443 | 7488 |
msg = result.fail_msg |
7444 | 7489 |
if msg: |
7445 |
logging.error("Instance migration succeeded, but finalization failed:"
|
|
7446 |
" %s", msg) |
|
7490 |
logging.error("Instance migration succeeded, but finalization failed" |
|
7491 |
" on the target node: %s", msg)
|
|
7447 | 7492 |
raise errors.OpExecError("Could not finalize instance migration: %s" % |
7448 | 7493 |
msg) |
7449 | 7494 |
|
b/lib/constants.py | ||
---|---|---|
797 | 797 |
|
798 | 798 |
HVS_PARAMETERS = frozenset(HVS_PARAMETER_TYPES.keys()) |
799 | 799 |
|
800 |
# Migration statuses |
|
801 |
HV_MIGRATION_COMPLETED = "completed" |
|
802 |
HV_MIGRATION_ACTIVE = "active" |
|
803 |
HV_MIGRATION_FAILED = "failed" |
|
804 |
HV_MIGRATION_CANCELLED = "cancelled" |
|
805 |
|
|
806 |
HV_MIGRATION_VALID_STATUSES = frozenset([ |
|
807 |
HV_MIGRATION_COMPLETED, |
|
808 |
HV_MIGRATION_ACTIVE, |
|
809 |
HV_MIGRATION_FAILED, |
|
810 |
HV_MIGRATION_CANCELLED, |
|
811 |
]) |
|
812 |
|
|
813 |
HV_MIGRATION_FAILED_STATUSES = frozenset([ |
|
814 |
HV_MIGRATION_FAILED, |
|
815 |
HV_MIGRATION_CANCELLED, |
|
816 |
]) |
|
817 |
|
|
818 |
# KVM-specific statuses |
|
819 |
HV_KVM_MIGRATION_VALID_STATUSES = HV_MIGRATION_VALID_STATUSES |
|
820 |
|
|
800 | 821 |
# Backend parameter names |
801 | 822 |
BE_MEMORY = "memory" |
802 | 823 |
BE_VCPUS = "vcpus" |
b/lib/hypervisor/hv_base.py | ||
---|---|---|
291 | 291 |
""" |
292 | 292 |
pass |
293 | 293 |
|
294 |
def FinalizeMigration(self, instance, info, success): |
|
295 |
"""Finalized an instance migration.
|
|
294 |
def FinalizeMigrationDst(self, instance, info, success):
|
|
295 |
"""Finalize the instance migration on the target node.
|
|
296 | 296 |
|
297 | 297 |
Should finalize or revert any preparation done to accept the instance. |
298 | 298 |
Since by default we do no preparation, we also don't have anything to do |
... | ... | |
320 | 320 |
""" |
321 | 321 |
raise NotImplementedError |
322 | 322 |
|
323 |
def FinalizeMigrationSource(self, instance, success, live): |
|
324 |
"""Finalize the instance migration on the source node. |
|
325 |
|
|
326 |
@type instance: L{objects.Instance} |
|
327 |
@param instance: the instance that was migrated |
|
328 |
@type success: bool |
|
329 |
@param success: whether the migration succeeded or not |
|
330 |
@type live: bool |
|
331 |
@param live: whether the user requested a live migration or not |
|
332 |
|
|
333 |
""" |
|
334 |
pass |
|
335 |
|
|
336 |
def GetMigrationStatus(self, instance): |
|
337 |
"""Get the migration status |
|
338 |
|
|
339 |
@type instance: L{objects.Instance} |
|
340 |
@param instance: the instance that is being migrated |
|
341 |
@rtype: L{objects.MigrationStatus} |
|
342 |
@return: the status of the current migration (one of |
|
343 |
L{constants.HV_MIGRATION_VALID_STATUSES}), plus any additional |
|
344 |
progress info that can be retrieved from the hypervisor |
|
345 |
|
|
346 |
""" |
|
347 |
raise NotImplementedError |
|
348 |
|
|
323 | 349 |
@classmethod |
324 | 350 |
def CheckParameterSyntax(cls, hvparams): |
325 | 351 |
"""Check the given parameters for validity. |
b/lib/hypervisor/hv_kvm.py | ||
---|---|---|
1574 | 1574 |
incoming_address = (target, instance.hvparams[constants.HV_MIGRATION_PORT]) |
1575 | 1575 |
self._ExecuteKVMRuntime(instance, kvm_runtime, incoming=incoming_address) |
1576 | 1576 |
|
1577 |
def FinalizeMigration(self, instance, info, success): |
|
1578 |
"""Finalize an instance migration.
|
|
1577 |
def FinalizeMigrationDst(self, instance, info, success):
|
|
1578 |
"""Finalize the instance migration on the target node.
|
|
1579 | 1579 |
|
1580 | 1580 |
Stop the incoming mode KVM. |
1581 | 1581 |
|
... | ... | |
1622 | 1622 |
""" |
1623 | 1623 |
instance_name = instance.name |
1624 | 1624 |
port = instance.hvparams[constants.HV_MIGRATION_PORT] |
1625 |
pidfile, pid, alive = self._InstancePidAlive(instance_name)
|
|
1625 |
_, _, alive = self._InstancePidAlive(instance_name)
|
|
1626 | 1626 |
if not alive: |
1627 | 1627 |
raise errors.HypervisorError("Instance not running, cannot migrate") |
1628 | 1628 |
|
... | ... | |
1640 | 1640 |
migrate_command = "migrate -d tcp:%s:%s" % (target, port) |
1641 | 1641 |
self._CallMonitorCommand(instance_name, migrate_command) |
1642 | 1642 |
|
1643 |
def FinalizeMigrationSource(self, instance, success, live): |
|
1644 |
"""Finalize the instance migration on the source node. |
|
1645 |
|
|
1646 |
@type instance: L{objects.Instance} |
|
1647 |
@param instance: the instance that was migrated |
|
1648 |
@type success: bool |
|
1649 |
@param success: whether the migration succeeded or not |
|
1650 |
@type live: bool |
|
1651 |
@param live: whether the user requested a live migration or not |
|
1652 |
|
|
1653 |
""" |
|
1654 |
if success: |
|
1655 |
pidfile, pid, _ = self._InstancePidAlive(instance.name) |
|
1656 |
utils.KillProcess(pid) |
|
1657 |
self._RemoveInstanceRuntimeFiles(pidfile, instance.name) |
|
1658 |
elif live: |
|
1659 |
self._CallMonitorCommand(instance.name, self._CONT_CMD) |
|
1660 |
|
|
1661 |
def GetMigrationStatus(self, instance): |
|
1662 |
"""Get the migration status |
|
1663 |
|
|
1664 |
@type instance: L{objects.Instance} |
|
1665 |
@param instance: the instance that is being migrated |
|
1666 |
@rtype: L{objects.MigrationStatus} |
|
1667 |
@return: the status of the current migration (one of |
|
1668 |
L{constants.HV_MIGRATION_VALID_STATUSES}), plus any additional |
|
1669 |
progress info that can be retrieved from the hypervisor |
|
1670 |
|
|
1671 |
""" |
|
1643 | 1672 |
info_command = "info migrate" |
1644 |
done = False |
|
1645 |
broken_answers = 0 |
|
1646 |
while not done: |
|
1647 |
result = self._CallMonitorCommand(instance_name, info_command) |
|
1673 |
for _ in range(self._MIGRATION_INFO_MAX_BAD_ANSWERS): |
|
1674 |
result = self._CallMonitorCommand(instance.name, info_command) |
|
1648 | 1675 |
match = self._MIGRATION_STATUS_RE.search(result.stdout) |
1649 | 1676 |
if not match: |
1650 |
broken_answers += 1 |
|
1651 | 1677 |
if not result.stdout: |
1652 | 1678 |
logging.info("KVM: empty 'info migrate' result") |
1653 | 1679 |
else: |
1654 | 1680 |
logging.warning("KVM: unknown 'info migrate' result: %s", |
1655 | 1681 |
result.stdout) |
1656 |
time.sleep(self._MIGRATION_INFO_RETRY_DELAY) |
|
1657 | 1682 |
else: |
1658 | 1683 |
status = match.group(1) |
1659 |
if status == "completed": |
|
1660 |
done = True |
|
1661 |
elif status == "active": |
|
1662 |
# reset the broken answers count |
|
1663 |
broken_answers = 0 |
|
1664 |
time.sleep(self._MIGRATION_INFO_RETRY_DELAY) |
|
1665 |
elif status == "failed" or status == "cancelled": |
|
1666 |
if not live: |
|
1667 |
self._CallMonitorCommand(instance_name, self._CONT_CMD) |
|
1668 |
raise errors.HypervisorError("Migration %s at the kvm level" % |
|
1669 |
status) |
|
1670 |
else: |
|
1671 |
logging.warning("KVM: unknown migration status '%s'", status) |
|
1672 |
broken_answers += 1 |
|
1673 |
time.sleep(self._MIGRATION_INFO_RETRY_DELAY) |
|
1674 |
if broken_answers >= self._MIGRATION_INFO_MAX_BAD_ANSWERS: |
|
1675 |
raise errors.HypervisorError("Too many 'info migrate' broken answers") |
|
1684 |
if status in constants.HV_KVM_MIGRATION_VALID_STATUSES: |
|
1685 |
migration_status = objects.MigrationStatus(status=status) |
|
1686 |
return migration_status |
|
1676 | 1687 |
|
1677 |
utils.KillProcess(pid) |
|
1678 |
self._RemoveInstanceRuntimeFiles(pidfile, instance_name) |
|
1688 |
logging.warning("KVM: unknown migration status '%s'", status) |
|
1689 |
|
|
1690 |
time.sleep(self._MIGRATION_INFO_RETRY_DELAY) |
|
1691 |
|
|
1692 |
return objects.MigrationStatus(status=constants.HV_MIGRATION_FAILED, |
|
1693 |
info="Too many 'info migrate' broken answers") |
|
1679 | 1694 |
|
1680 | 1695 |
def GetNodeInfo(self): |
1681 | 1696 |
"""Return information about the node. |
b/lib/objects.py | ||
---|---|---|
1503 | 1503 |
] |
1504 | 1504 |
|
1505 | 1505 |
|
1506 |
class MigrationStatus(ConfigObject): |
|
1507 |
"""Object holding the status of a migration. |
|
1508 |
|
|
1509 |
""" |
|
1510 |
__slots__ = [ |
|
1511 |
"status", |
|
1512 |
"transferred_ram", |
|
1513 |
"total_ram", |
|
1514 |
] |
|
1515 |
|
|
1516 |
|
|
1506 | 1517 |
class InstanceConsole(ConfigObject): |
1507 | 1518 |
"""Object describing how to access the console of an instance. |
1508 | 1519 |
|
b/lib/rpc.py | ||
---|---|---|
703 | 703 |
[self._InstDict(instance), info, target]) |
704 | 704 |
|
705 | 705 |
@_RpcTimeout(_TMO_NORMAL) |
706 |
def call_finalize_migration(self, node, instance, info, success):
|
|
706 |
def call_instance_finalize_migration_dst(self, node, instance, info, success):
|
|
707 | 707 |
"""Finalize any target-node migration specific operation. |
708 | 708 |
|
709 | 709 |
This is called both in case of a successful migration and in case of error |
... | ... | |
721 | 721 |
@param success: whether the migration was a success or a failure |
722 | 722 |
|
723 | 723 |
""" |
724 |
return self._SingleNodeCall(node, "finalize_migration",
|
|
724 |
return self._SingleNodeCall(node, "instance_finalize_migration_dst",
|
|
725 | 725 |
[self._InstDict(instance), info, success]) |
726 | 726 |
|
727 | 727 |
@_RpcTimeout(_TMO_SLOW) |
... | ... | |
744 | 744 |
return self._SingleNodeCall(node, "instance_migrate", |
745 | 745 |
[self._InstDict(instance), target, live]) |
746 | 746 |
|
747 |
@_RpcTimeout(_TMO_SLOW) |
|
748 |
def call_instance_finalize_migration_src(self, node, instance, success, live): |
|
749 |
"""Finalize the instance migration on the source node. |
|
750 |
|
|
751 |
This is a single-node call. |
|
752 |
|
|
753 |
@type instance: L{objects.Instance} |
|
754 |
@param instance: the instance that was migrated |
|
755 |
@type success: bool |
|
756 |
@param success: whether the migration succeeded or not |
|
757 |
@type live: bool |
|
758 |
@param live: whether the user requested a live migration or not |
|
759 |
|
|
760 |
""" |
|
761 |
return self._SingleNodeCall(node, "instance_finalize_migration_src", |
|
762 |
[self._InstDict(instance), success, live]) |
|
763 |
|
|
764 |
@_RpcTimeout(_TMO_SLOW) |
|
765 |
def call_instance_get_migration_status(self, node, instance): |
|
766 |
"""Report migration status. |
|
767 |
|
|
768 |
This is a single-node call that must be executed on the source node. |
|
769 |
|
|
770 |
@type instance: L{objects.Instance} |
|
771 |
@param instance: the instance that is being migrated |
|
772 |
@rtype: L{objects.MigrationStatus} |
|
773 |
@return: the status of the current migration (one of |
|
774 |
L{constants.HV_MIGRATION_VALID_STATUSES}), plus any additional |
|
775 |
progress info that can be retrieved from the hypervisor |
|
776 |
|
|
777 |
""" |
|
778 |
result = self._SingleNodeCall(node, "instance_get_migration_status", |
|
779 |
[self._InstDict(instance)]) |
|
780 |
if not result.fail_msg and result.payload is not None: |
|
781 |
result.payload = objects.MigrationStatus.FromDict(result.payload) |
|
782 |
return result |
|
783 |
|
|
747 | 784 |
@_RpcTimeout(_TMO_NORMAL) |
748 | 785 |
def call_instance_reboot(self, node, inst, reboot_type, shutdown_timeout): |
749 | 786 |
"""Reboots an instance. |
b/lib/server/noded.py | ||
---|---|---|
579 | 579 |
return backend.AcceptInstance(instance, info, target) |
580 | 580 |
|
581 | 581 |
@staticmethod |
582 |
def perspective_finalize_migration(params):
|
|
583 |
"""Finalize the instance migration. |
|
582 |
def perspective_instance_finalize_migration_dst(params):
|
|
583 |
"""Finalize the instance migration on the destination node.
|
|
584 | 584 |
|
585 | 585 |
""" |
586 | 586 |
instance, info, success = params |
587 | 587 |
instance = objects.Instance.FromDict(instance) |
588 |
return backend.FinalizeMigration(instance, info, success) |
|
588 |
return backend.FinalizeMigrationDst(instance, info, success)
|
|
589 | 589 |
|
590 | 590 |
@staticmethod |
591 | 591 |
def perspective_instance_migrate(params): |
... | ... | |
597 | 597 |
return backend.MigrateInstance(instance, target, live) |
598 | 598 |
|
599 | 599 |
@staticmethod |
600 |
def perspective_instance_finalize_migration_src(params): |
|
601 |
"""Finalize the instance migration on the source node. |
|
602 |
|
|
603 |
""" |
|
604 |
instance, success, live = params |
|
605 |
instance = objects.Instance.FromDict(instance) |
|
606 |
return backend.FinalizeMigrationSource(instance, success, live) |
|
607 |
|
|
608 |
@staticmethod |
|
609 |
def perspective_instance_get_migration_status(params): |
|
610 |
"""Reports migration status. |
|
611 |
|
|
612 |
""" |
|
613 |
instance = objects.Instance.FromDict(params[0]) |
|
614 |
return backend.GetMigrationStatus(instance).ToDict() |
|
615 |
|
|
616 |
@staticmethod |
|
600 | 617 |
def perspective_instance_reboot(params): |
601 | 618 |
"""Reboot an instance. |
602 | 619 |
|
Also available in: Unified diff