Revision 41310c68
| b/docs/rdma.txt | ||
|---|---|---|
| 66 | 66 |
high-performance RDMA hardware using the following command: |
| 67 | 67 |
|
| 68 | 68 |
QEMU Monitor Command: |
| 69 |
$ migrate_set_capability x-rdma-pin-all on # disabled by default
|
|
| 69 |
$ migrate_set_capability rdma-pin-all on # disabled by default |
|
| 70 | 70 |
|
| 71 | 71 |
Performing this action will cause all 8GB to be pinned, so if that's |
| 72 | 72 |
not what you want, then please ignore this step altogether. |
| ... | ... | |
| 93 | 93 |
|
| 94 | 94 |
Next, on the destination machine, add the following to the QEMU command line: |
| 95 | 95 |
|
| 96 |
qemu ..... -incoming x-rdma:host:port
|
|
| 96 |
qemu ..... -incoming rdma:host:port |
|
| 97 | 97 |
|
| 98 | 98 |
Finally, perform the actual migration on the source machine: |
| 99 | 99 |
|
| 100 | 100 |
QEMU Monitor Command: |
| 101 |
$ migrate -d x-rdma:host:port
|
|
| 101 |
$ migrate -d rdma:host:port |
|
| 102 | 102 |
|
| 103 | 103 |
PERFORMANCE |
| 104 | 104 |
=========== |
| ... | ... | |
| 120 | 120 |
active use and the VM itself is completely idle using the same 40 gbps |
| 121 | 121 |
infiniband link: |
| 122 | 122 |
|
| 123 |
1. x-rdma-pin-all disabled total time: approximately 7.5 seconds @ 9.5 Gbps
|
|
| 124 |
2. x-rdma-pin-all enabled total time: approximately 4 seconds @ 26 Gbps
|
|
| 123 |
1. rdma-pin-all disabled total time: approximately 7.5 seconds @ 9.5 Gbps |
|
| 124 |
2. rdma-pin-all enabled total time: approximately 4 seconds @ 26 Gbps |
|
| 125 | 125 |
|
| 126 | 126 |
These numbers would of course scale up to whatever size virtual machine |
| 127 | 127 |
you have to migrate using RDMA. |
| ... | ... | |
| 407 | 407 |
|
| 408 | 408 |
TODO: |
| 409 | 409 |
===== |
| 410 |
1. 'migrate x-rdma:host:port' and '-incoming x-rdma' options will be |
|
| 411 |
renamed to 'rdma' after the experimental phase of this work has |
|
| 412 |
completed upstream. |
|
| 413 |
2. Currently, 'ulimit -l' mlock() limits as well as cgroups swap limits |
|
| 410 |
1. Currently, 'ulimit -l' mlock() limits as well as cgroups swap limits |
|
| 414 | 411 |
are not compatible with infinband memory pinning and will result in |
| 415 | 412 |
an aborted migration (but with the source VM left unaffected). |
| 416 |
3. Use of the recent /proc/<pid>/pagemap would likely speed up
|
|
| 413 |
2. Use of the recent /proc/<pid>/pagemap would likely speed up
|
|
| 417 | 414 |
the use of KSM and ballooning while using RDMA. |
| 418 |
4. Also, some form of balloon-device usage tracking would also
|
|
| 415 |
3. Also, some form of balloon-device usage tracking would also
|
|
| 419 | 416 |
help alleviate some issues. |
| 420 |
5. Move UNREGISTER requests to a separate thread. |
|
| 421 |
6. Use LRU to provide more fine-grained direction of UNREGISTER |
|
| 417 |
4. Use LRU to provide more fine-grained direction of UNREGISTER |
|
| 422 | 418 |
requests for unpinning memory in an overcommitted environment. |
| 423 |
7. Expose UNREGISTER support to the user by way of workload-specific
|
|
| 419 |
5. Expose UNREGISTER support to the user by way of workload-specific
|
|
| 424 | 420 |
hints about application behavior. |
| b/migration-rdma.c | ||
|---|---|---|
| 3412 | 3412 |
} |
| 3413 | 3413 |
|
| 3414 | 3414 |
ret = qemu_rdma_source_init(rdma, &local_err, |
| 3415 |
s->enabled_capabilities[MIGRATION_CAPABILITY_X_RDMA_PIN_ALL]);
|
|
| 3415 |
s->enabled_capabilities[MIGRATION_CAPABILITY_RDMA_PIN_ALL]); |
|
| 3416 | 3416 |
|
| 3417 | 3417 |
if (ret) {
|
| 3418 | 3418 |
goto err; |
| b/migration.c | ||
|---|---|---|
| 82 | 82 |
if (strstart(uri, "tcp:", &p)) |
| 83 | 83 |
tcp_start_incoming_migration(p, errp); |
| 84 | 84 |
#ifdef CONFIG_RDMA |
| 85 |
else if (strstart(uri, "x-rdma:", &p))
|
|
| 85 |
else if (strstart(uri, "rdma:", &p)) |
|
| 86 | 86 |
rdma_start_incoming_migration(p, errp); |
| 87 | 87 |
#endif |
| 88 | 88 |
#if !defined(WIN32) |
| ... | ... | |
| 438 | 438 |
if (strstart(uri, "tcp:", &p)) {
|
| 439 | 439 |
tcp_start_outgoing_migration(s, p, &local_err); |
| 440 | 440 |
#ifdef CONFIG_RDMA |
| 441 |
} else if (strstart(uri, "x-rdma:", &p)) {
|
|
| 441 |
} else if (strstart(uri, "rdma:", &p)) {
|
|
| 442 | 442 |
rdma_start_outgoing_migration(s, p, &local_err); |
| 443 | 443 |
#endif |
| 444 | 444 |
#if !defined(WIN32) |
| ... | ... | |
| 532 | 532 |
|
| 533 | 533 |
s = migrate_get_current(); |
| 534 | 534 |
|
| 535 |
return s->enabled_capabilities[MIGRATION_CAPABILITY_X_RDMA_PIN_ALL];
|
|
| 535 |
return s->enabled_capabilities[MIGRATION_CAPABILITY_RDMA_PIN_ALL]; |
|
| 536 | 536 |
} |
| 537 | 537 |
|
| 538 | 538 |
bool migrate_auto_converge(void) |
| b/qapi-schema.json | ||
|---|---|---|
| 751 | 751 |
# This feature allows us to minimize migration traffic for certain work |
| 752 | 752 |
# loads, by sending compressed difference of the pages |
| 753 | 753 |
# |
| 754 |
# @x-rdma-pin-all: Controls whether or not the entire VM memory footprint is
|
|
| 754 |
# @rdma-pin-all: Controls whether or not the entire VM memory footprint is |
|
| 755 | 755 |
# mlock()'d on demand or all at once. Refer to docs/rdma.txt for usage. |
| 756 |
# Disabled by default. Experimental: may (or may not) be renamed after |
|
| 757 |
# further testing is complete. (since 1.6) |
|
| 756 |
# Disabled by default. (since 2.0) |
|
| 758 | 757 |
# |
| 759 | 758 |
# @zero-blocks: During storage migration encode blocks of zeroes efficiently. This |
| 760 | 759 |
# essentially saves 1MB of zeroes per block on the wire. Enabling requires |
| ... | ... | |
| 768 | 767 |
# Since: 1.2 |
| 769 | 768 |
## |
| 770 | 769 |
{ 'enum': 'MigrationCapability',
|
| 771 |
'data': ['xbzrle', 'x-rdma-pin-all', 'auto-converge', 'zero-blocks'] }
|
|
| 770 |
'data': ['xbzrle', 'rdma-pin-all', 'auto-converge', 'zero-blocks'] } |
|
| 772 | 771 |
|
| 773 | 772 |
## |
| 774 | 773 |
# @MigrationCapabilityStatus |
Also available in: Unified diff