Revision 41310c68
b/docs/rdma.txt | ||
---|---|---|
66 | 66 |
high-performance RDMA hardware using the following command: |
67 | 67 |
|
68 | 68 |
QEMU Monitor Command: |
69 |
$ migrate_set_capability x-rdma-pin-all on # disabled by default
|
|
69 |
$ migrate_set_capability rdma-pin-all on # disabled by default |
|
70 | 70 |
|
71 | 71 |
Performing this action will cause all 8GB to be pinned, so if that's |
72 | 72 |
not what you want, then please ignore this step altogether. |
... | ... | |
93 | 93 |
|
94 | 94 |
Next, on the destination machine, add the following to the QEMU command line: |
95 | 95 |
|
96 |
qemu ..... -incoming x-rdma:host:port
|
|
96 |
qemu ..... -incoming rdma:host:port |
|
97 | 97 |
|
98 | 98 |
Finally, perform the actual migration on the source machine: |
99 | 99 |
|
100 | 100 |
QEMU Monitor Command: |
101 |
$ migrate -d x-rdma:host:port
|
|
101 |
$ migrate -d rdma:host:port |
|
102 | 102 |
|
103 | 103 |
PERFORMANCE |
104 | 104 |
=========== |
... | ... | |
120 | 120 |
active use and the VM itself is completely idle using the same 40 gbps |
121 | 121 |
infiniband link: |
122 | 122 |
|
123 |
1. x-rdma-pin-all disabled total time: approximately 7.5 seconds @ 9.5 Gbps
|
|
124 |
2. x-rdma-pin-all enabled total time: approximately 4 seconds @ 26 Gbps
|
|
123 |
1. rdma-pin-all disabled total time: approximately 7.5 seconds @ 9.5 Gbps |
|
124 |
2. rdma-pin-all enabled total time: approximately 4 seconds @ 26 Gbps |
|
125 | 125 |
|
126 | 126 |
These numbers would of course scale up to whatever size virtual machine |
127 | 127 |
you have to migrate using RDMA. |
... | ... | |
407 | 407 |
|
408 | 408 |
TODO: |
409 | 409 |
===== |
410 |
1. 'migrate x-rdma:host:port' and '-incoming x-rdma' options will be |
|
411 |
renamed to 'rdma' after the experimental phase of this work has |
|
412 |
completed upstream. |
|
413 |
2. Currently, 'ulimit -l' mlock() limits as well as cgroups swap limits |
|
410 |
1. Currently, 'ulimit -l' mlock() limits as well as cgroups swap limits |
|
414 | 411 |
are not compatible with infinband memory pinning and will result in |
415 | 412 |
an aborted migration (but with the source VM left unaffected). |
416 |
3. Use of the recent /proc/<pid>/pagemap would likely speed up
|
|
413 |
2. Use of the recent /proc/<pid>/pagemap would likely speed up
|
|
417 | 414 |
the use of KSM and ballooning while using RDMA. |
418 |
4. Also, some form of balloon-device usage tracking would also
|
|
415 |
3. Also, some form of balloon-device usage tracking would also
|
|
419 | 416 |
help alleviate some issues. |
420 |
5. Move UNREGISTER requests to a separate thread. |
|
421 |
6. Use LRU to provide more fine-grained direction of UNREGISTER |
|
417 |
4. Use LRU to provide more fine-grained direction of UNREGISTER |
|
422 | 418 |
requests for unpinning memory in an overcommitted environment. |
423 |
7. Expose UNREGISTER support to the user by way of workload-specific
|
|
419 |
5. Expose UNREGISTER support to the user by way of workload-specific
|
|
424 | 420 |
hints about application behavior. |
b/migration-rdma.c | ||
---|---|---|
3412 | 3412 |
} |
3413 | 3413 |
|
3414 | 3414 |
ret = qemu_rdma_source_init(rdma, &local_err, |
3415 |
s->enabled_capabilities[MIGRATION_CAPABILITY_X_RDMA_PIN_ALL]);
|
|
3415 |
s->enabled_capabilities[MIGRATION_CAPABILITY_RDMA_PIN_ALL]); |
|
3416 | 3416 |
|
3417 | 3417 |
if (ret) { |
3418 | 3418 |
goto err; |
b/migration.c | ||
---|---|---|
82 | 82 |
if (strstart(uri, "tcp:", &p)) |
83 | 83 |
tcp_start_incoming_migration(p, errp); |
84 | 84 |
#ifdef CONFIG_RDMA |
85 |
else if (strstart(uri, "x-rdma:", &p))
|
|
85 |
else if (strstart(uri, "rdma:", &p)) |
|
86 | 86 |
rdma_start_incoming_migration(p, errp); |
87 | 87 |
#endif |
88 | 88 |
#if !defined(WIN32) |
... | ... | |
438 | 438 |
if (strstart(uri, "tcp:", &p)) { |
439 | 439 |
tcp_start_outgoing_migration(s, p, &local_err); |
440 | 440 |
#ifdef CONFIG_RDMA |
441 |
} else if (strstart(uri, "x-rdma:", &p)) {
|
|
441 |
} else if (strstart(uri, "rdma:", &p)) { |
|
442 | 442 |
rdma_start_outgoing_migration(s, p, &local_err); |
443 | 443 |
#endif |
444 | 444 |
#if !defined(WIN32) |
... | ... | |
532 | 532 |
|
533 | 533 |
s = migrate_get_current(); |
534 | 534 |
|
535 |
return s->enabled_capabilities[MIGRATION_CAPABILITY_X_RDMA_PIN_ALL];
|
|
535 |
return s->enabled_capabilities[MIGRATION_CAPABILITY_RDMA_PIN_ALL]; |
|
536 | 536 |
} |
537 | 537 |
|
538 | 538 |
bool migrate_auto_converge(void) |
b/qapi-schema.json | ||
---|---|---|
751 | 751 |
# This feature allows us to minimize migration traffic for certain work |
752 | 752 |
# loads, by sending compressed difference of the pages |
753 | 753 |
# |
754 |
# @x-rdma-pin-all: Controls whether or not the entire VM memory footprint is
|
|
754 |
# @rdma-pin-all: Controls whether or not the entire VM memory footprint is |
|
755 | 755 |
# mlock()'d on demand or all at once. Refer to docs/rdma.txt for usage. |
756 |
# Disabled by default. Experimental: may (or may not) be renamed after |
|
757 |
# further testing is complete. (since 1.6) |
|
756 |
# Disabled by default. (since 2.0) |
|
758 | 757 |
# |
759 | 758 |
# @zero-blocks: During storage migration encode blocks of zeroes efficiently. This |
760 | 759 |
# essentially saves 1MB of zeroes per block on the wire. Enabling requires |
... | ... | |
768 | 767 |
# Since: 1.2 |
769 | 768 |
## |
770 | 769 |
{ 'enum': 'MigrationCapability', |
771 |
'data': ['xbzrle', 'x-rdma-pin-all', 'auto-converge', 'zero-blocks'] }
|
|
770 |
'data': ['xbzrle', 'rdma-pin-all', 'auto-converge', 'zero-blocks'] } |
|
772 | 771 |
|
773 | 772 |
## |
774 | 773 |
# @MigrationCapabilityStatus |
Also available in: Unified diff