Re: [RFC PATCH v3 11/30] migration/multifd: Allow multifd without packets
On Mon, Jan 15, 2024 at 03:39:29PM -0300, Fabiano Rosas wrote: > > Maybe multifd_use_packets()? Dropping the migrate_ prefix as this is not a > > global API but multifd-only. Then if multifd_packets() reads too weird and > > unclear, "add" makes it clear. > > We removed all the instances of migrate_use_* from the migration code > recently. Not sure we should introduce them back, it seems like a step > back. > > We're setting 'use_packets = migrate_multifd_packets()' in most places, > so I guess 'use_packets = multifd_packets()' wouldn't be too bad. I actually prefers keep using "_use_" all over the places because it's clearer to me. :) While I don't see much benefit saving three chars. Try "git grep _use_ | wc -l" in both QEMU / Linux, then we'll see that reports 275 / 4680 corrrespondingly. But yeah that's trivial, multifd_packets() is still okay. -- Peter Xu
Re: [RFC PATCH v3 11/30] migration/multifd: Allow multifd without packets
Peter Xu writes:
> On Mon, Nov 27, 2023 at 05:25:53PM -0300, Fabiano Rosas wrote:
>> For the upcoming support to the new 'fixed-ram' migration stream
>> format, we cannot use multifd packets because each write into the
>> ramblock section in the migration file is expected to contain only the
>> guest pages. They are written at their respective offsets relative to
>> the ramblock section header.
>>
>> There is no space for the packet information and the expected gains
>> from the new approach come partly from being able to write the pages
>> sequentially without extraneous data in between.
>>
>> The new format also doesn't need the packets and all necessary
>> information can be taken from the standard migration headers with some
>> (future) changes to multifd code.
>>
>> Use the presence of the fixed-ram capability to decide whether to send
>> packets. For now this has no effect as fixed-ram cannot yet be enabled
>> with multifd.
>>
>> Signed-off-by: Fabiano Rosas
>> ---
>> - moved more of the packet code under use_packets
>> ---
>> migration/multifd.c | 138 +++-
>> migration/options.c | 5 ++
>> migration/options.h | 1 +
>> 3 files changed, 91 insertions(+), 53 deletions(-)
>>
>> diff --git a/migration/multifd.c b/migration/multifd.c
>> index ec58c58082..9625640d61 100644
>> --- a/migration/multifd.c
>> +++ b/migration/multifd.c
>> @@ -654,18 +654,22 @@ static void *multifd_send_thread(void *opaque)
>> Error *local_err = NULL;
>> int ret = 0;
>> bool use_zero_copy_send = migrate_zero_copy_send();
>> +bool use_packets = migrate_multifd_packets();
>>
>> thread = migration_threads_add(p->name, qemu_get_thread_id());
>>
>> trace_multifd_send_thread_start(p->id);
>> rcu_register_thread();
>>
>> -if (multifd_send_initial_packet(p, _err) < 0) {
>> -ret = -1;
>> -goto out;
>> +if (use_packets) {
>> +if (multifd_send_initial_packet(p, _err) < 0) {
>> +ret = -1;
>> +goto out;
>> +}
>> +
>> +/* initial packet */
>> +p->num_packets = 1;
>> }
>> -/* initial packet */
>> -p->num_packets = 1;
>>
>> while (true) {
>> qemu_sem_post(_send_state->channels_ready);
>> @@ -677,11 +681,10 @@ static void *multifd_send_thread(void *opaque)
>> qemu_mutex_lock(>mutex);
>>
>> if (p->pending_job) {
>> -uint64_t packet_num = p->packet_num;
>> uint32_t flags;
>> p->normal_num = 0;
>>
>> -if (use_zero_copy_send) {
>> +if (!use_packets || use_zero_copy_send) {
>> p->iovs_num = 0;
>> } else {
>> p->iovs_num = 1;
>> @@ -699,16 +702,20 @@ static void *multifd_send_thread(void *opaque)
>> break;
>> }
>> }
>> -multifd_send_fill_packet(p);
>> +
>> +if (use_packets) {
>> +multifd_send_fill_packet(p);
>> +p->num_packets++;
>> +}
>> +
>> flags = p->flags;
>> p->flags = 0;
>> -p->num_packets++;
>> p->total_normal_pages += p->normal_num;
>> p->pages->num = 0;
>> p->pages->block = NULL;
>> qemu_mutex_unlock(>mutex);
>>
>> -trace_multifd_send(p->id, packet_num, p->normal_num, flags,
>> +trace_multifd_send(p->id, p->packet_num, p->normal_num, flags,
>> p->next_packet_size);
>>
>> if (use_zero_copy_send) {
>> @@ -718,7 +725,7 @@ static void *multifd_send_thread(void *opaque)
>> if (ret != 0) {
>> break;
>> }
>> -} else {
>> +} else if (use_packets) {
>> /* Send header using the same writev call */
>> p->iov[0].iov_len = p->packet_len;
>> p->iov[0].iov_base = p->packet;
>> @@ -904,6 +911,7 @@ int multifd_save_setup(Error **errp)
>> {
>> int thread_count;
>> uint32_t page_count = MULTIFD_PACKET_SIZE / qemu_target_page_size();
>> +bool use_packets = migrate_multifd_packets();
>> uint8_t i;
>>
>> if (!migrate_multifd()) {
>> @@ -928,14 +936,20 @@ int multifd_save_setup(Error **errp)
>> p->pending_job = 0;
>> p->id = i;
>> p->pages = multifd_pages_init(page_count);
>> -p->packet_len = sizeof(MultiFDPacket_t)
>> - + sizeof(uint64_t) * page_count;
>> -p->packet = g_malloc0(p->packet_len);
>> -p->packet->magic = cpu_to_be32(MULTIFD_MAGIC);
>> -p->packet->version = cpu_to_be32(MULTIFD_VERSION);
>> +
>> +if (use_packets) {
>> +p->packet_len = sizeof(MultiFDPacket_t)
>> + + sizeof(uint64_t) * page_count;
>> +p->packet = g_malloc0(p->packet_len);
>> +
Re: [RFC PATCH v3 11/30] migration/multifd: Allow multifd without packets
On Mon, Nov 27, 2023 at 05:25:53PM -0300, Fabiano Rosas wrote:
> For the upcoming support to the new 'fixed-ram' migration stream
> format, we cannot use multifd packets because each write into the
> ramblock section in the migration file is expected to contain only the
> guest pages. They are written at their respective offsets relative to
> the ramblock section header.
>
> There is no space for the packet information and the expected gains
> from the new approach come partly from being able to write the pages
> sequentially without extraneous data in between.
>
> The new format also doesn't need the packets and all necessary
> information can be taken from the standard migration headers with some
> (future) changes to multifd code.
>
> Use the presence of the fixed-ram capability to decide whether to send
> packets. For now this has no effect as fixed-ram cannot yet be enabled
> with multifd.
>
> Signed-off-by: Fabiano Rosas
> ---
> - moved more of the packet code under use_packets
> ---
> migration/multifd.c | 138 +++-
> migration/options.c | 5 ++
> migration/options.h | 1 +
> 3 files changed, 91 insertions(+), 53 deletions(-)
>
> diff --git a/migration/multifd.c b/migration/multifd.c
> index ec58c58082..9625640d61 100644
> --- a/migration/multifd.c
> +++ b/migration/multifd.c
> @@ -654,18 +654,22 @@ static void *multifd_send_thread(void *opaque)
> Error *local_err = NULL;
> int ret = 0;
> bool use_zero_copy_send = migrate_zero_copy_send();
> +bool use_packets = migrate_multifd_packets();
>
> thread = migration_threads_add(p->name, qemu_get_thread_id());
>
> trace_multifd_send_thread_start(p->id);
> rcu_register_thread();
>
> -if (multifd_send_initial_packet(p, _err) < 0) {
> -ret = -1;
> -goto out;
> +if (use_packets) {
> +if (multifd_send_initial_packet(p, _err) < 0) {
> +ret = -1;
> +goto out;
> +}
> +
> +/* initial packet */
> +p->num_packets = 1;
> }
> -/* initial packet */
> -p->num_packets = 1;
>
> while (true) {
> qemu_sem_post(_send_state->channels_ready);
> @@ -677,11 +681,10 @@ static void *multifd_send_thread(void *opaque)
> qemu_mutex_lock(>mutex);
>
> if (p->pending_job) {
> -uint64_t packet_num = p->packet_num;
> uint32_t flags;
> p->normal_num = 0;
>
> -if (use_zero_copy_send) {
> +if (!use_packets || use_zero_copy_send) {
> p->iovs_num = 0;
> } else {
> p->iovs_num = 1;
> @@ -699,16 +702,20 @@ static void *multifd_send_thread(void *opaque)
> break;
> }
> }
> -multifd_send_fill_packet(p);
> +
> +if (use_packets) {
> +multifd_send_fill_packet(p);
> +p->num_packets++;
> +}
> +
> flags = p->flags;
> p->flags = 0;
> -p->num_packets++;
> p->total_normal_pages += p->normal_num;
> p->pages->num = 0;
> p->pages->block = NULL;
> qemu_mutex_unlock(>mutex);
>
> -trace_multifd_send(p->id, packet_num, p->normal_num, flags,
> +trace_multifd_send(p->id, p->packet_num, p->normal_num, flags,
> p->next_packet_size);
>
> if (use_zero_copy_send) {
> @@ -718,7 +725,7 @@ static void *multifd_send_thread(void *opaque)
> if (ret != 0) {
> break;
> }
> -} else {
> +} else if (use_packets) {
> /* Send header using the same writev call */
> p->iov[0].iov_len = p->packet_len;
> p->iov[0].iov_base = p->packet;
> @@ -904,6 +911,7 @@ int multifd_save_setup(Error **errp)
> {
> int thread_count;
> uint32_t page_count = MULTIFD_PACKET_SIZE / qemu_target_page_size();
> +bool use_packets = migrate_multifd_packets();
> uint8_t i;
>
> if (!migrate_multifd()) {
> @@ -928,14 +936,20 @@ int multifd_save_setup(Error **errp)
> p->pending_job = 0;
> p->id = i;
> p->pages = multifd_pages_init(page_count);
> -p->packet_len = sizeof(MultiFDPacket_t)
> - + sizeof(uint64_t) * page_count;
> -p->packet = g_malloc0(p->packet_len);
> -p->packet->magic = cpu_to_be32(MULTIFD_MAGIC);
> -p->packet->version = cpu_to_be32(MULTIFD_VERSION);
> +
> +if (use_packets) {
> +p->packet_len = sizeof(MultiFDPacket_t)
> + + sizeof(uint64_t) * page_count;
> +p->packet = g_malloc0(p->packet_len);
> +p->packet->magic = cpu_to_be32(MULTIFD_MAGIC);
> +p->packet->version = cpu_to_be32(MULTIFD_VERSION);
> +
> +/* We need one extra
[RFC PATCH v3 11/30] migration/multifd: Allow multifd without packets
For the upcoming support to the new 'fixed-ram' migration stream
format, we cannot use multifd packets because each write into the
ramblock section in the migration file is expected to contain only the
guest pages. They are written at their respective offsets relative to
the ramblock section header.
There is no space for the packet information and the expected gains
from the new approach come partly from being able to write the pages
sequentially without extraneous data in between.
The new format also doesn't need the packets and all necessary
information can be taken from the standard migration headers with some
(future) changes to multifd code.
Use the presence of the fixed-ram capability to decide whether to send
packets. For now this has no effect as fixed-ram cannot yet be enabled
with multifd.
Signed-off-by: Fabiano Rosas
---
- moved more of the packet code under use_packets
---
migration/multifd.c | 138 +++-
migration/options.c | 5 ++
migration/options.h | 1 +
3 files changed, 91 insertions(+), 53 deletions(-)
diff --git a/migration/multifd.c b/migration/multifd.c
index ec58c58082..9625640d61 100644
--- a/migration/multifd.c
+++ b/migration/multifd.c
@@ -654,18 +654,22 @@ static void *multifd_send_thread(void *opaque)
Error *local_err = NULL;
int ret = 0;
bool use_zero_copy_send = migrate_zero_copy_send();
+bool use_packets = migrate_multifd_packets();
thread = migration_threads_add(p->name, qemu_get_thread_id());
trace_multifd_send_thread_start(p->id);
rcu_register_thread();
-if (multifd_send_initial_packet(p, _err) < 0) {
-ret = -1;
-goto out;
+if (use_packets) {
+if (multifd_send_initial_packet(p, _err) < 0) {
+ret = -1;
+goto out;
+}
+
+/* initial packet */
+p->num_packets = 1;
}
-/* initial packet */
-p->num_packets = 1;
while (true) {
qemu_sem_post(_send_state->channels_ready);
@@ -677,11 +681,10 @@ static void *multifd_send_thread(void *opaque)
qemu_mutex_lock(>mutex);
if (p->pending_job) {
-uint64_t packet_num = p->packet_num;
uint32_t flags;
p->normal_num = 0;
-if (use_zero_copy_send) {
+if (!use_packets || use_zero_copy_send) {
p->iovs_num = 0;
} else {
p->iovs_num = 1;
@@ -699,16 +702,20 @@ static void *multifd_send_thread(void *opaque)
break;
}
}
-multifd_send_fill_packet(p);
+
+if (use_packets) {
+multifd_send_fill_packet(p);
+p->num_packets++;
+}
+
flags = p->flags;
p->flags = 0;
-p->num_packets++;
p->total_normal_pages += p->normal_num;
p->pages->num = 0;
p->pages->block = NULL;
qemu_mutex_unlock(>mutex);
-trace_multifd_send(p->id, packet_num, p->normal_num, flags,
+trace_multifd_send(p->id, p->packet_num, p->normal_num, flags,
p->next_packet_size);
if (use_zero_copy_send) {
@@ -718,7 +725,7 @@ static void *multifd_send_thread(void *opaque)
if (ret != 0) {
break;
}
-} else {
+} else if (use_packets) {
/* Send header using the same writev call */
p->iov[0].iov_len = p->packet_len;
p->iov[0].iov_base = p->packet;
@@ -904,6 +911,7 @@ int multifd_save_setup(Error **errp)
{
int thread_count;
uint32_t page_count = MULTIFD_PACKET_SIZE / qemu_target_page_size();
+bool use_packets = migrate_multifd_packets();
uint8_t i;
if (!migrate_multifd()) {
@@ -928,14 +936,20 @@ int multifd_save_setup(Error **errp)
p->pending_job = 0;
p->id = i;
p->pages = multifd_pages_init(page_count);
-p->packet_len = sizeof(MultiFDPacket_t)
- + sizeof(uint64_t) * page_count;
-p->packet = g_malloc0(p->packet_len);
-p->packet->magic = cpu_to_be32(MULTIFD_MAGIC);
-p->packet->version = cpu_to_be32(MULTIFD_VERSION);
+
+if (use_packets) {
+p->packet_len = sizeof(MultiFDPacket_t)
+ + sizeof(uint64_t) * page_count;
+p->packet = g_malloc0(p->packet_len);
+p->packet->magic = cpu_to_be32(MULTIFD_MAGIC);
+p->packet->version = cpu_to_be32(MULTIFD_VERSION);
+
+/* We need one extra place for the packet header */
+p->iov = g_new0(struct iovec, page_count + 1);
+} else {
+p->iov = g_new0(struct iovec, page_count);
+}
p->name = g_strdup_printf("multifdsend_%d", i);
-/* We need one extra place for the packet header */
-p->iov = g_new0(struct
