On 30/04/2025 16:28, Li Zhijian wrote:
>
>
> On 30/04/2025 05:09, Peter Xu wrote:
>> On Thu, Mar 27, 2025 at 10:12:34AM +0800, Li Zhijian wrote:
>>> Recently, we removed ipv6 restriction[0] from RDMA migration, add a
>>> test for it.
>>>
>>> [0]
>>> https://lore.kernel.org/qemu-devel/20250326095224.9918-1-jinpu.w...@ionos.com/
>>>
>>> Cc: Jack Wang <jinpu.w...@ionos.com>
>>> Cc: Michael R. Galaxy <mrgal...@nvidia.com>
>>> Cc: Peter Xu <pet...@redhat.com>
>>> Cc: Yu Zhang <yu.zh...@ionos.com>
>>> Signed-off-by: Li Zhijian <lizhij...@fujitsu.com>
>>> ---
>>>
>>> This test is added based on [1]
>>
>> Hmm, this cannot apply cleanly on top of your rdma test + Jake's ipv6 fix.
>> Did I made it wrong somewhere?
>>
>> At the meantime, I found that the current rdma unit test failing if I did
>> the "setup" phase then run this test without root:
>>
>
> Thanks for you report, I reproduced it on fedora40 while ubuntu2204 always
> works well.
>
> I will look into it.
A work around in my fedora40 is:
$ sudo setcap 'cap_ipc_lock+ep' ./build/qemu-system-x86_64
Well, that's interesting, the ./build/qemu-system-x86_64 in my ubuntu2204 works
without cap_ipc_lock cap..
>
>
> Thanks
> Zhijian
>
>
>> stderr:
>> qemu-system-x86_64: cannot get rkey
>> qemu-system-x86_64: error while loading state section id 2(ram)
>> qemu-system-x86_64: load of migration failed: Operation not permitted
>> qemu-system-x86_64: rdma migration: recv polling control error!
>> qemu-system-x86_64: RDMA is in an error state waiting migration to abort!
>> qemu-system-x86_64: failed to save SaveStateEntry with id(name): 2(ram): -1
>> qemu-system-x86_64: Channel error: Operation not permitted
>> **
>> ERROR:../tests/qtest/migration/migration-qmp.c:200:check_migration_status:
>> assertion failed (current_status != "failed"): ("failed" != "failed")
>> qemu-system-x86_64: warning: Early error. Sending error.
>> qemu-system-x86_64: warning: rdma migration: send polling control error
>> ../tests/qtest/libqtest.c:199: kill_qemu() tried to terminate QEMU process
>> but encountered exit status 1 (expected 0)
>>
>> So running the test also needs root? Is it possible we fix the test so it
>> can also be smart enough to skip if it knows it'll hit the "cannot get
>> rkey" error (even if it sees the rdma link setup)? Not something urgent
>> but definitely good to have.
It seems it's a security problem, I have no a good idea yet.
Let me see see...
>>
>> OTOH, running the test using root always works for me.
>
>
>>
>>>
>>> Signed-off-by: Li Zhijian <lizhij...@fujitsu.com>
>>> ---
>>> scripts/rdma-migration-helper.sh | 26 +++++++++++++++++++++++---
>>> tests/qtest/migration/precopy-tests.c | 21 +++++++++++++++++----
>>> 2 files changed, 40 insertions(+), 7 deletions(-)
>>>
>>> diff --git a/scripts/rdma-migration-helper.sh
>>> b/scripts/rdma-migration-helper.sh
>>> index 019b9ad932..c358392b72 100755
>>> --- a/scripts/rdma-migration-helper.sh
>>> +++ b/scripts/rdma-migration-helper.sh
>>> @@ -8,6 +8,15 @@ get_ipv4_addr()
>>> head -1 | tr -d '\n'
>>> }
>>> +get_ipv6_addr() {
>>> + ipv6=$(ip -6 -o addr show dev "$1" |
>>> + sed -n 's/.*[[:blank:]]inet6[[:blank:]]*\([^[:blank:]/]*\).*/\1/p'
>>> |
>>> + head -1 | tr -d '\n')
>>> +
>>> + [ $? -eq 0 ] || return
>>> + echo -n ["$ipv6"%$1]
>>> +}
>>> +
>>> # existing rdma interfaces
>>> rdma_interfaces()
>>> {
>>> @@ -20,11 +29,16 @@ ipv4_interfaces()
>>> ip -o addr show | awk '/inet / {print $2}' | sort -u | grep -v -w lo
>>> }
>>> +ipv6_interfaces()
>>> +{
>>> + ip -o addr show | awk '/inet6 / {print $2}' | sort -u | grep -v -w lo
>>> +}
>>> +
>>> rdma_rxe_detect()
>>> {
>>> for r in $(rdma_interfaces)
>>> do
>>> - ipv4_interfaces | grep -qw $r && get_ipv4_addr $r && return
>>> + "$IP_FAMILY"_interfaces | grep -qw $r && get_"$IP_FAMILY"_addr $r
>>> && return
>>> done
>>> return 1
>>> @@ -32,11 +46,11 @@ rdma_rxe_detect()
>>> rdma_rxe_setup()
>>> {
>>> - for i in $(ipv4_interfaces)
>>> + for i in $("$IP_FAMILY"_interfaces)
>>> do
>>> rdma_interfaces | grep -qw $i && continue
>>> rdma link add "${i}_rxe" type rxe netdev "$i" && {
>>> - echo "Setup new rdma/rxe ${i}_rxe for $i with $(get_ipv4_addr
>>> $i)"
>>> + echo "Setup new rdma/rxe ${i}_rxe for $i with
>>> $(get_"$IP_FAMILY"_addr $i)"
>>> return
>>> }
>>> done
>>> @@ -50,6 +64,12 @@ rdma_rxe_clean()
>>> modprobe -r rdma_rxe
>>> }
>>> +IP_FAMILY=${IP_FAMILY:-ipv4}
>>> +if [ "$IP_FAMILY" != "ipv6" ] && [ "$IP_FAMILY" != "ipv4" ]; then
>>> + echo "Unknown ip family '$IP_FAMILY', only ipv4 or ipv6 is supported,"
>>> >&2
>>> + exit 1
>>> +fi
>>> +
>>> operation=${1:-detect}
>>> command -v rdma >/dev/null || {
>>> diff --git a/tests/qtest/migration/precopy-tests.c
>>> b/tests/qtest/migration/precopy-tests.c
>>> index a137ea9f2c..9f7236dc59 100644
>>> --- a/tests/qtest/migration/precopy-tests.c
>>> +++ b/tests/qtest/migration/precopy-tests.c
>>> @@ -102,12 +102,13 @@ static void test_precopy_unix_dirty_ring(void)
>>> #ifdef CONFIG_RDMA
>>> #define RDMA_MIGRATION_HELPER "scripts/rdma-migration-helper.sh"
>>> -static int new_rdma_link(char *buffer)
>>> +static int new_rdma_link(char *buffer, bool ipv6)
>>> {
>>> char cmd[256];
>>> bool verbose = g_getenv("QTEST_LOG");
>>> - snprintf(cmd, sizeof(cmd), "%s detect %s", RDMA_MIGRATION_HELPER,
>>> + snprintf(cmd, sizeof(cmd), "IP_FAMILY=%s %s detect %s",
>>> + ipv6 ? "ipv6" : "ipv4", RDMA_MIGRATION_HELPER,
>>> verbose ? "" : "2>/dev/null");
>>> FILE *pipe = popen(cmd, "r");
>>> @@ -132,11 +133,11 @@ static int new_rdma_link(char *buffer)
>>> return -1;
>>> }
>>> -static void test_precopy_rdma_plain(void)
>>> +static void test_precopy_rdma_plain_ip(bool ipv6)
>>> {
>>> char buffer[128] = {};
>>> - if (new_rdma_link(buffer)) {
>>> + if (new_rdma_link(buffer, ipv6)) {
>>> g_test_skip("No rdma link available\n"
>>> "# To enable the test:\n"
>>> "# Run \'" RDMA_MIGRATION_HELPER " setup\' with root
>>> to setup"
>>> @@ -159,6 +160,16 @@ static void test_precopy_rdma_plain(void)
>>> test_precopy_common(&args);
>>> }
>>> +
>>> +static void test_precopy_rdma_plain(void)
>>> +{
>>> + test_precopy_rdma_plain_ip(0);
>>> +}
>>> +
>>> +static void test_precopy_rdma_plain_ipv6(void)
>>> +{
>>> + test_precopy_rdma_plain_ip(1);
>>> +}
>>> #endif
>>> static void test_precopy_tcp_plain(void)
>>> @@ -1189,6 +1200,8 @@ static void
>>> migration_test_add_precopy_smoke(MigrationTestEnv *env)
>>> #ifdef CONFIG_RDMA
>>> migration_test_add("/migration/precopy/rdma/plain",
>>> test_precopy_rdma_plain);
>>> + migration_test_add("/migration/precopy/rdma/plain/ipv6",
>>> + test_precopy_rdma_plain_ipv6);
>>> #endif
>>> }
>>> --
>>> 2.41.0
>>>
>>
