This is v4 of this patch to add the function of saving and restoring the
running status of vCPU during migration
This patch fixes two critical bugs in QEMU with KVM:
Post-Migration Failure in User Mode: When QEMU with KVM is running in user
mode, the guest may fail to function correctly after migration.
Multi-Core Guest Inconsistency: After migration, only the first CPU (core 0)
remains functional, while all other cores become unresponsive.
This patch addresses both problems to ensure stable guest operation after
migration.
Signed-off-by: Xie Bo <x...@ultrarisc.com>
---
target/riscv/cpu.h | 1 +
target/riscv/kvm/kvm-cpu.c | 63 ++++++++++++++++++++++++++++++++----
target/riscv/kvm/kvm_riscv.h | 3 +-
target/riscv/machine.c | 1 +
4 files changed, 61 insertions(+), 7 deletions(-)
diff --git a/target/riscv/cpu.h b/target/riscv/cpu.h
index 51e49e03dec..1d7ad598faa 100644
--- a/target/riscv/cpu.h
+++ b/target/riscv/cpu.h
@@ -256,6 +256,7 @@ struct CPUArchState {
#endif
target_ulong priv;
+ uint32_t mp_state; /*current multiprocessor state of this vCPU*/
/* CSRs for execution environment configuration */
uint64_t menvcfg;
target_ulong senvcfg;
diff --git a/target/riscv/kvm/kvm-cpu.c b/target/riscv/kvm/kvm-cpu.c
index 0f4997a9186..c4c7c606a33 100644
--- a/target/riscv/kvm/kvm-cpu.c
+++ b/target/riscv/kvm/kvm-cpu.c
@@ -576,6 +576,15 @@ static int kvm_riscv_get_regs_core(CPUState *cs)
}
env->pc = reg;
+ /*Save the guest's privileged state before migration*/
+ ret = kvm_get_one_reg(cs, RISCV_CORE_REG(env, mode), ®);
+ if (ret) {
+ return ret;
+ }
+ if(reg != PRV_M) {
+ env->priv = reg;
+ }
+
for (i = 1; i < 32; i++) {
uint64_t id = kvm_riscv_reg_id_ulong(env, KVM_REG_RISCV_CORE, i);
ret = kvm_get_one_reg(cs, id, ®);
@@ -601,6 +610,16 @@ static int kvm_riscv_put_regs_core(CPUState *cs)
return ret;
}
+ /*Restore the guest's privileged state after migration*/
+ reg = env->priv;
+
+ if(reg != PRV_M) {
+ ret = kvm_set_one_reg(cs, RISCV_CORE_REG(env, mode), ®);
+ if (ret) {
+ return ret;
+ }
+ }
+
for (i = 1; i < 32; i++) {
uint64_t id = kvm_riscv_reg_id_ulong(env, KVM_REG_RISCV_CORE, i);
reg = env->gpr[i];
@@ -1244,22 +1263,46 @@ int kvm_arch_get_registers(CPUState *cs, Error **errp)
return ret;
}
+ ret = kvm_riscv_sync_mpstate_to_qemu(cs);
+ if (ret) {
+ return ret;
+ }
+
return ret;
}
-int kvm_riscv_sync_mpstate_to_kvm(RISCVCPU *cpu, int state)
+int kvm_riscv_sync_mpstate_to_kvm(CPUState *cs)
{
+ CPURISCVState *env = &RISCV_CPU(cs)->env;
if (cap_has_mp_state) {
struct kvm_mp_state mp_state = {
- .mp_state = state
+ .mp_state = env->mp_state
};
- int ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MP_STATE, &mp_state);
+ int ret = kvm_vcpu_ioctl(cs, KVM_SET_MP_STATE, &mp_state);
+ if (ret) {
+ fprintf(stderr, "%s: failed to sync MP_STATE %d/%s\n",
+ __func__, ret, strerror(-ret));
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+int kvm_riscv_sync_mpstate_to_qemu(CPUState *cs)
+{
+ CPURISCVState *env = &RISCV_CPU(cs)->env;
+ if (cap_has_mp_state) {
+ struct kvm_mp_state mp_state;
+
+ int ret = kvm_vcpu_ioctl(cs, KVM_GET_MP_STATE, &mp_state);
if (ret) {
fprintf(stderr, "%s: failed to sync MP_STATE %d/%s\n",
__func__, ret, strerror(-ret));
return -1;
}
+ env->mp_state = mp_state.mp_state;
}
return 0;
@@ -1290,16 +1333,24 @@ int kvm_arch_put_registers(CPUState *cs, int level,
Error **errp)
}
if (KVM_PUT_RESET_STATE == level) {
- RISCVCPU *cpu = RISCV_CPU(cs);
+ CPURISCVState *env = &RISCV_CPU(cs)->env;
if (cs->cpu_index == 0) {
- ret = kvm_riscv_sync_mpstate_to_kvm(cpu, KVM_MP_STATE_RUNNABLE);
+ env->mp_state = KVM_MP_STATE_RUNNABLE;
+ ret = kvm_riscv_sync_mpstate_to_kvm(cs);
} else {
- ret = kvm_riscv_sync_mpstate_to_kvm(cpu, KVM_MP_STATE_STOPPED);
+ env->mp_state = KVM_MP_STATE_STOPPED;
+ ret = kvm_riscv_sync_mpstate_to_kvm(cs);
}
if (ret) {
return ret;
}
}
+ else {
+ ret = kvm_riscv_sync_mpstate_to_kvm(cs);
+ if (ret) {
+ return ret;
+ }
+ }
return ret;
}
diff --git a/target/riscv/kvm/kvm_riscv.h b/target/riscv/kvm/kvm_riscv.h
index b2bcd1041f6..953db941605 100644
--- a/target/riscv/kvm/kvm_riscv.h
+++ b/target/riscv/kvm/kvm_riscv.h
@@ -28,7 +28,8 @@ void kvm_riscv_aia_create(MachineState *machine, uint64_t
group_shift,
uint64_t aplic_base, uint64_t imsic_base,
uint64_t guest_num);
void riscv_kvm_aplic_request(void *opaque, int irq, int level);
-int kvm_riscv_sync_mpstate_to_kvm(RISCVCPU *cpu, int state);
+int kvm_riscv_sync_mpstate_to_kvm(CPUState *cs);
+int kvm_riscv_sync_mpstate_to_qemu(CPUState *cs);
void riscv_kvm_cpu_finalize_features(RISCVCPU *cpu, Error **errp);
uint64_t kvm_riscv_get_timebase_frequency(RISCVCPU *cpu);
diff --git a/target/riscv/machine.c b/target/riscv/machine.c
index 889e2b65701..22edd2dd744 100644
--- a/target/riscv/machine.c
+++ b/target/riscv/machine.c
@@ -422,6 +422,7 @@ const VMStateDescription vmstate_riscv_cpu = {
VMSTATE_UNUSED(4),
VMSTATE_UINT32(env.misa_ext_mask, RISCVCPU),
VMSTATE_UINTTL(env.priv, RISCVCPU),
+ VMSTATE_UINT32(env.mp_state, RISCVCPU),
VMSTATE_BOOL(env.virt_enabled, RISCVCPU),
VMSTATE_UINT64(env.resetvec, RISCVCPU),
VMSTATE_UINTTL(env.mhartid, RISCVCPU),
--
> -----原始邮件-----
> 发件人: 谢波 <x...@ultrarisc.com>
> 发送时间:2025-07-11 17:28:10 (星期五)
> 收件人: qemu-devel@nongnu.org
> 抄送: alistair.fran...@wdc.com, pal...@dabbelt.com, pbonz...@redhat.com,
> a...@brainfault.org, anup.pa...@wdc.com
> 主题: [PATCH for v10.0.0] target/riscv/kvm/kvm-cpu: Fixed the issue of resume
> after QEMU+KVM migration
>
> This is v3 of this patch to fix patch format
>
> This patch fixes two critical issues in QEMU with KVM:
> 1. Post-Migration Failure in User Mode: When QEMU with KVM is running in user
> mode, the guest may fail to function correctly after migration.
> 2. Multi-Core Guest Inconsistency: After migration, only the first CPU (core
> 0) remains functional, while all other cores become unresponsive.
>
> Changes include:
> - Properly restoring guest privileged state during register synchronization.
> - Correctly updating multi-core state after migration to ensure all cores are
> active.
>
> Signed-off-by: Xie Bo <x...@ultrarisc.com>
> ---
> target/riscv/kvm/kvm-cpu.c | 23 +++++++++++++++++++++++
> 1 file changed, 23 insertions(+)
>
> diff --git a/target/riscv/kvm/kvm-cpu.c b/target/riscv/kvm/kvm-cpu.c
> index 75724b6af4..a15caa20ce 100644
> --- a/target/riscv/kvm/kvm-cpu.c
> +++ b/target/riscv/kvm/kvm-cpu.c
> @@ -576,6 +576,14 @@ static int kvm_riscv_get_regs_core(CPUState *cs)
> }
> env->pc = reg;
>
> + /*Restore the guest's privileged level after migration*/
> + ret = kvm_get_one_reg(cs, RISCV_CORE_REG(env, mode), ®);
> + if (ret) {
> + return ret;
> + }
> + if(reg != 3) {
> + env->priv = reg;
> + }
> for (i = 1; i < 32; i++) {
> uint64_t id = kvm_riscv_reg_id_ulong(env, KVM_REG_RISCV_CORE, i);
> ret = kvm_get_one_reg(cs, id, ®);
> @@ -601,6 +609,15 @@ static int kvm_riscv_put_regs_core(CPUState *cs)
> return ret;
> }
>
> + /*Save guest privilege level before migration*/
> + reg = env->priv;
> + if(reg != 3) {
> + ret = kvm_set_one_reg(cs, RISCV_CORE_REG(env, mode), ®);
> + if (ret) {
> + return ret;
> + }
> + }
> +
> for (i = 1; i < 32; i++) {
> uint64_t id = kvm_riscv_reg_id_ulong(env, KVM_REG_RISCV_CORE, i);
> reg = env->gpr[i];
> @@ -1289,6 +1306,12 @@ int kvm_arch_put_registers(CPUState *cs, int level,
> Error **errp)
> return ret;
> }
>
> + /*Ensure all non-core 0 CPUs are runnable after migration*/
> + if((level == KVM_PUT_FULL_STATE) && (cs->cpu_index != 0)){
> + RISCVCPU *cpu = RISCV_CPU(cs);
> + ret = kvm_riscv_sync_mpstate_to_kvm(cpu, KVM_MP_STATE_RUNNABLE);
> + }
> +
> if (KVM_PUT_RESET_STATE == level) {
> RISCVCPU *cpu = RISCV_CPU(cs);
> if (cs->cpu_index == 0) {
> --
> 2.34.1
>
>
>
>
> > -----原始邮件-----
> > 发件人: 谢波 <x...@ultrarisc.com>
> > 发送时间:2025-05-26 15:45:52 (星期一)
> > 收件人: qemu-devel@nongnu.org
> > 抄送: alistair.fran...@wdc.com, pal...@dabbelt.com, pbonz...@redhat.com,
> > a...@brainfault.org, anup.pa...@wdc.com
> > 主题: Re: [PATCH V2] target/riscv/kvm/kvm-cpu: Fixed the issue of resume
> > after QEMU+KVM migration
> >
> > This is v2 of this patch with no functional changes; adding CC.
> >
> > ---
> > target/riscv/kvm/kvm-cpu.c | 23 +++++++++++++++++++++++
> > 1 file changed, 23 insertions(+)
> >
> > --- a/target/riscv/kvm/kvm-cpu.c
> > +++ b/target/riscv/kvm/kvm-cpu.c
> > @@ -576,6 +576,14 @@ static int kvm_riscv_get_regs_core(CPUState *cs)
> > }
> > env->pc = reg;
> >
> > + /* Restore guest privilege level after migration */
> > + ret = kvm_get_one_reg(cs, RISCV_CORE_REG(env, mode), ®);
> > + if (ret) {
> > + return ret;
> > + }
> > + if (reg != 3) {
> > + env->priv = reg;
> > + }
> >
> > for (i = 1; i < 32; i++) {
> > uint64_t id = kvm_riscv_reg_id_ulong(env, KVM_REG_RISCV_CORE, i);
> > ret = kvm_get_one_reg(cs, id, ®);
> > @@ -601,6 +609,15 @@ static int kvm_riscv_put_regs_core(CPUState *cs)
> > return ret;
> > }
> >
> > + /* Save guest privilege level before migration */
> > + reg = env->priv;
> > + if (reg != 3) {
> > + ret = kvm_set_one_reg(cs, RISCV_CORE_REG(env, mode), ®);
> > + if (ret) {
> > + return ret;
> > + }
> > + }
> > +
> > for (i = 1; i < 32; i++) {
> > uint64_t id = kvm_riscv_reg_id_ulong(env, KVM_REG_RISCV_CORE, i);
> > reg = env->gpr[i];
> > @@ -1289,6 +1306,12 @@ int kvm_arch_put_registers(CPUState *cs, int level,
> > Error **errp)
> > return ret;
> > }
> >
> > + /* Ensure all non-core 0 CPUs are runnable after migration */
> > + if ((level == KVM_PUT_FULL_STATE) && (cs->cpu_index != 0)) {
> > + RISCVCPU *cpu = RISCV_CPU(cs);
> > + ret = kvm_riscv_sync_mpstate_to_kvm(cpu, KVM_MP_STATE_RUNNABLE);
> > + if (ret) {
> > + return ret;
> > + }
> > + }
> >
> > if (KVM_PUT_RESET_STATE == level) {
> > RISCVCPU *cpu = RISCV_CPU(cs);
> > if (cs->cpu_index == 0) {
> > --
> > 2.34.1
> >
> >
> >
> >
> > > -----原始邮件-----
> > > 发件人: 谢波 <x...@ultrarisc.com>
> > > 发送时间:2025-05-19 17:41:36 (星期一)
> > > 收件人: qemu-devel@nongnu.org
> > > 抄送: alistair.fran...@wdc.com, pal...@dabbelt.com, pbonz...@redhat.com
> > > 主题: [PATCH] target/riscv/kvm/kvm-cpu: Fixed the issue of resume after
> > > QEMU+KVM migration
> > >
> > > This patch fixes two critical issues in QEMU with KVM:
> > >
> > > 1. Post-Migration Failure in User Mode: When QEMU with KVM is running in
> > > user mode, the guest may fail to function correctly after migration due
> > > to incorrect privilege state restoration.
> > >
> > > 2. Multi-Core Guest Inconsistency: After migration, only the first CPU
> > > (core 0) remains functional, while all other cores become unresponsive.
> > > This patch ensures all cores are properly set to runnable state after
> > > migration.
> > >
> > > Changes include:
> > > - Properly restoring guest privileged state during register
> > > synchronization.
> > > - Correctly updating multi-core state after migration to ensure all cores
> > > are active.
> > >
> > > Signed-off-by: Xie Bo <x...@ultrarisc.com>
> > >
> > > ---
> > > target/riscv/kvm/kvm-cpu.c | 23 +++++++++++++++++++++++
> > > 1 file changed, 23 insertions(+)
> > >
> > > --- a/target/riscv/kvm/kvm-cpu.c
> > > +++ b/target/riscv/kvm/kvm-cpu.c
> > > @@ -576,6 +576,14 @@ static int kvm_riscv_get_regs_core(CPUState *cs)
> > > }
> > > env->pc = reg;
> > >
> > > + /* Restore guest privilege level after migration */
> > > + ret = kvm_get_one_reg(cs, RISCV_CORE_REG(env, mode), ®);
> > > + if (ret) {
> > > + return ret;
> > > + }
> > > + if (reg != 3) {
> > > + env->priv = reg;
> > > + }
> > >
> > > for (i = 1; i < 32; i++) {
> > > uint64_t id = kvm_riscv_reg_id_ulong(env, KVM_REG_RISCV_CORE, i);
> > > ret = kvm_get_one_reg(cs, id, ®);
> > > @@ -601,6 +609,15 @@ static int kvm_riscv_put_regs_core(CPUState *cs)
> > > return ret;
> > > }
> > >
> > > + /* Save guest privilege level before migration */
> > > + reg = env->priv;
> > > + if (reg != 3) {
> > > + ret = kvm_set_one_reg(cs, RISCV_CORE_REG(env, mode), ®);
> > > + if (ret) {
> > > + return ret;
> > > + }
> > > + }
> > > +
> > > for (i = 1; i < 32; i++) {
> > > uint64_t id = kvm_riscv_reg_id_ulong(env, KVM_REG_RISCV_CORE, i);
> > > reg = env->gpr[i];
> > > @@ -1289,6 +1306,12 @@ int kvm_arch_put_registers(CPUState *cs, int
> > > level, Error **errp)
> > > return ret;
> > > }
> > >
> > > + /* Ensure all non-core 0 CPUs are runnable after migration */
> > > + if ((level == KVM_PUT_FULL_STATE) && (cs->cpu_index != 0)) {
> > > + RISCVCPU *cpu = RISCV_CPU(cs);
> > > + ret = kvm_riscv_sync_mpstate_to_kvm(cpu, KVM_MP_STATE_RUNNABLE);
> > > + if (ret) {
> > > + return ret;
> > > + }
> > > + }
> > >
> > > if (KVM_PUT_RESET_STATE == level) {
> > > RISCVCPU *cpu = RISCV_CPU(cs);
> > > if (cs->cpu_index == 0) {
> > > --
> > > 2.34.1
> > >
> > > ______________________www.ultrarisc.com
> > > 重要提示：本邮件包括附件的内容是受法律保护的保密信息，如果您不是指定收件人，请立即将本邮件删除，法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定，如无我方的正式书面澄清或授权，不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT
> > > NOTICE: This email, including its attachment if any, is confidential. If
> > > you are not the intended recipient, please delete it from your computer
> > > immediately. Any disclosure, copying, or distribution of this message, or
> > > taking of any action based on it is strictly prohibited. Any opinions
> > > and suggestions contained in this email are subject to the terms and
> > > conditions expressed and defined by us and should not be relied upon
> > > unconditionally under any circumstances unless they are confirmed in
> > > official written clarification or authorization from us. Thank you for
> > > your understanding and cooperation.All rights reserved.
> >
> >
> > ______________________www.ultrarisc.com
> > 重要提示：本邮件包括附件的内容是受法律保护的保密信息，如果您不是指定收件人，请立即将本邮件删除，法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定，如无我方的正式书面澄清或授权，不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT
> > NOTICE: This email, including its attachment if any, is confidential. If
> > you are not the intended recipient, please delete it from your computer
> > immediately. Any disclosure, copying, or distribution of this message, or
> > taking of any action based on it is strictly prohibited. Any opinions and
> > suggestions contained in this email are subject to the terms and conditions
> > expressed and defined by us and should not be relied upon unconditionally
> > under any circumstances unless they are confirmed in official written
> > clarification or authorization from us. Thank you for your understanding
> > and cooperation.All rights reserved.
>
>
> ______________________www.ultrarisc.com
> 重要提示：本邮件包括附件的内容是受法律保护的保密信息，如果您不是指定收件人，请立即将本邮件删除，法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定，如无我方的正式书面澄清或授权，不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT
> NOTICE: This email, including its attachment if any, is confidential. If you
> are not the intended recipient, please delete it from your computer
> immediately. Any disclosure, copying, or distribution of this message, or
> taking of any action based on it is strictly prohibited. Any opinions and
> suggestions contained in this email are subject to the terms and conditions
> expressed and defined by us and should not be relied upon unconditionally
> under any circumstances unless they are confirmed in official written
> clarification or authorization from us. Thank you for your understanding and
> cooperation.All rights reserved.
______________________www.ultrarisc.com
重要提示：本邮件包括附件的内容是受法律保护的保密信息，如果您不是指定收件人，请立即将本邮件删除，法律禁止任何非法的披露、复制、传播或以任何方式使用本邮件。本邮件中包含的意见、建议是基于或受到我方表达和定义的条款及条件的限定，如无我方的正式书面澄清或授权，不可被单独作为任何情形下的证据或依据。感谢您的理解与配合。版权所有。IMPORTANT
NOTICE: This email, including its attachment if any, is confidential. If you
are not the intended recipient, please delete it from your computer
immediately. Any disclosure, copying, or distribution of this message, or
taking of any action based on it is strictly prohibited. Any opinions and
suggestions contained in this email are subject to the terms and conditions
expressed and defined by us and should not be relied upon unconditionally under
any circumstances unless they are confirmed in official written clarification
or authorization from us. Thank you for your understanding and cooperation.All
rights reserved.
