Re: [Qemu-devel] [PATCH v2 06/13] spapr/xive: add migration support for KVM
On Fri, Feb 22, 2019 at 02:13:15PM +0100, Cédric Le Goater wrote:
> When the VM is stopped, the VM state handler stabilizes the XIVE IC
> and marks the EQ pages dirty. These are then transferred to destination
> before the transfer of the device vmstates starts.
>
> The sPAPRXive interrupt controller model captures the XIVE internal
> tables, EAT and ENDT and the XiveTCTX model does the same for the
> thread interrupt context registers.
>
> At restart, the sPAPRXive 'post_load' method restores all the XIVE
> states. It is called by the sPAPR machine 'post_load' method, when all
> XIVE states have been transferred and loaded.
>
> Finally, the source states are restored in the VM change state handler
> when the machine reaches the running state.
>
> Signed-off-by: Cédric Le Goater
Reviewed-by: David Gibson
> ---
> include/hw/ppc/spapr_xive.h | 3 ++
> include/hw/ppc/xive.h | 1 +
> hw/intc/spapr_xive.c| 24 ++
> hw/intc/spapr_xive_kvm.c| 93 -
> hw/intc/xive.c | 17 +++
> hw/ppc/spapr_irq.c | 2 +-
> 6 files changed, 138 insertions(+), 2 deletions(-)
>
> diff --git a/include/hw/ppc/spapr_xive.h b/include/hw/ppc/spapr_xive.h
> index 298d204d54ef..22d70650b51f 100644
> --- a/include/hw/ppc/spapr_xive.h
> +++ b/include/hw/ppc/spapr_xive.h
> @@ -55,6 +55,7 @@ typedef struct sPAPRXive {
> bool spapr_xive_irq_claim(sPAPRXive *xive, uint32_t lisn, bool lsi);
> bool spapr_xive_irq_free(sPAPRXive *xive, uint32_t lisn);
> void spapr_xive_pic_print_info(sPAPRXive *xive, Monitor *mon);
> +int spapr_xive_post_load(sPAPRXive *xive, int version_id);
>
> void spapr_xive_hcall_init(sPAPRMachineState *spapr);
> void spapr_dt_xive(sPAPRMachineState *spapr, uint32_t nr_servers, void *fdt,
> @@ -83,5 +84,7 @@ void kvmppc_xive_get_queue_config(sPAPRXive *xive, uint8_t
> end_blk,
> uint32_t end_idx, XiveEND *end,
> Error **errp);
> void kvmppc_xive_synchronize_state(sPAPRXive *xive, Error **errp);
> +int kvmppc_xive_pre_save(sPAPRXive *xive);
> +int kvmppc_xive_post_load(sPAPRXive *xive, int version_id);
>
> #endif /* PPC_SPAPR_XIVE_H */
> diff --git a/include/hw/ppc/xive.h b/include/hw/ppc/xive.h
> index f3766fd881a2..3b1baa783975 100644
> --- a/include/hw/ppc/xive.h
> +++ b/include/hw/ppc/xive.h
> @@ -432,5 +432,6 @@ void kvmppc_xive_source_reset(XiveSource *xsrc, Error
> **errp);
> void kvmppc_xive_source_set_irq(void *opaque, int srcno, int val);
> void kvmppc_xive_cpu_connect(XiveTCTX *tctx, Error **errp);
> void kvmppc_xive_cpu_synchronize_state(XiveTCTX *tctx, Error **errp);
> +void kvmppc_xive_cpu_get_state(XiveTCTX *tctx, Error **errp);
>
> #endif /* PPC_XIVE_H */
> diff --git a/hw/intc/spapr_xive.c b/hw/intc/spapr_xive.c
> index 9f07567f4d78..21fe5e1aa39f 100644
> --- a/hw/intc/spapr_xive.c
> +++ b/hw/intc/spapr_xive.c
> @@ -469,10 +469,34 @@ static const VMStateDescription vmstate_spapr_xive_eas
> = {
> },
> };
>
> +static int vmstate_spapr_xive_pre_save(void *opaque)
> +{
> +if (kvm_irqchip_in_kernel()) {
> +return kvmppc_xive_pre_save(SPAPR_XIVE(opaque));
> +}
> +
> +return 0;
> +}
> +
> +/*
> + * Called by the sPAPR IRQ backend 'post_load' method at the machine
> + * level.
> + */
> +int spapr_xive_post_load(sPAPRXive *xive, int version_id)
> +{
> +if (kvm_irqchip_in_kernel()) {
> +return kvmppc_xive_post_load(xive, version_id);
> +}
> +
> +return 0;
> +}
> +
> static const VMStateDescription vmstate_spapr_xive = {
> .name = TYPE_SPAPR_XIVE,
> .version_id = 1,
> .minimum_version_id = 1,
> +.pre_save = vmstate_spapr_xive_pre_save,
> +.post_load = NULL, /* handled at the machine level */
> .fields = (VMStateField[]) {
> VMSTATE_UINT32_EQUAL(nr_irqs, sPAPRXive, NULL),
> VMSTATE_STRUCT_VARRAY_POINTER_UINT32(eat, sPAPRXive, nr_irqs,
> diff --git a/hw/intc/spapr_xive_kvm.c b/hw/intc/spapr_xive_kvm.c
> index 44d80175b1b5..119fd59fc9ae 100644
> --- a/hw/intc/spapr_xive_kvm.c
> +++ b/hw/intc/spapr_xive_kvm.c
> @@ -15,6 +15,7 @@
> #include "sysemu/cpus.h"
> #include "sysemu/kvm.h"
> #include "hw/ppc/spapr.h"
> +#include "hw/ppc/spapr_cpu_core.h"
> #include "hw/ppc/spapr_xive.h"
> #include "hw/ppc/xive.h"
> #include "kvm_ppc.h"
> @@ -60,7 +61,30 @@ static void kvm_cpu_enable(CPUState *cs)
> /*
> * XIVE Thread Interrupt Management context (KVM)
> */
> -static void kvmppc_xive_cpu_get_state(XiveTCTX *tctx, Error **errp)
> +
> +static void kvmppc_xive_cpu_set_state(XiveTCTX *tctx, Error **errp)
> +{
> +uint64_t state[4];
> +int ret;
> +
> +/* word0 and word1 of the OS ring. */
> +state[0] = *((uint64_t *) >regs[TM_QW1_OS]);
> +
> +/*
> + * OS CAM line. Used by KVM to print out the VP identifier. This
> + * is for debug only.
> + */
> +state[1] = *((uint64_t *) >regs[TM_QW1_OS + TM_WORD2]);
> +
> +ret
[Qemu-devel] [PATCH v2 06/13] spapr/xive: add migration support for KVM
When the VM is stopped, the VM state handler stabilizes the XIVE IC
and marks the EQ pages dirty. These are then transferred to destination
before the transfer of the device vmstates starts.
The sPAPRXive interrupt controller model captures the XIVE internal
tables, EAT and ENDT and the XiveTCTX model does the same for the
thread interrupt context registers.
At restart, the sPAPRXive 'post_load' method restores all the XIVE
states. It is called by the sPAPR machine 'post_load' method, when all
XIVE states have been transferred and loaded.
Finally, the source states are restored in the VM change state handler
when the machine reaches the running state.
Signed-off-by: Cédric Le Goater
---
include/hw/ppc/spapr_xive.h | 3 ++
include/hw/ppc/xive.h | 1 +
hw/intc/spapr_xive.c| 24 ++
hw/intc/spapr_xive_kvm.c| 93 -
hw/intc/xive.c | 17 +++
hw/ppc/spapr_irq.c | 2 +-
6 files changed, 138 insertions(+), 2 deletions(-)
diff --git a/include/hw/ppc/spapr_xive.h b/include/hw/ppc/spapr_xive.h
index 298d204d54ef..22d70650b51f 100644
--- a/include/hw/ppc/spapr_xive.h
+++ b/include/hw/ppc/spapr_xive.h
@@ -55,6 +55,7 @@ typedef struct sPAPRXive {
bool spapr_xive_irq_claim(sPAPRXive *xive, uint32_t lisn, bool lsi);
bool spapr_xive_irq_free(sPAPRXive *xive, uint32_t lisn);
void spapr_xive_pic_print_info(sPAPRXive *xive, Monitor *mon);
+int spapr_xive_post_load(sPAPRXive *xive, int version_id);
void spapr_xive_hcall_init(sPAPRMachineState *spapr);
void spapr_dt_xive(sPAPRMachineState *spapr, uint32_t nr_servers, void *fdt,
@@ -83,5 +84,7 @@ void kvmppc_xive_get_queue_config(sPAPRXive *xive, uint8_t
end_blk,
uint32_t end_idx, XiveEND *end,
Error **errp);
void kvmppc_xive_synchronize_state(sPAPRXive *xive, Error **errp);
+int kvmppc_xive_pre_save(sPAPRXive *xive);
+int kvmppc_xive_post_load(sPAPRXive *xive, int version_id);
#endif /* PPC_SPAPR_XIVE_H */
diff --git a/include/hw/ppc/xive.h b/include/hw/ppc/xive.h
index f3766fd881a2..3b1baa783975 100644
--- a/include/hw/ppc/xive.h
+++ b/include/hw/ppc/xive.h
@@ -432,5 +432,6 @@ void kvmppc_xive_source_reset(XiveSource *xsrc, Error
**errp);
void kvmppc_xive_source_set_irq(void *opaque, int srcno, int val);
void kvmppc_xive_cpu_connect(XiveTCTX *tctx, Error **errp);
void kvmppc_xive_cpu_synchronize_state(XiveTCTX *tctx, Error **errp);
+void kvmppc_xive_cpu_get_state(XiveTCTX *tctx, Error **errp);
#endif /* PPC_XIVE_H */
diff --git a/hw/intc/spapr_xive.c b/hw/intc/spapr_xive.c
index 9f07567f4d78..21fe5e1aa39f 100644
--- a/hw/intc/spapr_xive.c
+++ b/hw/intc/spapr_xive.c
@@ -469,10 +469,34 @@ static const VMStateDescription vmstate_spapr_xive_eas = {
},
};
+static int vmstate_spapr_xive_pre_save(void *opaque)
+{
+if (kvm_irqchip_in_kernel()) {
+return kvmppc_xive_pre_save(SPAPR_XIVE(opaque));
+}
+
+return 0;
+}
+
+/*
+ * Called by the sPAPR IRQ backend 'post_load' method at the machine
+ * level.
+ */
+int spapr_xive_post_load(sPAPRXive *xive, int version_id)
+{
+if (kvm_irqchip_in_kernel()) {
+return kvmppc_xive_post_load(xive, version_id);
+}
+
+return 0;
+}
+
static const VMStateDescription vmstate_spapr_xive = {
.name = TYPE_SPAPR_XIVE,
.version_id = 1,
.minimum_version_id = 1,
+.pre_save = vmstate_spapr_xive_pre_save,
+.post_load = NULL, /* handled at the machine level */
.fields = (VMStateField[]) {
VMSTATE_UINT32_EQUAL(nr_irqs, sPAPRXive, NULL),
VMSTATE_STRUCT_VARRAY_POINTER_UINT32(eat, sPAPRXive, nr_irqs,
diff --git a/hw/intc/spapr_xive_kvm.c b/hw/intc/spapr_xive_kvm.c
index 44d80175b1b5..119fd59fc9ae 100644
--- a/hw/intc/spapr_xive_kvm.c
+++ b/hw/intc/spapr_xive_kvm.c
@@ -15,6 +15,7 @@
#include "sysemu/cpus.h"
#include "sysemu/kvm.h"
#include "hw/ppc/spapr.h"
+#include "hw/ppc/spapr_cpu_core.h"
#include "hw/ppc/spapr_xive.h"
#include "hw/ppc/xive.h"
#include "kvm_ppc.h"
@@ -60,7 +61,30 @@ static void kvm_cpu_enable(CPUState *cs)
/*
* XIVE Thread Interrupt Management context (KVM)
*/
-static void kvmppc_xive_cpu_get_state(XiveTCTX *tctx, Error **errp)
+
+static void kvmppc_xive_cpu_set_state(XiveTCTX *tctx, Error **errp)
+{
+uint64_t state[4];
+int ret;
+
+/* word0 and word1 of the OS ring. */
+state[0] = *((uint64_t *) >regs[TM_QW1_OS]);
+
+/*
+ * OS CAM line. Used by KVM to print out the VP identifier. This
+ * is for debug only.
+ */
+state[1] = *((uint64_t *) >regs[TM_QW1_OS + TM_WORD2]);
+
+ret = kvm_set_one_reg(tctx->cs, KVM_REG_PPC_NVT_STATE, state);
+if (ret != 0) {
+error_setg_errno(errp, errno,
+ "XIVE: could not restore KVM state of CPU %ld",
+ kvm_arch_vcpu_id(tctx->cs));
+}
+}
+
+void kvmppc_xive_cpu_get_state(XiveTCTX *tctx, Error **errp)
{
uint64_t state[4] = { 0 };
