Re: [PATCH V4 17/43] vfio-pci: skip reset during cpr

2025-06-01 Thread Cédric Le Goater 

On 5/29/25 21:24, Steve Sistare wrote:

Do not reset a vfio-pci device during CPR, and do not complain if the
kernel's PCI config space changes for non-emulated bits between the
vmstate save and load, which can happen due to ongoing interrupt activity.

Signed-off-by: Steve Sistare 



Reviewed-by: Cédric Le Goater 

Thanks,

C.



---
  include/hw/vfio/vfio-cpr.h |  2 ++
  hw/vfio/cpr.c  | 31 +++
  hw/vfio/pci.c  |  7 +++
  3 files changed, 40 insertions(+)

diff --git a/include/hw/vfio/vfio-cpr.h b/include/hw/vfio/vfio-cpr.h
index 56ede04..8bf85b9 100644
--- a/include/hw/vfio/vfio-cpr.h
+++ b/include/hw/vfio/vfio-cpr.h
@@ -52,4 +52,6 @@ void vfio_cpr_giommu_remap(struct VFIOContainerBase 
*bcontainer,
  bool vfio_cpr_ram_discard_register_listener(
  struct VFIOContainerBase *bcontainer, MemoryRegionSection *section);
  
+extern const VMStateDescription vfio_cpr_pci_vmstate;

+
  #endif /* HW_VFIO_VFIO_CPR_H */
diff --git a/hw/vfio/cpr.c b/hw/vfio/cpr.c
index 0e59612..fdbb58e 100644
--- a/hw/vfio/cpr.c
+++ b/hw/vfio/cpr.c
@@ -8,6 +8,8 @@
  #include "qemu/osdep.h"
  #include "hw/vfio/vfio-device.h"
  #include "hw/vfio/vfio-cpr.h"
+#include "hw/vfio/pci.h"
+#include "migration/cpr.h"
  #include "qapi/error.h"
  #include "system/runstate.h"
  
@@ -37,3 +39,32 @@ void vfio_cpr_unregister_container(VFIOContainerBase *bcontainer)

  {
  migration_remove_notifier(&bcontainer->cpr_reboot_notifier);
  }
+
+/*
+ * The kernel may change non-emulated config bits.  Exclude them from the
+ * changed-bits check in get_pci_config_device.
+ */
+static int vfio_cpr_pci_pre_load(void *opaque)
+{
+VFIOPCIDevice *vdev = opaque;
+PCIDevice *pdev = &vdev->pdev;
+int size = MIN(pci_config_size(pdev), vdev->config_size);
+int i;
+
+for (i = 0; i < size; i++) {
+pdev->cmask[i] &= vdev->emulated_config_bits[i];
+}
+
+return 0;
+}
+
+const VMStateDescription vfio_cpr_pci_vmstate = {
+.name = "vfio-cpr-pci",
+.version_id = 0,
+.minimum_version_id = 0,
+.pre_load = vfio_cpr_pci_pre_load,
+.needed = cpr_incoming_needed,
+.fields = (VMStateField[]) {
+VMSTATE_END_OF_LIST()
+}
+};
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index 56e7fdd..840590c 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -30,6 +30,7 @@
  #include "hw/qdev-properties.h"
  #include "hw/qdev-properties-system.h"
  #include "migration/vmstate.h"
+#include "migration/cpr.h"
  #include "qobject/qdict.h"
  #include "qemu/error-report.h"
  #include "qemu/main-loop.h"
@@ -3345,6 +3346,11 @@ static void vfio_pci_reset(DeviceState *dev)
  {
  VFIOPCIDevice *vdev = VFIO_PCI_BASE(dev);
  
+/* Do not reset the device during qemu_system_reset prior to cpr load */

+if (cpr_is_incoming()) {
+return;
+}
+
  trace_vfio_pci_reset(vdev->vbasedev.name);
  
  vfio_pci_pre_reset(vdev);

@@ -3521,6 +3527,7 @@ static void vfio_pci_dev_class_init(ObjectClass *klass, 
const void *data)
  #ifdef CONFIG_IOMMUFD
  object_class_property_add_str(klass, "fd", NULL, vfio_pci_set_fd);
  #endif
+dc->vmsd = &vfio_cpr_pci_vmstate;
  dc->desc = "VFIO-based PCI device assignment";
  pdc->realize = vfio_realize;

[PATCH V4 17/43] vfio-pci: skip reset during cpr

2025-05-29 Thread Steve Sistare 
Do not reset a vfio-pci device during CPR, and do not complain if the
kernel's PCI config space changes for non-emulated bits between the
vmstate save and load, which can happen due to ongoing interrupt activity.

Signed-off-by: Steve Sistare 
---
 include/hw/vfio/vfio-cpr.h |  2 ++
 hw/vfio/cpr.c  | 31 +++
 hw/vfio/pci.c  |  7 +++
 3 files changed, 40 insertions(+)

diff --git a/include/hw/vfio/vfio-cpr.h b/include/hw/vfio/vfio-cpr.h
index 56ede04..8bf85b9 100644
--- a/include/hw/vfio/vfio-cpr.h
+++ b/include/hw/vfio/vfio-cpr.h
@@ -52,4 +52,6 @@ void vfio_cpr_giommu_remap(struct VFIOContainerBase 
*bcontainer,
 bool vfio_cpr_ram_discard_register_listener(
 struct VFIOContainerBase *bcontainer, MemoryRegionSection *section);
 
+extern const VMStateDescription vfio_cpr_pci_vmstate;
+
 #endif /* HW_VFIO_VFIO_CPR_H */
diff --git a/hw/vfio/cpr.c b/hw/vfio/cpr.c
index 0e59612..fdbb58e 100644
--- a/hw/vfio/cpr.c
+++ b/hw/vfio/cpr.c
@@ -8,6 +8,8 @@
 #include "qemu/osdep.h"
 #include "hw/vfio/vfio-device.h"
 #include "hw/vfio/vfio-cpr.h"
+#include "hw/vfio/pci.h"
+#include "migration/cpr.h"
 #include "qapi/error.h"
 #include "system/runstate.h"
 
@@ -37,3 +39,32 @@ void vfio_cpr_unregister_container(VFIOContainerBase 
*bcontainer)
 {
 migration_remove_notifier(&bcontainer->cpr_reboot_notifier);
 }
+
+/*
+ * The kernel may change non-emulated config bits.  Exclude them from the
+ * changed-bits check in get_pci_config_device.
+ */
+static int vfio_cpr_pci_pre_load(void *opaque)
+{
+VFIOPCIDevice *vdev = opaque;
+PCIDevice *pdev = &vdev->pdev;
+int size = MIN(pci_config_size(pdev), vdev->config_size);
+int i;
+
+for (i = 0; i < size; i++) {
+pdev->cmask[i] &= vdev->emulated_config_bits[i];
+}
+
+return 0;
+}
+
+const VMStateDescription vfio_cpr_pci_vmstate = {
+.name = "vfio-cpr-pci",
+.version_id = 0,
+.minimum_version_id = 0,
+.pre_load = vfio_cpr_pci_pre_load,
+.needed = cpr_incoming_needed,
+.fields = (VMStateField[]) {
+VMSTATE_END_OF_LIST()
+}
+};
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index 56e7fdd..840590c 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -30,6 +30,7 @@
 #include "hw/qdev-properties.h"
 #include "hw/qdev-properties-system.h"
 #include "migration/vmstate.h"
+#include "migration/cpr.h"
 #include "qobject/qdict.h"
 #include "qemu/error-report.h"
 #include "qemu/main-loop.h"
@@ -3345,6 +3346,11 @@ static void vfio_pci_reset(DeviceState *dev)
 {
 VFIOPCIDevice *vdev = VFIO_PCI_BASE(dev);
 
+/* Do not reset the device during qemu_system_reset prior to cpr load */
+if (cpr_is_incoming()) {
+return;
+}
+
 trace_vfio_pci_reset(vdev->vbasedev.name);
 
 vfio_pci_pre_reset(vdev);
@@ -3521,6 +3527,7 @@ static void vfio_pci_dev_class_init(ObjectClass *klass, 
const void *data)
 #ifdef CONFIG_IOMMUFD
 object_class_property_add_str(klass, "fd", NULL, vfio_pci_set_fd);
 #endif
+dc->vmsd = &vfio_cpr_pci_vmstate;
 dc->desc = "VFIO-based PCI device assignment";
 pdc->realize = vfio_realize;
 
-- 
1.8.3.1