Re: [PATCH v27 05/17] vfio: Add VM state change handler to know state of VM
On Thu, 22 Oct 2020 23:11:39 +0530
Kirti Wankhede wrote:
> On 10/22/2020 10:05 PM, Alex Williamson wrote:
> > On Thu, 22 Oct 2020 16:41:55 +0530
> > Kirti Wankhede wrote:
> >
> >> VM state change handler is called on change in VM's state. Based on
> >> VM state, VFIO device state should be changed.
> >> Added read/write helper functions for migration region.
> >> Added function to set device_state.
> >>
> >> Signed-off-by: Kirti Wankhede
> >> Reviewed-by: Neo Jia
> >> Reviewed-by: Dr. David Alan Gilbert
> >> ---
> >> hw/vfio/migration.c | 158
> >> ++
> >> hw/vfio/trace-events | 2 +
> >> include/hw/vfio/vfio-common.h | 4 ++
> >> 3 files changed, 164 insertions(+)
> >>
> >> diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c
> >> index 5f74a3ad1d72..34f39c7e2e28 100644
> >> --- a/hw/vfio/migration.c
> >> +++ b/hw/vfio/migration.c
> >> @@ -10,6 +10,7 @@
> >> #include "qemu/osdep.h"
> >> #include
> >>
> >> +#include "sysemu/runstate.h"
> >> #include "hw/vfio/vfio-common.h"
> >> #include "cpu.h"
> >> #include "migration/migration.h"
> >> @@ -22,6 +23,157 @@
> >> #include "exec/ram_addr.h"
> >> #include "pci.h"
> >> #include "trace.h"
> >> +#include "hw/hw.h"
> >> +
> >> +static inline int vfio_mig_access(VFIODevice *vbasedev, void *val, int
> >> count,
> >> + off_t off, bool iswrite)
> >> +{
> >> +int ret;
> >> +
> >> +ret = iswrite ? pwrite(vbasedev->fd, val, count, off) :
> >> +pread(vbasedev->fd, val, count, off);
> >> +if (ret < count) {
> >> +error_report("vfio_mig_%s %d byte %s: failed at offset 0x%lx,
> >> err: %s",
> >> + iswrite ? "write" : "read", count,
> >> + vbasedev->name, off, strerror(errno));
> >> +return (ret < 0) ? ret : -EINVAL;
> >> +}
> >> +return 0;
> >> +}
> >> +
> >> +static int vfio_mig_rw(VFIODevice *vbasedev, __u8 *buf, size_t count,
> >> + off_t off, bool iswrite)
> >> +{
> >> +int ret, done = 0;
> >> +__u8 *tbuf = buf;
> >> +
> >> +while (count) {
> >> +int bytes = 0;
> >> +
> >> +if (count >= 8 && !(off % 8)) {
> >> +bytes = 8;
> >> +} else if (count >= 4 && !(off % 4)) {
> >> +bytes = 4;
> >> +} else if (count >= 2 && !(off % 2)) {
> >> +bytes = 2;
> >> +} else {
> >> +bytes = 1;
> >> +}
> >> +
> >> +ret = vfio_mig_access(vbasedev, tbuf, bytes, off, iswrite);
> >> +if (ret) {
> >> +return ret;
> >> +}
> >> +
> >> +count -= bytes;
> >> +done += bytes;
> >> +off += bytes;
> >> +tbuf += bytes;
> >> +}
> >> +return done;
> >> +}
> >> +
> >> +#define vfio_mig_read(f, v, c, o) vfio_mig_rw(f, (__u8 *)v, c, o,
> >> false)
> >> +#define vfio_mig_write(f, v, c, o) vfio_mig_rw(f, (__u8 *)v, c, o,
> >> true)
> >> +
> >> +#define VFIO_MIG_STRUCT_OFFSET(f) \
> >> + offsetof(struct
> >> vfio_device_migration_info, f)
> >> +/*
> >> + * Change the device_state register for device @vbasedev. Bits set in
> >> @mask
> >> + * are preserved, bits set in @value are set, and bits not set in either
> >> @mask
> >> + * or @value are cleared in device_state. If the register cannot be
> >> accessed,
> >> + * the resulting state would be invalid, or the device enters an error
> >> state,
> >> + * an error is returned.
> >> + */
> >> +
> >> +static int vfio_migration_set_state(VFIODevice *vbasedev, uint32_t mask,
> >> +uint32_t value)
> >> +{
> >> +VFIOMigration *migration = vbasedev->migration;
> >> +VFIORegion *region = >region;
> >> +off_t dev_state_off = region->fd_offset +
> >> + VFIO_MIG_STRUCT_OFFSET(device_state);
> >> +uint32_t device_state;
> >> +int ret;
> >> +
> >> +ret = vfio_mig_read(vbasedev, _state, sizeof(device_state),
> >> +dev_state_off);
> >> +if (ret < 0) {
> >> +return ret;
> >> +}
> >> +
> >> +device_state = (device_state & mask) | value;
> >> +
> >> +if (!VFIO_DEVICE_STATE_VALID(device_state)) {
> >> +return -EINVAL;
> >> +}
> >> +
> >> +ret = vfio_mig_write(vbasedev, _state, sizeof(device_state),
> >> + dev_state_off);
> >> +if (ret < 0) {
> >> +int rret;
> >> +
> >> +rret = vfio_mig_read(vbasedev, _state,
> >> sizeof(device_state),
> >> + dev_state_off);
> >> +
> >> +if ((rret < 0) || (VFIO_DEVICE_STATE_IS_ERROR(device_state))) {
> >> +hw_error("%s: Device in error state 0x%x", vbasedev->name,
> >> + device_state);
> >> +return rret ? rret : -EIO;
> >> +}
> >> +return ret;
> >> +}
> >> +
> >> +
Re: [PATCH v27 05/17] vfio: Add VM state change handler to know state of VM
On 10/22/2020 10:05 PM, Alex Williamson wrote:
On Thu, 22 Oct 2020 16:41:55 +0530
Kirti Wankhede wrote:
VM state change handler is called on change in VM's state. Based on
VM state, VFIO device state should be changed.
Added read/write helper functions for migration region.
Added function to set device_state.
Signed-off-by: Kirti Wankhede
Reviewed-by: Neo Jia
Reviewed-by: Dr. David Alan Gilbert
---
hw/vfio/migration.c | 158 ++
hw/vfio/trace-events | 2 +
include/hw/vfio/vfio-common.h | 4 ++
3 files changed, 164 insertions(+)
diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c
index 5f74a3ad1d72..34f39c7e2e28 100644
--- a/hw/vfio/migration.c
+++ b/hw/vfio/migration.c
@@ -10,6 +10,7 @@
#include "qemu/osdep.h"
#include
+#include "sysemu/runstate.h"
#include "hw/vfio/vfio-common.h"
#include "cpu.h"
#include "migration/migration.h"
@@ -22,6 +23,157 @@
#include "exec/ram_addr.h"
#include "pci.h"
#include "trace.h"
+#include "hw/hw.h"
+
+static inline int vfio_mig_access(VFIODevice *vbasedev, void *val, int count,
+ off_t off, bool iswrite)
+{
+int ret;
+
+ret = iswrite ? pwrite(vbasedev->fd, val, count, off) :
+pread(vbasedev->fd, val, count, off);
+if (ret < count) {
+error_report("vfio_mig_%s %d byte %s: failed at offset 0x%lx, err: %s",
+ iswrite ? "write" : "read", count,
+ vbasedev->name, off, strerror(errno));
+return (ret < 0) ? ret : -EINVAL;
+}
+return 0;
+}
+
+static int vfio_mig_rw(VFIODevice *vbasedev, __u8 *buf, size_t count,
+ off_t off, bool iswrite)
+{
+int ret, done = 0;
+__u8 *tbuf = buf;
+
+while (count) {
+int bytes = 0;
+
+if (count >= 8 && !(off % 8)) {
+bytes = 8;
+} else if (count >= 4 && !(off % 4)) {
+bytes = 4;
+} else if (count >= 2 && !(off % 2)) {
+bytes = 2;
+} else {
+bytes = 1;
+}
+
+ret = vfio_mig_access(vbasedev, tbuf, bytes, off, iswrite);
+if (ret) {
+return ret;
+}
+
+count -= bytes;
+done += bytes;
+off += bytes;
+tbuf += bytes;
+}
+return done;
+}
+
+#define vfio_mig_read(f, v, c, o) vfio_mig_rw(f, (__u8 *)v, c, o, false)
+#define vfio_mig_write(f, v, c, o) vfio_mig_rw(f, (__u8 *)v, c, o, true)
+
+#define VFIO_MIG_STRUCT_OFFSET(f) \
+ offsetof(struct vfio_device_migration_info, f)
+/*
+ * Change the device_state register for device @vbasedev. Bits set in @mask
+ * are preserved, bits set in @value are set, and bits not set in either @mask
+ * or @value are cleared in device_state. If the register cannot be accessed,
+ * the resulting state would be invalid, or the device enters an error state,
+ * an error is returned.
+ */
+
+static int vfio_migration_set_state(VFIODevice *vbasedev, uint32_t mask,
+uint32_t value)
+{
+VFIOMigration *migration = vbasedev->migration;
+VFIORegion *region = >region;
+off_t dev_state_off = region->fd_offset +
+ VFIO_MIG_STRUCT_OFFSET(device_state);
+uint32_t device_state;
+int ret;
+
+ret = vfio_mig_read(vbasedev, _state, sizeof(device_state),
+dev_state_off);
+if (ret < 0) {
+return ret;
+}
+
+device_state = (device_state & mask) | value;
+
+if (!VFIO_DEVICE_STATE_VALID(device_state)) {
+return -EINVAL;
+}
+
+ret = vfio_mig_write(vbasedev, _state, sizeof(device_state),
+ dev_state_off);
+if (ret < 0) {
+int rret;
+
+rret = vfio_mig_read(vbasedev, _state, sizeof(device_state),
+ dev_state_off);
+
+if ((rret < 0) || (VFIO_DEVICE_STATE_IS_ERROR(device_state))) {
+hw_error("%s: Device in error state 0x%x", vbasedev->name,
+ device_state);
+return rret ? rret : -EIO;
+}
+return ret;
+}
+
+migration->device_state = device_state;
+trace_vfio_migration_set_state(vbasedev->name, device_state);
+return 0;
+}
+
+static void vfio_vmstate_change(void *opaque, int running, RunState state)
+{
+VFIODevice *vbasedev = opaque;
+VFIOMigration *migration = vbasedev->migration;
+uint32_t value, mask;
+int ret;
+
+if ((vbasedev->migration->vm_running == running)) {
+return;
+}
+
+if (running) {
+/*
+ * Here device state can have one of _SAVING, _RESUMING or _STOP bit.
+ * Transition from _SAVING to _RUNNING can happen if there is migration
+ * failure, in that case clear _SAVING bit.
+ * Transition from _RESUMING to _RUNNING occurs during resuming
+ * phase, in that case clear _RESUMING bit.
+
Re: [PATCH v27 05/17] vfio: Add VM state change handler to know state of VM
On Thu, 22 Oct 2020 16:41:55 +0530
Kirti Wankhede wrote:
> VM state change handler is called on change in VM's state. Based on
> VM state, VFIO device state should be changed.
> Added read/write helper functions for migration region.
> Added function to set device_state.
>
> Signed-off-by: Kirti Wankhede
> Reviewed-by: Neo Jia
> Reviewed-by: Dr. David Alan Gilbert
> ---
> hw/vfio/migration.c | 158
> ++
> hw/vfio/trace-events | 2 +
> include/hw/vfio/vfio-common.h | 4 ++
> 3 files changed, 164 insertions(+)
>
> diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c
> index 5f74a3ad1d72..34f39c7e2e28 100644
> --- a/hw/vfio/migration.c
> +++ b/hw/vfio/migration.c
> @@ -10,6 +10,7 @@
> #include "qemu/osdep.h"
> #include
>
> +#include "sysemu/runstate.h"
> #include "hw/vfio/vfio-common.h"
> #include "cpu.h"
> #include "migration/migration.h"
> @@ -22,6 +23,157 @@
> #include "exec/ram_addr.h"
> #include "pci.h"
> #include "trace.h"
> +#include "hw/hw.h"
> +
> +static inline int vfio_mig_access(VFIODevice *vbasedev, void *val, int count,
> + off_t off, bool iswrite)
> +{
> +int ret;
> +
> +ret = iswrite ? pwrite(vbasedev->fd, val, count, off) :
> +pread(vbasedev->fd, val, count, off);
> +if (ret < count) {
> +error_report("vfio_mig_%s %d byte %s: failed at offset 0x%lx, err:
> %s",
> + iswrite ? "write" : "read", count,
> + vbasedev->name, off, strerror(errno));
> +return (ret < 0) ? ret : -EINVAL;
> +}
> +return 0;
> +}
> +
> +static int vfio_mig_rw(VFIODevice *vbasedev, __u8 *buf, size_t count,
> + off_t off, bool iswrite)
> +{
> +int ret, done = 0;
> +__u8 *tbuf = buf;
> +
> +while (count) {
> +int bytes = 0;
> +
> +if (count >= 8 && !(off % 8)) {
> +bytes = 8;
> +} else if (count >= 4 && !(off % 4)) {
> +bytes = 4;
> +} else if (count >= 2 && !(off % 2)) {
> +bytes = 2;
> +} else {
> +bytes = 1;
> +}
> +
> +ret = vfio_mig_access(vbasedev, tbuf, bytes, off, iswrite);
> +if (ret) {
> +return ret;
> +}
> +
> +count -= bytes;
> +done += bytes;
> +off += bytes;
> +tbuf += bytes;
> +}
> +return done;
> +}
> +
> +#define vfio_mig_read(f, v, c, o) vfio_mig_rw(f, (__u8 *)v, c, o,
> false)
> +#define vfio_mig_write(f, v, c, o) vfio_mig_rw(f, (__u8 *)v, c, o, true)
> +
> +#define VFIO_MIG_STRUCT_OFFSET(f) \
> + offsetof(struct vfio_device_migration_info,
> f)
> +/*
> + * Change the device_state register for device @vbasedev. Bits set in @mask
> + * are preserved, bits set in @value are set, and bits not set in either
> @mask
> + * or @value are cleared in device_state. If the register cannot be accessed,
> + * the resulting state would be invalid, or the device enters an error state,
> + * an error is returned.
> + */
> +
> +static int vfio_migration_set_state(VFIODevice *vbasedev, uint32_t mask,
> +uint32_t value)
> +{
> +VFIOMigration *migration = vbasedev->migration;
> +VFIORegion *region = >region;
> +off_t dev_state_off = region->fd_offset +
> + VFIO_MIG_STRUCT_OFFSET(device_state);
> +uint32_t device_state;
> +int ret;
> +
> +ret = vfio_mig_read(vbasedev, _state, sizeof(device_state),
> +dev_state_off);
> +if (ret < 0) {
> +return ret;
> +}
> +
> +device_state = (device_state & mask) | value;
> +
> +if (!VFIO_DEVICE_STATE_VALID(device_state)) {
> +return -EINVAL;
> +}
> +
> +ret = vfio_mig_write(vbasedev, _state, sizeof(device_state),
> + dev_state_off);
> +if (ret < 0) {
> +int rret;
> +
> +rret = vfio_mig_read(vbasedev, _state, sizeof(device_state),
> + dev_state_off);
> +
> +if ((rret < 0) || (VFIO_DEVICE_STATE_IS_ERROR(device_state))) {
> +hw_error("%s: Device in error state 0x%x", vbasedev->name,
> + device_state);
> +return rret ? rret : -EIO;
> +}
> +return ret;
> +}
> +
> +migration->device_state = device_state;
> +trace_vfio_migration_set_state(vbasedev->name, device_state);
> +return 0;
> +}
> +
> +static void vfio_vmstate_change(void *opaque, int running, RunState state)
> +{
> +VFIODevice *vbasedev = opaque;
> +VFIOMigration *migration = vbasedev->migration;
> +uint32_t value, mask;
> +int ret;
> +
> +if ((vbasedev->migration->vm_running == running)) {
> +return;
> +}
> +
> +if (running) {
> +/*
> + * Here device state can have one of _SAVING, _RESUMING or _STOP bit.
> + *
[PATCH v27 05/17] vfio: Add VM state change handler to know state of VM
VM state change handler is called on change in VM's state. Based on
VM state, VFIO device state should be changed.
Added read/write helper functions for migration region.
Added function to set device_state.
Signed-off-by: Kirti Wankhede
Reviewed-by: Neo Jia
Reviewed-by: Dr. David Alan Gilbert
---
hw/vfio/migration.c | 158 ++
hw/vfio/trace-events | 2 +
include/hw/vfio/vfio-common.h | 4 ++
3 files changed, 164 insertions(+)
diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c
index 5f74a3ad1d72..34f39c7e2e28 100644
--- a/hw/vfio/migration.c
+++ b/hw/vfio/migration.c
@@ -10,6 +10,7 @@
#include "qemu/osdep.h"
#include
+#include "sysemu/runstate.h"
#include "hw/vfio/vfio-common.h"
#include "cpu.h"
#include "migration/migration.h"
@@ -22,6 +23,157 @@
#include "exec/ram_addr.h"
#include "pci.h"
#include "trace.h"
+#include "hw/hw.h"
+
+static inline int vfio_mig_access(VFIODevice *vbasedev, void *val, int count,
+ off_t off, bool iswrite)
+{
+int ret;
+
+ret = iswrite ? pwrite(vbasedev->fd, val, count, off) :
+pread(vbasedev->fd, val, count, off);
+if (ret < count) {
+error_report("vfio_mig_%s %d byte %s: failed at offset 0x%lx, err: %s",
+ iswrite ? "write" : "read", count,
+ vbasedev->name, off, strerror(errno));
+return (ret < 0) ? ret : -EINVAL;
+}
+return 0;
+}
+
+static int vfio_mig_rw(VFIODevice *vbasedev, __u8 *buf, size_t count,
+ off_t off, bool iswrite)
+{
+int ret, done = 0;
+__u8 *tbuf = buf;
+
+while (count) {
+int bytes = 0;
+
+if (count >= 8 && !(off % 8)) {
+bytes = 8;
+} else if (count >= 4 && !(off % 4)) {
+bytes = 4;
+} else if (count >= 2 && !(off % 2)) {
+bytes = 2;
+} else {
+bytes = 1;
+}
+
+ret = vfio_mig_access(vbasedev, tbuf, bytes, off, iswrite);
+if (ret) {
+return ret;
+}
+
+count -= bytes;
+done += bytes;
+off += bytes;
+tbuf += bytes;
+}
+return done;
+}
+
+#define vfio_mig_read(f, v, c, o) vfio_mig_rw(f, (__u8 *)v, c, o, false)
+#define vfio_mig_write(f, v, c, o) vfio_mig_rw(f, (__u8 *)v, c, o, true)
+
+#define VFIO_MIG_STRUCT_OFFSET(f) \
+ offsetof(struct vfio_device_migration_info, f)
+/*
+ * Change the device_state register for device @vbasedev. Bits set in @mask
+ * are preserved, bits set in @value are set, and bits not set in either @mask
+ * or @value are cleared in device_state. If the register cannot be accessed,
+ * the resulting state would be invalid, or the device enters an error state,
+ * an error is returned.
+ */
+
+static int vfio_migration_set_state(VFIODevice *vbasedev, uint32_t mask,
+uint32_t value)
+{
+VFIOMigration *migration = vbasedev->migration;
+VFIORegion *region = >region;
+off_t dev_state_off = region->fd_offset +
+ VFIO_MIG_STRUCT_OFFSET(device_state);
+uint32_t device_state;
+int ret;
+
+ret = vfio_mig_read(vbasedev, _state, sizeof(device_state),
+dev_state_off);
+if (ret < 0) {
+return ret;
+}
+
+device_state = (device_state & mask) | value;
+
+if (!VFIO_DEVICE_STATE_VALID(device_state)) {
+return -EINVAL;
+}
+
+ret = vfio_mig_write(vbasedev, _state, sizeof(device_state),
+ dev_state_off);
+if (ret < 0) {
+int rret;
+
+rret = vfio_mig_read(vbasedev, _state, sizeof(device_state),
+ dev_state_off);
+
+if ((rret < 0) || (VFIO_DEVICE_STATE_IS_ERROR(device_state))) {
+hw_error("%s: Device in error state 0x%x", vbasedev->name,
+ device_state);
+return rret ? rret : -EIO;
+}
+return ret;
+}
+
+migration->device_state = device_state;
+trace_vfio_migration_set_state(vbasedev->name, device_state);
+return 0;
+}
+
+static void vfio_vmstate_change(void *opaque, int running, RunState state)
+{
+VFIODevice *vbasedev = opaque;
+VFIOMigration *migration = vbasedev->migration;
+uint32_t value, mask;
+int ret;
+
+if ((vbasedev->migration->vm_running == running)) {
+return;
+}
+
+if (running) {
+/*
+ * Here device state can have one of _SAVING, _RESUMING or _STOP bit.
+ * Transition from _SAVING to _RUNNING can happen if there is migration
+ * failure, in that case clear _SAVING bit.
+ * Transition from _RESUMING to _RUNNING occurs during resuming
+ * phase, in that case clear _RESUMING bit.
+ * In both the above cases, set _RUNNING bit.
+ */
+mask = ~VFIO_DEVICE_STATE_MASK;
+value =
