Re: [PATCH v27 09/17] vfio: Add load state functions to SaveVMHandlers
On 10/23/2020 1:20 AM, Alex Williamson wrote: On Thu, 22 Oct 2020 16:41:59 +0530 Kirti Wankhede wrote: Sequence during _RESUMING device state: While data for this device is available, repeat below steps: a. read data_offset from where user application should write data. b. write data of data_size to migration region from data_offset. c. write data_size which indicates vendor driver that data is written in staging buffer. For user, data is opaque. User should write data in the same order as received. Signed-off-by: Kirti Wankhede Reviewed-by: Neo Jia Reviewed-by: Dr. David Alan Gilbert --- hw/vfio/migration.c | 192 +++ hw/vfio/trace-events | 3 + 2 files changed, 195 insertions(+) diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c index 5506cef15d88..46d05d230e2a 100644 --- a/hw/vfio/migration.c +++ b/hw/vfio/migration.c @@ -257,6 +257,77 @@ static int vfio_save_buffer(QEMUFile *f, VFIODevice *vbasedev, uint64_t *size) return ret; } +static int vfio_load_buffer(QEMUFile *f, VFIODevice *vbasedev, +uint64_t data_size) +{ +VFIORegion *region = &vbasedev->migration->region; +uint64_t data_offset = 0, size, report_size; +int ret; + +do { +ret = vfio_mig_read(vbasedev, &data_offset, sizeof(data_offset), + region->fd_offset + VFIO_MIG_STRUCT_OFFSET(data_offset)); +if (ret < 0) { +return ret; +} + +if (data_offset + data_size > region->size) { +/* + * If data_size is greater than the data section of migration region + * then iterate the write buffer operation. This case can occur if + * size of migration region at destination is smaller than size of + * migration region at source. + */ +report_size = size = region->size - data_offset; +data_size -= size; +} else { +report_size = size = data_size; +data_size = 0; +} + +trace_vfio_load_state_device_data(vbasedev->name, data_offset, size); + +while (size) { +void *buf; +uint64_t sec_size; +bool buf_alloc = false; + +buf = get_data_section_size(region, data_offset, size, &sec_size); + +if (!buf) { +buf = g_try_malloc(sec_size); +if (!buf) { +error_report("%s: Error allocating buffer ", __func__); +return -ENOMEM; +} +buf_alloc = true; +} + +qemu_get_buffer(f, buf, sec_size); + +if (buf_alloc) { +ret = vfio_mig_write(vbasedev, buf, sec_size, +region->fd_offset + data_offset); +g_free(buf); + +if (ret < 0) { +return ret; +} +} +size -= sec_size; +data_offset += sec_size; +} + +ret = vfio_mig_write(vbasedev, &report_size, sizeof(report_size), +region->fd_offset + VFIO_MIG_STRUCT_OFFSET(data_size)); +if (ret < 0) { +return ret; +} +} while (data_size); + +return 0; +} + static int vfio_update_pending(VFIODevice *vbasedev) { VFIOMigration *migration = vbasedev->migration; @@ -293,6 +364,33 @@ static int vfio_save_device_config_state(QEMUFile *f, void *opaque) return qemu_file_get_error(f); } +static int vfio_load_device_config_state(QEMUFile *f, void *opaque) +{ +VFIODevice *vbasedev = opaque; +uint64_t data; + +if (vbasedev->ops && vbasedev->ops->vfio_load_config) { +int ret; + +ret = vbasedev->ops->vfio_load_config(vbasedev, f); +if (ret) { +error_report("%s: Failed to load device config space", + vbasedev->name); +return ret; +} +} + +data = qemu_get_be64(f); +if (data != VFIO_MIG_FLAG_END_OF_STATE) { +error_report("%s: Failed loading device config space, " + "end flag incorrect 0x%"PRIx64, vbasedev->name, data); +return -EINVAL; +} + +trace_vfio_load_device_config_state(vbasedev->name); +return qemu_file_get_error(f); +} + /* -- */ static int vfio_save_setup(QEMUFile *f, void *opaque) @@ -477,12 +575,106 @@ static int vfio_save_complete_precopy(QEMUFile *f, void *opaque) return ret; } +static int vfio_load_setup(QEMUFile *f, void *opaque) +{ +VFIODevice *vbasedev = opaque; +VFIOMigration *migration = vbasedev->migration; +int ret = 0; + +if (migration->region.mmaps) { +ret = vfio_region_mmap(&migration->region); Checking, are we in the right thread context not to require locking the iothread as we did in vfio_save_setup()?
Re: [PATCH v27 09/17] vfio: Add load state functions to SaveVMHandlers
On Thu, 22 Oct 2020 16:41:59 +0530 Kirti Wankhede wrote: > Sequence during _RESUMING device state: > While data for this device is available, repeat below steps: > a. read data_offset from where user application should write data. > b. write data of data_size to migration region from data_offset. > c. write data_size which indicates vendor driver that data is written in >staging buffer. > > For user, data is opaque. User should write data in the same order as > received. > > Signed-off-by: Kirti Wankhede > Reviewed-by: Neo Jia > Reviewed-by: Dr. David Alan Gilbert > --- > hw/vfio/migration.c | 192 > +++ > hw/vfio/trace-events | 3 + > 2 files changed, 195 insertions(+) > > diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c > index 5506cef15d88..46d05d230e2a 100644 > --- a/hw/vfio/migration.c > +++ b/hw/vfio/migration.c > @@ -257,6 +257,77 @@ static int vfio_save_buffer(QEMUFile *f, VFIODevice > *vbasedev, uint64_t *size) > return ret; > } > > +static int vfio_load_buffer(QEMUFile *f, VFIODevice *vbasedev, > +uint64_t data_size) > +{ > +VFIORegion *region = &vbasedev->migration->region; > +uint64_t data_offset = 0, size, report_size; > +int ret; > + > +do { > +ret = vfio_mig_read(vbasedev, &data_offset, sizeof(data_offset), > + region->fd_offset + > VFIO_MIG_STRUCT_OFFSET(data_offset)); > +if (ret < 0) { > +return ret; > +} > + > +if (data_offset + data_size > region->size) { > +/* > + * If data_size is greater than the data section of migration > region > + * then iterate the write buffer operation. This case can occur > if > + * size of migration region at destination is smaller than size > of > + * migration region at source. > + */ > +report_size = size = region->size - data_offset; > +data_size -= size; > +} else { > +report_size = size = data_size; > +data_size = 0; > +} > + > +trace_vfio_load_state_device_data(vbasedev->name, data_offset, size); > + > +while (size) { > +void *buf; > +uint64_t sec_size; > +bool buf_alloc = false; > + > +buf = get_data_section_size(region, data_offset, size, > &sec_size); > + > +if (!buf) { > +buf = g_try_malloc(sec_size); > +if (!buf) { > +error_report("%s: Error allocating buffer ", __func__); > +return -ENOMEM; > +} > +buf_alloc = true; > +} > + > +qemu_get_buffer(f, buf, sec_size); > + > +if (buf_alloc) { > +ret = vfio_mig_write(vbasedev, buf, sec_size, > +region->fd_offset + data_offset); > +g_free(buf); > + > +if (ret < 0) { > +return ret; > +} > +} > +size -= sec_size; > +data_offset += sec_size; > +} > + > +ret = vfio_mig_write(vbasedev, &report_size, sizeof(report_size), > +region->fd_offset + > VFIO_MIG_STRUCT_OFFSET(data_size)); > +if (ret < 0) { > +return ret; > +} > +} while (data_size); > + > +return 0; > +} > + > static int vfio_update_pending(VFIODevice *vbasedev) > { > VFIOMigration *migration = vbasedev->migration; > @@ -293,6 +364,33 @@ static int vfio_save_device_config_state(QEMUFile *f, > void *opaque) > return qemu_file_get_error(f); > } > > +static int vfio_load_device_config_state(QEMUFile *f, void *opaque) > +{ > +VFIODevice *vbasedev = opaque; > +uint64_t data; > + > +if (vbasedev->ops && vbasedev->ops->vfio_load_config) { > +int ret; > + > +ret = vbasedev->ops->vfio_load_config(vbasedev, f); > +if (ret) { > +error_report("%s: Failed to load device config space", > + vbasedev->name); > +return ret; > +} > +} > + > +data = qemu_get_be64(f); > +if (data != VFIO_MIG_FLAG_END_OF_STATE) { > +error_report("%s: Failed loading device config space, " > + "end flag incorrect 0x%"PRIx64, vbasedev->name, data); > +return -EINVAL; > +} > + > +trace_vfio_load_device_config_state(vbasedev->name); > +return qemu_file_get_error(f); > +} > + > /* -- */ > > static int vfio_save_setup(QEMUFile *f, void *opaque) > @@ -477,12 +575,106 @@ static int vfio_save_complete_precopy(QEMUFile *f, > void *opaque) > return ret; > } > > +static int vfio_load_setup(QEMUFile *f, void *opaque) > +{ > +VFIODevice *vbasedev = opaque; > +VFIOMigration *migration = vbasedev->mig
[PATCH v27 09/17] vfio: Add load state functions to SaveVMHandlers
Sequence during _RESUMING device state: While data for this device is available, repeat below steps: a. read data_offset from where user application should write data. b. write data of data_size to migration region from data_offset. c. write data_size which indicates vendor driver that data is written in staging buffer. For user, data is opaque. User should write data in the same order as received. Signed-off-by: Kirti Wankhede Reviewed-by: Neo Jia Reviewed-by: Dr. David Alan Gilbert --- hw/vfio/migration.c | 192 +++ hw/vfio/trace-events | 3 + 2 files changed, 195 insertions(+) diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c index 5506cef15d88..46d05d230e2a 100644 --- a/hw/vfio/migration.c +++ b/hw/vfio/migration.c @@ -257,6 +257,77 @@ static int vfio_save_buffer(QEMUFile *f, VFIODevice *vbasedev, uint64_t *size) return ret; } +static int vfio_load_buffer(QEMUFile *f, VFIODevice *vbasedev, +uint64_t data_size) +{ +VFIORegion *region = &vbasedev->migration->region; +uint64_t data_offset = 0, size, report_size; +int ret; + +do { +ret = vfio_mig_read(vbasedev, &data_offset, sizeof(data_offset), + region->fd_offset + VFIO_MIG_STRUCT_OFFSET(data_offset)); +if (ret < 0) { +return ret; +} + +if (data_offset + data_size > region->size) { +/* + * If data_size is greater than the data section of migration region + * then iterate the write buffer operation. This case can occur if + * size of migration region at destination is smaller than size of + * migration region at source. + */ +report_size = size = region->size - data_offset; +data_size -= size; +} else { +report_size = size = data_size; +data_size = 0; +} + +trace_vfio_load_state_device_data(vbasedev->name, data_offset, size); + +while (size) { +void *buf; +uint64_t sec_size; +bool buf_alloc = false; + +buf = get_data_section_size(region, data_offset, size, &sec_size); + +if (!buf) { +buf = g_try_malloc(sec_size); +if (!buf) { +error_report("%s: Error allocating buffer ", __func__); +return -ENOMEM; +} +buf_alloc = true; +} + +qemu_get_buffer(f, buf, sec_size); + +if (buf_alloc) { +ret = vfio_mig_write(vbasedev, buf, sec_size, +region->fd_offset + data_offset); +g_free(buf); + +if (ret < 0) { +return ret; +} +} +size -= sec_size; +data_offset += sec_size; +} + +ret = vfio_mig_write(vbasedev, &report_size, sizeof(report_size), +region->fd_offset + VFIO_MIG_STRUCT_OFFSET(data_size)); +if (ret < 0) { +return ret; +} +} while (data_size); + +return 0; +} + static int vfio_update_pending(VFIODevice *vbasedev) { VFIOMigration *migration = vbasedev->migration; @@ -293,6 +364,33 @@ static int vfio_save_device_config_state(QEMUFile *f, void *opaque) return qemu_file_get_error(f); } +static int vfio_load_device_config_state(QEMUFile *f, void *opaque) +{ +VFIODevice *vbasedev = opaque; +uint64_t data; + +if (vbasedev->ops && vbasedev->ops->vfio_load_config) { +int ret; + +ret = vbasedev->ops->vfio_load_config(vbasedev, f); +if (ret) { +error_report("%s: Failed to load device config space", + vbasedev->name); +return ret; +} +} + +data = qemu_get_be64(f); +if (data != VFIO_MIG_FLAG_END_OF_STATE) { +error_report("%s: Failed loading device config space, " + "end flag incorrect 0x%"PRIx64, vbasedev->name, data); +return -EINVAL; +} + +trace_vfio_load_device_config_state(vbasedev->name); +return qemu_file_get_error(f); +} + /* -- */ static int vfio_save_setup(QEMUFile *f, void *opaque) @@ -477,12 +575,106 @@ static int vfio_save_complete_precopy(QEMUFile *f, void *opaque) return ret; } +static int vfio_load_setup(QEMUFile *f, void *opaque) +{ +VFIODevice *vbasedev = opaque; +VFIOMigration *migration = vbasedev->migration; +int ret = 0; + +if (migration->region.mmaps) { +ret = vfio_region_mmap(&migration->region); +if (ret) { +error_report("%s: Failed to mmap VFIO migration region %d: %s", + vbasedev->name, migration->region.nr, + strerror(-ret)); +error_report("%s: Falling back to s