Hi ----- Original Message ----- > Add VHOST_USER_GET_CONFIG/VHOST_USER_SET_CONFIG messages which can be > used for live migration for vhost user devices, also vhost user devices > can benifit from the messages to get/set virtio config space from/to the
benefit > I/O target besides Qemu. For the purpose to support virtio config space > change, VHOST_USER_SET_CONFIG_FD message is added as the event notifier > in case virtio config space change. > > Signed-off-by: Changpeng Liu <changpeng....@intel.com> > --- > docs/interop/vhost-user.txt | 31 ++++++++++++++ > hw/virtio/vhost-user.c | 86 > +++++++++++++++++++++++++++++++++++++++ > hw/virtio/vhost.c | 63 ++++++++++++++++++++++++++++ > include/hw/virtio/vhost-backend.h | 8 ++++ > include/hw/virtio/vhost.h | 16 ++++++++ > 5 files changed, 204 insertions(+) > > diff --git a/docs/interop/vhost-user.txt b/docs/interop/vhost-user.txt > index 954771d..19dfc61 100644 > --- a/docs/interop/vhost-user.txt > +++ b/docs/interop/vhost-user.txt > @@ -116,6 +116,10 @@ Depending on the request type, payload can be: > - 3: IOTLB invalidate > - 4: IOTLB access fail > > + * Virtio device config space > + > + 256 Bytes static virito config space > + trailing spaces > In QEMU the vhost-user message is implemented with the following struct: > > typedef struct VhostUserMsg { > @@ -129,6 +133,7 @@ typedef struct VhostUserMsg { > VhostUserMemory memory; > VhostUserLog log; > struct vhost_iotlb_msg iotlb; > + uint8_t config[256]; > }; > } QEMU_PACKED VhostUserMsg; > > @@ -596,6 +601,32 @@ Master message types > and expect this message once (per VQ) during device configuration > (ie. before the master starts the VQ). > > + * VHOST_USER_GET_CONFIG > + Id: 24 > + Equivalent ioctl: N/A > + Master payload: virtio device config space > + > + Submitted by the vhost-user master to fetch the contents of the virtio > + config space. The vhost-user master may cache the contents to avoid > + repeated VHOST_USER_GET_CONFIG calls. > + > +* VHOST_USER_SET_CONFIG > + Id: 25 > + Equivalent ioctl: N/A > + Master payload: virtio device config space > + > + Submitted by the vhost-user master when the guest writes to virtio > + config space and also after live migration on the destination host. > + > +* VHOST_USER_SET_CONFIG_FD > + Id: 26 > + Equivalent ioctl: N/A > + Master payload: N/A > + > + Sets the notifier file descriptor, which is passed as ancillary data. > + Vhost-user master uses the file descriptor as event callback when the > + virtio config space changed. So this is a fd for the slave to notify of config change? Shouldn't we use the "slave communication" instead? > + > Slave message types > ------------------- > > diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c > index 093675e..4b402c5 100644 > --- a/hw/virtio/vhost-user.c > +++ b/hw/virtio/vhost-user.c > @@ -26,6 +26,11 @@ > #define VHOST_MEMORY_MAX_NREGIONS 8 > #define VHOST_USER_F_PROTOCOL_FEATURES 30 > > +/* > + * Maximum size of virtio device config space > + */ > +#define VHOST_USER_MAX_CONFIG_SIZE 256 > + > enum VhostUserProtocolFeature { > VHOST_USER_PROTOCOL_F_MQ = 0, > VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1, > @@ -65,6 +70,9 @@ typedef enum VhostUserRequest { > VHOST_USER_SET_SLAVE_REQ_FD = 21, > VHOST_USER_IOTLB_MSG = 22, > VHOST_USER_SET_VRING_ENDIAN = 23, > + VHOST_USER_GET_CONFIG = 24, > + VHOST_USER_SET_CONFIG = 25, > + VHOST_USER_SET_CONFIG_FD = 26, > VHOST_USER_MAX > } VhostUserRequest; > > @@ -109,6 +117,7 @@ typedef struct VhostUserMsg { > VhostUserMemory memory; > VhostUserLog log; > struct vhost_iotlb_msg iotlb; > + uint8_t config[VHOST_USER_MAX_CONFIG_SIZE]; > } payload; > } QEMU_PACKED VhostUserMsg; > > @@ -922,6 +931,80 @@ static void vhost_user_set_iotlb_callback(struct > vhost_dev *dev, int enabled) > /* No-op as the receive channel is not dedicated to IOTLB messages. */ > } > > +static int vhost_user_get_config(struct vhost_dev *dev, uint8_t *config, > + size_t config_len) > +{ > + VhostUserMsg msg = { > + .request = VHOST_USER_GET_CONFIG, > + .flags = VHOST_USER_VERSION, > + .size = config_len, > + }; > + > + if (config_len == 0 || config_len > VHOST_USER_PAYLOAD_SIZE) { > + error_report("bad config length"); > + return -1; > + } > + > + if (vhost_user_write(dev, &msg, NULL, 0) < 0) { > + return -1; > + } > + > + if (vhost_user_read(dev, &msg) < 0) { > + return -1; > + } > + > + if (msg.request != VHOST_USER_GET_CONFIG) { > + error_report("Received unexpected msg type. Expected %d received > %d", > + VHOST_USER_GET_CONFIG, msg.request); > + return -1; > + } > + > + if (msg.size != config_len) { > + error_report("Received bad msg size."); > + return -1; > + } > + > + memcpy(config, &msg.payload.config, config_len); > + > + return 0; > +} > + > +static int vhost_user_set_config(struct vhost_dev *dev, const uint8_t > *config, > + size_t config_len) > +{ > + VhostUserMsg msg = { > + .request = VHOST_USER_SET_CONFIG, > + .flags = VHOST_USER_VERSION, > + .size = config_len, > + }; > + > + if (config_len == 0 || config_len > VHOST_USER_PAYLOAD_SIZE) { > + error_report("bad config length"); > + return -1; > + } > + > + memcpy(&msg.payload.config, config, config_len); > + if (vhost_user_write(dev, &msg, NULL, 0) < 0) { > + return -1; > + } > + > + return 0; > +} > + > +static int vhost_user_set_config_fd(struct vhost_dev *dev, int fd) > +{ > + VhostUserMsg msg = { > + .request = VHOST_USER_SET_CONFIG_FD, > + .flags = VHOST_USER_VERSION, > + }; > + > + if (vhost_user_write(dev, &msg, &fd, 1) < 0) { > + return -1; > + } > + > + return 0; > +} > + > const VhostOps user_ops = { > .backend_type = VHOST_BACKEND_TYPE_USER, > .vhost_backend_init = vhost_user_init, > @@ -948,4 +1031,7 @@ const VhostOps user_ops = { > .vhost_net_set_mtu = vhost_user_net_set_mtu, > .vhost_set_iotlb_callback = vhost_user_set_iotlb_callback, > .vhost_send_device_iotlb_msg = vhost_user_send_device_iotlb_msg, > + .vhost_get_config = vhost_user_get_config, > + .vhost_set_config = vhost_user_set_config, > + .vhost_set_config_fd = vhost_user_set_config_fd, > }; > diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c > index 6eddb09..0d39a55 100644 > --- a/hw/virtio/vhost.c > +++ b/hw/virtio/vhost.c > @@ -1353,6 +1353,9 @@ void vhost_dev_cleanup(struct vhost_dev *hdev) > for (i = 0; i < hdev->nvqs; ++i) { > vhost_virtqueue_cleanup(hdev->vqs + i); > } > + if (hdev->config_ops) { > + event_notifier_cleanup(&hdev->config_notifier); > + } > if (hdev->mem) { > /* those are only safe after successful init */ > memory_listener_unregister(&hdev->memory_listener); > @@ -1496,6 +1499,66 @@ void vhost_ack_features(struct vhost_dev *hdev, const > int *feature_bits, > } > } > > +int vhost_dev_get_config(struct vhost_dev *hdev, uint8_t *config, > + size_t config_len) > +{ > + assert(hdev->vhost_ops); > + > + if (hdev->vhost_ops->vhost_get_config) { > + return hdev->vhost_ops->vhost_get_config(hdev, config, config_len); > + } > + > + return 0; > +} > + > +int vhost_dev_set_config(struct vhost_dev *hdev, const uint8_t *config, > + size_t config_len) > +{ > + assert(hdev->vhost_ops); > + > + if (hdev->vhost_ops->vhost_set_config) { > + return hdev->vhost_ops->vhost_set_config(hdev, config, config_len); > + } > + > + return 0; > +} > + > +static void vhost_dev_config_notifier_read(EventNotifier *n) > +{ > + struct vhost_dev *hdev = container_of(n, struct vhost_dev, > + config_notifier); > + > + if (event_notifier_test_and_clear(n)) { > + if (hdev->config_ops) { > + hdev->config_ops->vhost_dev_config_notifier(hdev); > + } > + } > +} > + > +int vhost_dev_set_config_notifier(struct vhost_dev *hdev, > + const VhostDevConfigOps *ops) > +{ > + int r, fd; > + > + assert(hdev->vhost_ops); > + > + r = event_notifier_init(&hdev->config_notifier, 0); > + if (r < 0) { > + return r; > + } > + > + hdev->config_ops = ops; > + event_notifier_set_handler(&hdev->config_notifier, > + vhost_dev_config_notifier_read); > + > + if (hdev->vhost_ops->vhost_set_config_fd) { > + fd = event_notifier_get_fd(&hdev->config_notifier); > + return hdev->vhost_ops->vhost_set_config_fd(hdev, fd); > + } > + > + return 0; > +} > + > /* Host notifiers must be enabled at this point. */ > int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev) > { > diff --git a/include/hw/virtio/vhost-backend.h > b/include/hw/virtio/vhost-backend.h > index a7a5f22..df6769e 100644 > --- a/include/hw/virtio/vhost-backend.h > +++ b/include/hw/virtio/vhost-backend.h > @@ -84,6 +84,11 @@ typedef void (*vhost_set_iotlb_callback_op)(struct > vhost_dev *dev, > int enabled); > typedef int (*vhost_send_device_iotlb_msg_op)(struct vhost_dev *dev, > struct vhost_iotlb_msg *imsg); > +typedef int (*vhost_set_config_op)(struct vhost_dev *dev, const uint8_t > *config, > + size_t config_len); > +typedef int (*vhost_get_config_op)(struct vhost_dev *dev, uint8_t *config, > + size_t config_len); > +typedef int (*vhost_set_config_fd_op)(struct vhost_dev *dev, int fd); > > typedef struct VhostOps { > VhostBackendType backend_type; > @@ -118,6 +123,9 @@ typedef struct VhostOps { > vhost_vsock_set_running_op vhost_vsock_set_running; > vhost_set_iotlb_callback_op vhost_set_iotlb_callback; > vhost_send_device_iotlb_msg_op vhost_send_device_iotlb_msg; > + vhost_get_config_op vhost_get_config; > + vhost_set_config_op vhost_set_config; > + vhost_set_config_fd_op vhost_set_config_fd; > } VhostOps; > > extern const VhostOps user_ops; > diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h > index 467dc77..ff172f2 100644 > --- a/include/hw/virtio/vhost.h > +++ b/include/hw/virtio/vhost.h > @@ -46,6 +46,12 @@ struct vhost_iommu { > QLIST_ENTRY(vhost_iommu) iommu_next; > }; > > +typedef struct VhostDevConfigOps { > + /* Vhost device config space changed callback > + */ > + void (*vhost_dev_config_notifier)(struct vhost_dev *dev); > +} VhostDevConfigOps; > + > struct vhost_memory; > struct vhost_dev { > VirtIODevice *vdev; > @@ -76,6 +82,8 @@ struct vhost_dev { > QLIST_ENTRY(vhost_dev) entry; > QLIST_HEAD(, vhost_iommu) iommu_list; > IOMMUNotifier n; > + EventNotifier config_notifier; > + const VhostDevConfigOps *config_ops; > }; > > int vhost_dev_init(struct vhost_dev *hdev, void *opaque, > @@ -106,4 +114,12 @@ int vhost_net_set_backend(struct vhost_dev *hdev, > struct vhost_vring_file *file); > > int vhost_device_iotlb_miss(struct vhost_dev *dev, uint64_t iova, int > write); > +int vhost_dev_get_config(struct vhost_dev *dev, uint8_t *config, > + size_t config_len); > +int vhost_dev_set_config(struct vhost_dev *dev, const uint8_t *config, > + size_t config_len); > +/* notifier callback in case vhost device config space changed > + */ > +int vhost_dev_set_config_notifier(struct vhost_dev *dev, > + const VhostDevConfigOps *ops); > #endif > -- > 1.9.3 Looks good otherwise