Re: [PATCH v1 16/24] vfio-user: proxy container connect/disconnect
On 11/9/22 00:13, John Johnson wrote:
Signed-off-by: Elena Ufimtseva
Signed-off-by: John G Johnson
Signed-off-by: Jagannathan Raman
---
hw/vfio/common.c | 207 +-
hw/vfio/pci.c | 18 +++-
hw/vfio/user.c| 3 +
hw/vfio/user.h| 1 +
include/hw/vfio/vfio-common.h | 6 ++
5 files changed, 231 insertions(+), 4 deletions(-)
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index b540195..e73a772 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -19,6 +19,7 @@
*/
#include "qemu/osdep.h"
+#include CONFIG_DEVICES
#include
#ifdef CONFIG_KVM
#include
@@ -2267,6 +2268,208 @@ put_space_exit:
return ret;
}
+
+#ifdef CONFIG_VFIO_USER
this code belongs to hw/vfio/user.c. You will need to export the required
vfio subroutines for it.
+
+static int vfio_connect_proxy(VFIOProxy *proxy, VFIOGroup *group,
+ AddressSpace *as, Error **errp)
+{
+VFIOAddressSpace *space;
+VFIOContainer *container;
+int ret;
+
+/*
+ * try to mirror vfio_connect_container()
+ * as much as possible
+ */
Yes. This is very much like the VFIO_TYPE1_IOMMU case.
Could we wrap the vfio_get_iommu_info() routine in some way to bring it
even close ? to avoid all this code duplication.
+
+space = vfio_get_address_space(as);
+
+container = g_malloc0(sizeof(*container));
+container->space = space;
+container->fd = -1;
+container->error = NULL;
+container->io_ops = _cont_io_sock;
+QLIST_INIT(>giommu_list);
+QLIST_INIT(>hostwin_list);
+QLIST_INIT(>vrdl_list);
+container->proxy = proxy;
+
+/*
+ * The proxy uses a SW IOMMU in lieu of the HW one
+ * used in the ioctl() version. Mascarade as TYPE1
+ * for maximum capatibility
+ */
+container->iommu_type = VFIO_TYPE1_IOMMU;
+
+/*
+ * VFIO user allows the device server to map guest
+ * memory so it has the same issue with discards as
+ * a local IOMMU has.
+ */
+ret = vfio_ram_block_discard_disable(container, true);
+if (ret) {
+error_setg_errno(errp, -ret, "Cannot set discarding of RAM broken");
+goto free_container_exit;
+}
+
+vfio_host_win_add(container, 0, (hwaddr)-1, proxy->dma_pgsizes);
+container->pgsizes = proxy->dma_pgsizes;
+container->dma_max_mappings = proxy->max_dma;
+
+/* setup bitmask now, but migration support won't be ready until v2 */
+container->dirty_pages_supported = true;
+container->max_dirty_bitmap_size = proxy->max_bitmap;
+container->dirty_pgsizes = proxy->migr_pgsize;
+
+QLIST_INIT(>group_list);
+QLIST_INSERT_HEAD(>containers, container, next);
+
+group->container = container;
+QLIST_INSERT_HEAD(>group_list, group, container_next);
+
+container->listener = vfio_memory_listener;
+memory_listener_register(>listener, container->space->as);
+
+if (container->error) {
+ret = -1;
+error_propagate_prepend(errp, container->error,
+"memory listener initialization failed: ");
+goto listener_release_exit;
+}
+
+container->initialized = true;
+
+return 0;
+
+listener_release_exit:
+QLIST_REMOVE(group, container_next);
+QLIST_REMOVE(container, next);
+vfio_listener_release(container);
+vfio_ram_block_discard_disable(container, false);
+
+free_container_exit:
+g_free(container);
+
+vfio_put_address_space(space);
+
+return ret;
+}
+
+static void vfio_disconnect_proxy(VFIOGroup *group)
+{
+VFIOContainer *container = group->container;
+VFIOAddressSpace *space = container->space;
+VFIOGuestIOMMU *giommu, *tmp;
+VFIOHostDMAWindow *hostwin, *next;
+
+/*
+ * try to mirror vfio_disconnect_container()
same comment.
Given the code that follows, I wonder if we shouldn't introduce new
VFIOGroup models: Base, PCI, User, to isolate the differences and
reduce duplication.
Thanks,
C.
+ * as much as possible, knowing each device
+ * is in one group and one container
+ */
+
+QLIST_REMOVE(group, container_next);
+group->container = NULL;
+
+/*
+ * Explicitly release the listener first before unset container,
+ * since unset may destroy the backend container if it's the last
+ * group.
+ */
+memory_listener_unregister(>listener);
+
+QLIST_REMOVE(container, next);
+
+QLIST_FOREACH_SAFE(giommu, >giommu_list, giommu_next, tmp) {
+memory_region_unregister_iommu_notifier(
+MEMORY_REGION(giommu->iommu_mr), >n);
+QLIST_REMOVE(giommu, giommu_next);
+g_free(giommu);
+}
+
+QLIST_FOREACH_SAFE(hostwin, >hostwin_list, hostwin_next,
+ next) {
+QLIST_REMOVE(hostwin, hostwin_next);
+g_free(hostwin);
+}
+
+g_free(container);
+vfio_put_address_space(space);
+}
+
+int vfio_user_get_device(VFIOGroup *group,
Re: [PATCH v1 16/24] vfio-user: proxy container connect/disconnect
On Tue, Nov 08, 2022 at 03:13:38PM -0800, John Johnson wrote:
> +/*
> + * The proxy uses a SW IOMMU in lieu of the HW one
> + * used in the ioctl() version. Mascarade as TYPE1
> + * for maximum capatibility
> + */
capability
> @@ -3692,12 +3698,18 @@ static void vfio_user_instance_finalize(Object *obj)
> {
> VFIOPCIDevice *vdev = VFIO_PCI_BASE(obj);
> VFIODevice *vbasedev = >vbasedev;
> +VFIOGroup *group = vbasedev->group;
> +
> +vfio_bars_finalize(vdev);
> +g_free(vdev->emulated_config_bits);
> +g_free(vdev->rom);
These changes seem unrelated to this particular patch?
> diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
> index 793ca94..312ef9c 100644
> --- a/include/hw/vfio/vfio-common.h
> +++ b/include/hw/vfio/vfio-common.h
> @@ -94,6 +94,7 @@ typedef struct VFIOContainer {
> uint64_t max_dirty_bitmap_size;
> unsigned long pgsizes;
> unsigned int dma_max_mappings;
> +VFIOProxy *proxy;
> QLIST_HEAD(, VFIOGuestIOMMU) giommu_list;
> QLIST_HEAD(, VFIOHostDMAWindow) hostwin_list;
> QLIST_HEAD(, VFIOGroup) group_list;
> @@ -282,6 +283,11 @@ void vfio_put_group(VFIOGroup *group);
> int vfio_get_device(VFIOGroup *group, const char *name,
> VFIODevice *vbasedev, Error **errp);
>
> +int vfio_user_get_device(VFIOGroup *group, VFIODevice *vbasedev, Error
> **errp);
> +VFIOGroup *vfio_user_get_group(VFIOProxy *proxy, AddressSpace *as,
> + Error **errp);
> +void vfio_user_put_group(VFIOGroup *group);
> +
Why aren't these in user.h?
regards
john
[PATCH v1 16/24] vfio-user: proxy container connect/disconnect
Signed-off-by: Elena Ufimtseva
Signed-off-by: John G Johnson
Signed-off-by: Jagannathan Raman
---
hw/vfio/common.c | 207 +-
hw/vfio/pci.c | 18 +++-
hw/vfio/user.c| 3 +
hw/vfio/user.h| 1 +
include/hw/vfio/vfio-common.h | 6 ++
5 files changed, 231 insertions(+), 4 deletions(-)
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index b540195..e73a772 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -19,6 +19,7 @@
*/
#include "qemu/osdep.h"
+#include CONFIG_DEVICES
#include
#ifdef CONFIG_KVM
#include
@@ -2267,6 +2268,208 @@ put_space_exit:
return ret;
}
+
+#ifdef CONFIG_VFIO_USER
+
+static int vfio_connect_proxy(VFIOProxy *proxy, VFIOGroup *group,
+ AddressSpace *as, Error **errp)
+{
+VFIOAddressSpace *space;
+VFIOContainer *container;
+int ret;
+
+/*
+ * try to mirror vfio_connect_container()
+ * as much as possible
+ */
+
+space = vfio_get_address_space(as);
+
+container = g_malloc0(sizeof(*container));
+container->space = space;
+container->fd = -1;
+container->error = NULL;
+container->io_ops = _cont_io_sock;
+QLIST_INIT(>giommu_list);
+QLIST_INIT(>hostwin_list);
+QLIST_INIT(>vrdl_list);
+container->proxy = proxy;
+
+/*
+ * The proxy uses a SW IOMMU in lieu of the HW one
+ * used in the ioctl() version. Mascarade as TYPE1
+ * for maximum capatibility
+ */
+container->iommu_type = VFIO_TYPE1_IOMMU;
+
+/*
+ * VFIO user allows the device server to map guest
+ * memory so it has the same issue with discards as
+ * a local IOMMU has.
+ */
+ret = vfio_ram_block_discard_disable(container, true);
+if (ret) {
+error_setg_errno(errp, -ret, "Cannot set discarding of RAM broken");
+goto free_container_exit;
+}
+
+vfio_host_win_add(container, 0, (hwaddr)-1, proxy->dma_pgsizes);
+container->pgsizes = proxy->dma_pgsizes;
+container->dma_max_mappings = proxy->max_dma;
+
+/* setup bitmask now, but migration support won't be ready until v2 */
+container->dirty_pages_supported = true;
+container->max_dirty_bitmap_size = proxy->max_bitmap;
+container->dirty_pgsizes = proxy->migr_pgsize;
+
+QLIST_INIT(>group_list);
+QLIST_INSERT_HEAD(>containers, container, next);
+
+group->container = container;
+QLIST_INSERT_HEAD(>group_list, group, container_next);
+
+container->listener = vfio_memory_listener;
+memory_listener_register(>listener, container->space->as);
+
+if (container->error) {
+ret = -1;
+error_propagate_prepend(errp, container->error,
+"memory listener initialization failed: ");
+goto listener_release_exit;
+}
+
+container->initialized = true;
+
+return 0;
+
+listener_release_exit:
+QLIST_REMOVE(group, container_next);
+QLIST_REMOVE(container, next);
+vfio_listener_release(container);
+vfio_ram_block_discard_disable(container, false);
+
+free_container_exit:
+g_free(container);
+
+vfio_put_address_space(space);
+
+return ret;
+}
+
+static void vfio_disconnect_proxy(VFIOGroup *group)
+{
+VFIOContainer *container = group->container;
+VFIOAddressSpace *space = container->space;
+VFIOGuestIOMMU *giommu, *tmp;
+VFIOHostDMAWindow *hostwin, *next;
+
+/*
+ * try to mirror vfio_disconnect_container()
+ * as much as possible, knowing each device
+ * is in one group and one container
+ */
+
+QLIST_REMOVE(group, container_next);
+group->container = NULL;
+
+/*
+ * Explicitly release the listener first before unset container,
+ * since unset may destroy the backend container if it's the last
+ * group.
+ */
+memory_listener_unregister(>listener);
+
+QLIST_REMOVE(container, next);
+
+QLIST_FOREACH_SAFE(giommu, >giommu_list, giommu_next, tmp) {
+memory_region_unregister_iommu_notifier(
+MEMORY_REGION(giommu->iommu_mr), >n);
+QLIST_REMOVE(giommu, giommu_next);
+g_free(giommu);
+}
+
+QLIST_FOREACH_SAFE(hostwin, >hostwin_list, hostwin_next,
+ next) {
+QLIST_REMOVE(hostwin, hostwin_next);
+g_free(hostwin);
+}
+
+g_free(container);
+vfio_put_address_space(space);
+}
+
+int vfio_user_get_device(VFIOGroup *group, VFIODevice *vbasedev, Error **errp)
+{
+struct vfio_device_info info = { .argsz = sizeof(info) };
+int ret;
+
+ret = VDEV_GET_INFO(vbasedev, );
+if (ret) {
+error_setg_errno(errp, -ret, "get info failure");
+return ret;
+}
+
+vbasedev->fd = -1;
+vbasedev->group = group;
+QLIST_INSERT_HEAD(>device_list, vbasedev, next);
+
+vbasedev->num_irqs = info.num_irqs;
+vbasedev->num_regions = info.num_regions;
+vbasedev->flags = info.flags;
+
+vfio_get_all_regions(vbasedev);
+
