Move VFIO setup for ETH, CRYPTO and QDMA devices from fslmc_vfio.c to fslmc_bus.c probe. Also move interrupt setup for ETH devices.
Add fslmc_vfio_dev_setup() (resp. fslmc_vfio_dev_close()) wrapper that handles VFIO device fd acquisition and interrupt setup (resp. device removal), hiding VFIO internals from the bus layer. Note: before this change, CRYPTO and QDMA devices were getting a VFIO device FD even though nothing seems to be done with it. This is kept as is, the device FD is closed on call to fslmc_vfio_dev_close(). Signed-off-by: David Marchand <[email protected]> --- drivers/bus/fslmc/fslmc_bus.c | 13 ++++++ drivers/bus/fslmc/fslmc_vfio.c | 83 ++++++++++++++++++++++++++-------- drivers/bus/fslmc/fslmc_vfio.h | 5 ++ 3 files changed, 83 insertions(+), 18 deletions(-) diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c index 54a74dc889..7863bb233e 100644 --- a/drivers/bus/fslmc/fslmc_bus.c +++ b/drivers/bus/fslmc/fslmc_bus.c @@ -573,6 +573,7 @@ rte_fslmc_scan(void) static int rte_fslmc_probe(struct rte_bus *bus) { + struct rte_dpaa2_device *dev; static const struct rte_mbuf_dynfield dpaa2_seqn_dynfield_desc = { .name = DPAA2_SEQN_DYNFIELD_NAME, .size = sizeof(dpaa2_seqn_t), @@ -620,8 +621,19 @@ rte_fslmc_probe(struct rte_bus *bus) goto vfio_dma_unmap; } + RTE_BUS_FOREACH_DEV(dev, &rte_fslmc_bus) { + ret = fslmc_vfio_dev_setup(dev); + if (ret) { + DPAA2_BUS_ERR("Dev (%s) VFIO setup failed", dev->device.name); + goto vfio_dev_close; + } + } + return rte_bus_generic_probe(bus); +vfio_dev_close: + RTE_BUS_FOREACH_DEV(dev, &rte_fslmc_bus) + fslmc_vfio_dev_close(dev); vfio_dma_unmap: fslmc_vfio_dmaunmap(); vfio_close_group: @@ -653,6 +665,7 @@ rte_fslmc_close(struct rte_bus *bus) RTE_BUS_FOREACH_DEV(dev, bus) { if (rte_dev_is_probed(&dev->device) && fslmc_bus_unplug_device(&dev->device)) DPAA2_BUS_ERR("Unable to remove %s", dev->device.name); + fslmc_vfio_dev_close(dev); } ret = fslmc_vfio_close_group(); diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c index b09e08854a..fdd2bffb6c 100644 --- a/drivers/bus/fslmc/fslmc_vfio.c +++ b/drivers/bus/fslmc/fslmc_vfio.c @@ -1455,6 +1455,15 @@ rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle, return -EIO; } +static void +dpaa2_close_intr(struct rte_intr_handle *intr_handle) +{ + if (rte_intr_fd_get(intr_handle) >= 0) { + close(rte_intr_fd_get(intr_handle)); + rte_intr_fd_set(intr_handle, -1); + } +} + static void fslmc_close_iodevices(struct rte_dpaa2_device *dev, int vfio_fd) @@ -1489,8 +1498,7 @@ fslmc_close_iodevices(struct rte_dpaa2_device *dev, } /* - * fslmc_process_iodevices for processing only IO (ETH, CRYPTO, and possibly - * EVENT) devices. + * fslmc_process_iodevices for processing only IO devices. */ static int fslmc_process_iodevices(struct rte_dpaa2_device *dev) @@ -1505,12 +1513,6 @@ fslmc_process_iodevices(struct rte_dpaa2_device *dev) return ret; switch (dev->dev_type) { - case DPAA2_ETH: - ret = rte_dpaa2_vfio_setup_intr(dev->intr_handle, dev_fd, - device_info.num_irqs); - if (ret) - return ret; - break; case DPAA2_CON: case DPAA2_IO: case DPAA2_CI: @@ -1534,6 +1536,61 @@ fslmc_process_iodevices(struct rte_dpaa2_device *dev) return 0; } +int +fslmc_vfio_dev_setup(struct rte_dpaa2_device *dev) +{ + struct vfio_device_info device_info = { .argsz = sizeof(device_info) }; + int dev_fd; + int ret; + + ret = fslmc_vfio_setup_device(dev->device.name, &dev_fd, &device_info); + if (ret) { + DPAA2_BUS_ERR("VFIO setup failed for %s: %d", + dev->device.name, ret); + return ret; + } + + if (dev->dev_type == DPAA2_ETH) { + ret = rte_dpaa2_vfio_setup_intr(dev->intr_handle, dev_fd, + device_info.num_irqs); + if (ret) { + DPAA2_BUS_ERR("Interrupt setup failed for %s: %d", + dev->device.name, ret); + fslmc_vfio_dev_close(dev); + return ret; + } + } + + DPAA2_BUS_DEBUG("Device (%s) VFIO setup completed", dev->device.name); + return 0; +} + +int +fslmc_vfio_dev_close(struct rte_dpaa2_device *dev) +{ + int vfio_group_fd; + int ret; + const char *group_name = fslmc_vfio_get_group_name(); + + vfio_group_fd = fslmc_vfio_group_fd_by_name(group_name); + if (vfio_group_fd <= 0) { + DPAA2_BUS_ERR("Get fd by name(%s) failed(%d)", + group_name, vfio_group_fd); + if (vfio_group_fd < 0) + return vfio_group_fd; + return -EIO; + } + + dpaa2_close_intr(dev->intr_handle); + + ret = fslmc_vfio_group_remove_dev(vfio_group_fd, dev->device.name); + if (ret) + DPAA2_BUS_ERR("Failed to remove %s from vfio", dev->device.name); + + DPAA2_BUS_DEBUG("Device (%s) closed", dev->device.name); + return ret; +} + static int fslmc_process_mcp(struct rte_dpaa2_device *dev) { @@ -1693,16 +1750,6 @@ fslmc_vfio_process_group(void) } } - /* Process regular devices */ - RTE_BUS_FOREACH_DEV(dev, &rte_fslmc_bus) { - ret = fslmc_process_iodevices(dev); - if (ret) { - DPAA2_BUS_DEBUG("Dev (%s) init failed", - dev->device.name); - return ret; - } - } - return 0; } diff --git a/drivers/bus/fslmc/fslmc_vfio.h b/drivers/bus/fslmc/fslmc_vfio.h index 57fe7038de..0973ddbe37 100644 --- a/drivers/bus/fslmc/fslmc_vfio.h +++ b/drivers/bus/fslmc/fslmc_vfio.h @@ -61,4 +61,9 @@ char *fslmc_get_container(void); int fslmc_get_container_group(const char *group_name, int *gropuid); int fslmc_vfio_dmamap(void); int fslmc_vfio_dmaunmap(void); + +struct rte_dpaa2_device; +int fslmc_vfio_dev_setup(struct rte_dpaa2_device *dev); +int fslmc_vfio_dev_close(struct rte_dpaa2_device *dev); + #endif /* _FSLMC_VFIO_H_ */ -- 2.54.0
