Move VFIO setup for ETH, CRYPTO and QDMA devices from fslmc_vfio.c to fslmc_bus.c scan layer. Also move interrupt setup for ETH devices.
Add fslmc_vfio_dev_setup() (resp. fslmc_vfio_dev_close()) wrapper that handles VFIO device fd acquisition and interrupt setup (resp. device removal), hiding VFIO internals from the bus layer. Note: before this change, CRYPTO and QDMA devices were getting a VFIO device FD even though nothing seems to be done with it. This is kept as is, the device FD is closed on call to fslmc_vfio_dev_close(). Signed-off-by: David Marchand <[email protected]> --- Changes since RFC v1: - added cleanup on failure, --- drivers/bus/fslmc/fslmc_bus.c | 12 +++++ drivers/bus/fslmc/fslmc_vfio.c | 83 ++++++++++++++++++++++++++-------- drivers/bus/fslmc/fslmc_vfio.h | 5 ++ 3 files changed, 82 insertions(+), 18 deletions(-) diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c index 3b22d3367d..4f4abd19ce 100644 --- a/drivers/bus/fslmc/fslmc_bus.c +++ b/drivers/bus/fslmc/fslmc_bus.c @@ -579,12 +579,23 @@ rte_fslmc_scan(void) DPAA2_BUS_ERR("Unable to setup devices %d", ret); goto vfio_dma_unmap; } + + RTE_BUS_FOREACH_DEV(dev, &rte_fslmc_bus) { + ret = fslmc_vfio_dev_setup(dev); + if (ret) { + DPAA2_BUS_ERR("Dev (%s) VFIO setup failed", dev->device.name); + goto vfio_dev_close; + } + } } process_once = 1; return 0; +vfio_dev_close: + RTE_BUS_FOREACH_DEV(dev, &rte_fslmc_bus) + fslmc_vfio_dev_close(dev); vfio_dma_unmap: fslmc_vfio_dmaunmap(); vfio_close_group: @@ -631,6 +642,7 @@ rte_fslmc_close(struct rte_bus *bus) RTE_BUS_FOREACH_DEV(dev, bus) { if (rte_dev_is_probed(&dev->device) && fslmc_bus_unplug_device(&dev->device)) DPAA2_BUS_ERR("Unable to remove %s", dev->device.name); + fslmc_vfio_dev_close(dev); } ret = fslmc_vfio_close_group(); diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c index 857f22f204..04582e38c6 100644 --- a/drivers/bus/fslmc/fslmc_vfio.c +++ b/drivers/bus/fslmc/fslmc_vfio.c @@ -1444,6 +1444,15 @@ rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle, return -EIO; } +static void +dpaa2_close_intr(struct rte_intr_handle *intr_handle) +{ + if (rte_intr_fd_get(intr_handle) >= 0) { + close(rte_intr_fd_get(intr_handle)); + rte_intr_fd_set(intr_handle, -1); + } +} + static void fslmc_close_iodevices(struct rte_dpaa2_device *dev, int vfio_fd) @@ -1478,8 +1487,7 @@ fslmc_close_iodevices(struct rte_dpaa2_device *dev, } /* - * fslmc_process_iodevices for processing only IO (ETH, CRYPTO, and possibly - * EVENT) devices. + * fslmc_process_iodevices for processing only IO devices. */ static int fslmc_process_iodevices(struct rte_dpaa2_device *dev) @@ -1494,12 +1502,6 @@ fslmc_process_iodevices(struct rte_dpaa2_device *dev) return ret; switch (dev->dev_type) { - case DPAA2_ETH: - ret = rte_dpaa2_vfio_setup_intr(dev->intr_handle, dev_fd, - device_info.num_irqs); - if (ret) - return ret; - break; case DPAA2_CON: case DPAA2_IO: case DPAA2_CI: @@ -1523,6 +1525,61 @@ fslmc_process_iodevices(struct rte_dpaa2_device *dev) return 0; } +int +fslmc_vfio_dev_setup(struct rte_dpaa2_device *dev) +{ + struct vfio_device_info device_info = { .argsz = sizeof(device_info) }; + int dev_fd; + int ret; + + ret = fslmc_vfio_setup_device(dev->device.name, &dev_fd, &device_info); + if (ret) { + DPAA2_BUS_ERR("VFIO setup failed for %s: %d", + dev->device.name, ret); + return ret; + } + + if (dev->dev_type == DPAA2_ETH) { + ret = rte_dpaa2_vfio_setup_intr(dev->intr_handle, dev_fd, + device_info.num_irqs); + if (ret) { + DPAA2_BUS_ERR("Interrupt setup failed for %s: %d", + dev->device.name, ret); + fslmc_vfio_dev_close(dev); + return ret; + } + } + + DPAA2_BUS_DEBUG("Device (%s) VFIO setup completed", dev->device.name); + return 0; +} + +int +fslmc_vfio_dev_close(struct rte_dpaa2_device *dev) +{ + int vfio_group_fd; + int ret; + const char *group_name = fslmc_vfio_get_group_name(); + + vfio_group_fd = fslmc_vfio_group_fd_by_name(group_name); + if (vfio_group_fd <= 0) { + DPAA2_BUS_ERR("Get fd by name(%s) failed(%d)", + group_name, vfio_group_fd); + if (vfio_group_fd < 0) + return vfio_group_fd; + return -EIO; + } + + dpaa2_close_intr(dev->intr_handle); + + ret = fslmc_vfio_group_remove_dev(vfio_group_fd, dev->device.name); + if (ret) + DPAA2_BUS_ERR("Failed to remove %s from vfio", dev->device.name); + + DPAA2_BUS_DEBUG("Device (%s) closed", dev->device.name); + return ret; +} + static int fslmc_process_mcp(struct rte_dpaa2_device *dev) { @@ -1682,16 +1739,6 @@ fslmc_vfio_process_group(void) } } - /* Process regular devices */ - RTE_BUS_FOREACH_DEV(dev, &rte_fslmc_bus) { - ret = fslmc_process_iodevices(dev); - if (ret) { - DPAA2_BUS_DEBUG("Dev (%s) init failed", - dev->device.name); - return ret; - } - } - return 0; } diff --git a/drivers/bus/fslmc/fslmc_vfio.h b/drivers/bus/fslmc/fslmc_vfio.h index 57fe7038de..0973ddbe37 100644 --- a/drivers/bus/fslmc/fslmc_vfio.h +++ b/drivers/bus/fslmc/fslmc_vfio.h @@ -61,4 +61,9 @@ char *fslmc_get_container(void); int fslmc_get_container_group(const char *group_name, int *gropuid); int fslmc_vfio_dmamap(void); int fslmc_vfio_dmaunmap(void); + +struct rte_dpaa2_device; +int fslmc_vfio_dev_setup(struct rte_dpaa2_device *dev); +int fslmc_vfio_dev_close(struct rte_dpaa2_device *dev); + #endif /* _FSLMC_VFIO_H_ */ -- 2.54.0
