Move VFIO device and interrupt setup for ETH, CRYPTO, and QDMA devices from fslmc_vfio.c to fslmc_bus.c scan layer.
Add fslmc_vfio_dev_setup() (resp. fslmc_vfio_dev_close()) wrapper that handles VFIO device fd acquisition and interrupt setup (resp. device removal), hiding VFIO internals from the bus layer. Signed-off-by: David Marchand <[email protected]> --- drivers/bus/fslmc/fslmc_bus.c | 13 ++++++ drivers/bus/fslmc/fslmc_vfio.c | 80 ++++++++++++++++++++++++++++++---- drivers/bus/fslmc/fslmc_vfio.h | 5 +++ 3 files changed, 90 insertions(+), 8 deletions(-) diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c index a03ef6051a..a1bc7d216c 100644 --- a/drivers/bus/fslmc/fslmc_bus.c +++ b/drivers/bus/fslmc/fslmc_bus.c @@ -488,6 +488,18 @@ rte_fslmc_scan(void) DPAA2_BUS_ERR("Unable to setup devices %d", ret); return 0; } + + RTE_BUS_FOREACH_DEV(dev, &rte_fslmc_bus) { + if (dev->dev_type != DPAA2_ETH && + dev->dev_type != DPAA2_CRYPTO && + dev->dev_type != DPAA2_QDMA) + continue; + ret = fslmc_vfio_dev_setup(dev); + if (ret) { + DPAA2_BUS_ERR("Dev (%s) VFIO setup failed", dev->device.name); + return 0; + } + } } process_once = 1; @@ -534,6 +546,7 @@ rte_fslmc_close(struct rte_bus *bus) continue; if (rte_dev_is_probed(&dev->device) && fslmc_bus_unplug_device(&dev->device)) DPAA2_BUS_ERR("Unable to remove %s", dev->device.name); + fslmc_vfio_dev_close(dev); } ret = fslmc_vfio_close_group(); diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c index 5720f366db..9655c27ca8 100644 --- a/drivers/bus/fslmc/fslmc_vfio.c +++ b/drivers/bus/fslmc/fslmc_vfio.c @@ -1387,6 +1387,15 @@ rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle, return -EIO; } +static void +dpaa2_close_intr(struct rte_intr_handle *intr_handle) +{ + if (rte_intr_fd_get(intr_handle) >= 0) { + close(rte_intr_fd_get(intr_handle)); + rte_intr_fd_set(intr_handle, -1); + } +} + static void fslmc_close_iodevices(struct rte_dpaa2_device *dev, int vfio_fd) @@ -1394,6 +1403,10 @@ fslmc_close_iodevices(struct rte_dpaa2_device *dev, struct rte_dpaa2_object *object = NULL; int ret; + RTE_VERIFY(dev->dev_type != DPAA2_CRYPTO); + RTE_VERIFY(dev->dev_type != DPAA2_ETH); + RTE_VERIFY(dev->dev_type != DPAA2_QDMA); + switch (dev->dev_type) { case DPAA2_IO: case DPAA2_CON: @@ -1421,8 +1434,7 @@ fslmc_close_iodevices(struct rte_dpaa2_device *dev, } /* - * fslmc_process_iodevices for processing only IO (ETH, CRYPTO, and possibly - * EVENT) devices. + * fslmc_process_iodevices for processing only IO devices. */ static int fslmc_process_iodevices(struct rte_dpaa2_device *dev) @@ -1437,12 +1449,6 @@ fslmc_process_iodevices(struct rte_dpaa2_device *dev) return ret; switch (dev->dev_type) { - case DPAA2_ETH: - ret = rte_dpaa2_vfio_setup_intr(dev->intr_handle, dev_fd, - device_info.num_irqs); - if (ret) - return ret; - break; case DPAA2_CON: case DPAA2_IO: case DPAA2_CI: @@ -1466,6 +1472,60 @@ fslmc_process_iodevices(struct rte_dpaa2_device *dev) return 0; } +int +fslmc_vfio_dev_setup(struct rte_dpaa2_device *dev) +{ + int dev_fd = -1; + int ret; + struct vfio_device_info device_info = { .argsz = sizeof(device_info) }; + + ret = fslmc_vfio_setup_device(dev->device.name, &dev_fd, &device_info); + if (ret) { + DPAA2_BUS_ERR("VFIO setup failed for %s: %d", + dev->device.name, ret); + return ret; + } + + if (dev->dev_type == DPAA2_ETH) { + ret = rte_dpaa2_vfio_setup_intr(dev->intr_handle, dev_fd, + device_info.num_irqs); + if (ret) { + DPAA2_BUS_ERR("Interrupt setup failed for %s: %d", + dev->device.name, ret); + return ret; + } + } + + DPAA2_BUS_DEBUG("Device (%s) VFIO setup completed", dev->device.name); + return 0; +} + +int +fslmc_vfio_dev_close(struct rte_dpaa2_device *dev) +{ + int vfio_group_fd; + int ret; + const char *group_name = fslmc_vfio_get_group_name(); + + vfio_group_fd = fslmc_vfio_group_fd_by_name(group_name); + if (vfio_group_fd <= 0) { + DPAA2_BUS_ERR("Get fd by name(%s) failed(%d)", + group_name, vfio_group_fd); + if (vfio_group_fd < 0) + return vfio_group_fd; + return -EIO; + } + + dpaa2_close_intr(dev->intr_handle); + + ret = fslmc_vfio_group_remove_dev(vfio_group_fd, dev->device.name); + if (ret) + DPAA2_BUS_ERR("Failed to remove %s from vfio", dev->device.name); + + DPAA2_BUS_DEBUG("Device (%s) closed", dev->device.name); + return ret; +} + static int fslmc_process_mcp(struct rte_dpaa2_device *dev) { @@ -1549,6 +1609,8 @@ fslmc_vfio_close_group(void) case DPAA2_ETH: case DPAA2_CRYPTO: case DPAA2_QDMA: + /* ethdev, cryptodev, dmadev are handled at the bus level */ + break; case DPAA2_IO: case DPAA2_CON: case DPAA2_CI: @@ -1607,6 +1669,8 @@ fslmc_vfio_process_group(void) case DPAA2_ETH: case DPAA2_CRYPTO: case DPAA2_QDMA: + /* ethdev, cryptodev, dmadev are handled at the bus level */ + break; case DPAA2_CON: case DPAA2_CI: case DPAA2_BPOOL: diff --git a/drivers/bus/fslmc/fslmc_vfio.h b/drivers/bus/fslmc/fslmc_vfio.h index c995fd67b8..530aaa0a32 100644 --- a/drivers/bus/fslmc/fslmc_vfio.h +++ b/drivers/bus/fslmc/fslmc_vfio.h @@ -60,4 +60,9 @@ int fslmc_vfio_close_group(void); char *fslmc_get_container(void); int fslmc_get_container_group(const char *group_name, int *gropuid); int fslmc_vfio_dmamap(void); + +struct rte_dpaa2_device; +int fslmc_vfio_dev_setup(struct rte_dpaa2_device *dev); +int fslmc_vfio_dev_close(struct rte_dpaa2_device *dev); + #endif /* _FSLMC_VFIO_H_ */ -- 2.54.0
