Add a flag managed_sriov to the pci_driver structure that allows a driver to opt into disabling the Single Root I/O Virtualization (SR-IOV) capability of the device when the driver is unbound.
Add a new function pci_iov_disable() that is invoked before the remove() callback of a PCI driver and checks for the presence of the new flag. If the flag is set, invoke the sriov_configure() callback to allow the driver to gracefully disable SR-IOV. Warn if the driver fails to do so and forcibly disable SR-IOV using sriov_disable(). Since a (broken) driver may theoretically re-enable SR-IOV during its remove() callback, extend pci_iov_remove() to forcibly disable SR-IOV after remove() if needed and only if the flag managed_sriov is set. Altogether the flag ensures that when a Virtual Function (VF) is bound to a driver, the corresponding Physical Function (PF) is bound to a driver, too, since the VF devices are destroyed when the PF driver is unbound. This guarantee is a prerequisite for exposing a safe Rust API that allows a VF driver to obtain the PF device for a VF device and subsequently access the device private data of the PF device. Suggested-by: Danilo Krummrich <[email protected]> Signed-off-by: Peter Colberg <[email protected]> --- Changes in v2: - Move logic to disable SR-IOV on remove() from Rust to C. - Add driver flag managed_sriov to opt into disabling SR-IOV on remove(). --- drivers/pci/iov.c | 41 ++++++++++++++++++++++++++++++++++++++++- drivers/pci/pci-driver.c | 3 ++- drivers/pci/pci.h | 2 ++ include/linux/pci.h | 8 ++++++++ 4 files changed, 52 insertions(+), 2 deletions(-) diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c index 00784a60ba80bb55ff2790d8f87e15a90c652a24..5b6ed251b4b1e940ec5781bb10dd5c58d3609fc8 100644 --- a/drivers/pci/iov.c +++ b/drivers/pci/iov.c @@ -1011,20 +1011,59 @@ void pci_iov_release(struct pci_dev *dev) sriov_release(dev); } +/** + * pci_iov_disable - disable SR-IOV before PF driver is detached + * @dev: the PCI device + * + * Invoke sriov_configure() callback to allow the driver to gracefully disable + * SR-IOV. Warn if the driver fails to do so and forcibly disable SR-IOV. + */ +void pci_iov_disable(struct pci_dev *dev) +{ + struct pci_driver *drv = dev->driver; + struct pci_sriov *iov = dev->sriov; + + if (WARN_ON(!drv)) + return; + + if (!dev->is_physfn || !iov->num_VFs || !drv->managed_sriov) + return; + + if (!drv->sriov_configure) { + sriov_disable(dev); + return; + } + + drv->sriov_configure(dev, 0); + + if (WARN_ON(iov->num_VFs)) + sriov_disable(dev); +} + /** * pci_iov_remove - clean up SR-IOV state after PF driver is detached * @dev: the PCI device */ void pci_iov_remove(struct pci_dev *dev) { + struct pci_driver *drv = dev->driver; struct pci_sriov *iov = dev->sriov; + if (WARN_ON(!drv)) + return; + if (!dev->is_physfn) return; iov->driver_max_VFs = iov->total_VFs; - if (iov->num_VFs) + + if (iov->num_VFs && !drv->managed_sriov) { pci_warn(dev, "driver left SR-IOV enabled after remove\n"); + return; + } + + if (WARN_ON(iov->num_VFs)) + sriov_disable(dev); } /** diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c index 7c2d9d59625868886d61d8d4045d656ee0165776..e44593c67d147cd70d2d1a8a436a26857b0e446a 100644 --- a/drivers/pci/pci-driver.c +++ b/drivers/pci/pci-driver.c @@ -462,6 +462,7 @@ static void pci_device_remove(struct device *dev) struct pci_dev *pci_dev = to_pci_dev(dev); struct pci_driver *drv = pci_dev->driver; + pci_iov_disable(pci_dev); if (drv->remove) { pm_runtime_get_sync(dev); /* @@ -475,8 +476,8 @@ static void pci_device_remove(struct device *dev) pm_runtime_put_noidle(dev); } pcibios_free_irq(pci_dev); - pci_dev->driver = NULL; pci_iov_remove(pci_dev); + pci_dev->driver = NULL; /* Undo the runtime PM settings in local_pci_probe() */ pm_runtime_put_sync(dev); diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 0e67014aa0013a7086c3a45d576d4b1ca2bb159f..53692e138ed347bfcf6d5923ddd418e9860399d7 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -826,6 +826,7 @@ static inline void pci_restore_pasid_state(struct pci_dev *pdev) { } #ifdef CONFIG_PCI_IOV int pci_iov_init(struct pci_dev *dev); void pci_iov_release(struct pci_dev *dev); +void pci_iov_disable(struct pci_dev *dev); void pci_iov_remove(struct pci_dev *dev); void pci_iov_update_resource(struct pci_dev *dev, int resno); resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno); @@ -860,6 +861,7 @@ static inline int pci_iov_init(struct pci_dev *dev) return -ENODEV; } static inline void pci_iov_release(struct pci_dev *dev) { } +static inline void pci_iov_disable(struct pci_dev *dev) { } static inline void pci_iov_remove(struct pci_dev *dev) { } static inline void pci_iov_update_resource(struct pci_dev *dev, int resno) { } static inline resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, diff --git a/include/linux/pci.h b/include/linux/pci.h index b5cc0c2b99065d4a1ee4581275362e79726a2145..768a02b12ff73aeb4dc3dc33fcca7c46b524c3c0 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -997,6 +997,13 @@ struct module; * how to manage the DMA themselves and set this flag so that * the IOMMU layer will allow them to setup and manage their * own I/O address space. + * @managed_sriov: Disable SR-IOV on remove(). + * If set, the Single Root I/O Virtualization (SR-IOV) + * capability of the device is disabled when the driver is + * unbound from the device, by calling sriov_configure() + * before remove(). The presence of this flag guarantees + * that when a Virtual Function (VF) is bound to a driver, + * the Physical Function (PF) is bound to a driver, too. */ struct pci_driver { const char *name; @@ -1015,6 +1022,7 @@ struct pci_driver { struct device_driver driver; struct pci_dynids dynids; bool driver_managed_dma; + bool managed_sriov; }; #define to_pci_driver(__drv) \ -- 2.52.0
