From: Manish Honap <[email protected]> The BARs of a vfio PCI device follow the memory enable bit: clearing it zaps their mmaps and revokes their dma-bufs, and setting it re-arms the dma-bufs. A device-specific region does not have to follow that bit. The HDM memory region of a CXL Type-2 device follows the committed HDM decoder, which changes across reset, bus reset and low power, not with Memory Space Enable. Other device-specific regions, such as nvgrace coherent memory or the IGD OpRegion, do not depend on it either.
Record the region index of each dma-buf at export. vfio_pci_dma_buf_move() now moves only the BAR dma-bufs, and vfio_pci_dma_buf_cleanup() still drains every dma-buf of the device. Export vfio_pci_core_revoke_region(), which zaps the user mmaps of one device-specific region and revokes its dma-bufs, and vfio_pci_core_rearm_region(), which re-arms them. A provider calls them under memory_lock when the state its region depends on changes. Also export __vfio_pci_vmf_insert_pfn(), the pfn insert of vfio_pci_vmf_insert_pfn() without its memory enable and runtime PM checks, for the fault handler of such a region. Export vfio_pci_core_clear_memory_enable() for a provider whose failed operation left Memory Space off in the device: it zaps the BAR mmaps, revokes the BAR dma-bufs and clears vfio's record of the bit. Suggested-by: Alex Williamson <[email protected]> Assisted-by: LLM Signed-off-by: Manish Honap <[email protected]> --- drivers/vfio/pci/vfio_pci_core.c | 109 +++++++++++++++++++++++++++-- drivers/vfio/pci/vfio_pci_dmabuf.c | 98 +++++++++++++++++--------- drivers/vfio/pci/vfio_pci_priv.h | 7 ++ include/linux/vfio_pci_core.h | 8 +++ 4 files changed, 180 insertions(+), 42 deletions(-) diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 6676c561465b..9d0c22a4f3c6 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -1982,6 +1982,77 @@ static void vfio_pci_zap_bars(struct vfio_pci_core_device *vdev) unmap_mapping_range(core_vdev->inode->i_mapping, start, len, true); } +/** + * vfio_pci_core_revoke_region() - take a device-specific region out of use + * @vdev: vfio PCI device, with memory_lock held for write + * @index: vfio region index of a device-specific region + * + * Zap the user mmaps of the region and revoke its dma-bufs, for a provider + * whose region depends on device state that is about to change. The BARs + * follow the memory enable bit instead; a device-specific region may not. + * A later fault on the region goes back to the region's own fault handler. + */ +void vfio_pci_core_revoke_region(struct vfio_pci_core_device *vdev, + unsigned int index) +{ + struct vfio_pci_region *region; + + lockdep_assert_held_write(&vdev->memory_lock); + + if (WARN_ON(index < VFIO_PCI_NUM_REGIONS || + index - VFIO_PCI_NUM_REGIONS >= vdev->num_regions)) + return; + + region = &vdev->region[index - VFIO_PCI_NUM_REGIONS]; + unmap_mapping_range(vdev->vdev.inode->i_mapping, + VFIO_PCI_INDEX_TO_OFFSET(index), region->size, true); + vfio_pci_dma_buf_move_region(vdev, index, true); +} +EXPORT_SYMBOL_GPL(vfio_pci_core_revoke_region); + +/** + * vfio_pci_core_rearm_region() - put a revoked device-specific region back + * @vdev: vfio PCI device, with memory_lock held for write + * @index: vfio region index passed to vfio_pci_core_revoke_region() + */ +void vfio_pci_core_rearm_region(struct vfio_pci_core_device *vdev, + unsigned int index) +{ + lockdep_assert_held_write(&vdev->memory_lock); + + vfio_pci_dma_buf_move_region(vdev, index, false); +} +EXPORT_SYMBOL_GPL(vfio_pci_core_rearm_region); + +/** + * vfio_pci_core_clear_memory_enable() - follow a device that lost Memory Space + * @vdev: vfio PCI device, with memory_lock held for write + * + * For a provider whose failed operation left Memory Space disabled in the + * device. Zap the BAR mmaps, revoke the BAR dma-bufs and clear vfio's record + * of the Memory Space bit, as a user write that clears the bit would. The + * user's next Memory Space write enables it again. + */ +void vfio_pci_core_clear_memory_enable(struct vfio_pci_core_device *vdev) +{ + __le16 *virt_cmd; + + lockdep_assert_held_write(&vdev->memory_lock); + + /* Closed, or not open yet: vfio_config_init() reads the device. */ + if (!vdev->vconfig) + return; + + virt_cmd = (__le16 *)&vdev->vconfig[PCI_COMMAND]; + if (!(*virt_cmd & cpu_to_le16(PCI_COMMAND_MEMORY))) + return; + + vfio_pci_zap_bars(vdev); + vfio_pci_dma_buf_move(vdev, true); + *virt_cmd &= cpu_to_le16(~PCI_COMMAND_MEMORY); +} +EXPORT_SYMBOL_GPL(vfio_pci_core_clear_memory_enable); + void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device *vdev) { down_write(&vdev->memory_lock); @@ -2019,16 +2090,26 @@ static unsigned long vma_to_pfn(struct vm_area_struct *vma) return (pci_resource_start(vdev->pdev, index) >> PAGE_SHIFT) + pgoff; } -vm_fault_t vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device *vdev, - struct vm_fault *vmf, - unsigned long pfn, - unsigned int order) +/** + * __vfio_pci_vmf_insert_pfn() - insert a pfn without the memory enable and + * runtime PM checks + * @vdev: vfio PCI device, with memory_lock held for read + * @vmf: the fault being served + * @pfn: first pfn to insert + * @order: page order of the mapping + * + * For a device-specific region that does not follow the memory enable bit. + * The caller checks the state its region depends on under memory_lock, and + * revokes the region with vfio_pci_core_revoke_region() when it changes or + * before the device leaves D0. + */ +vm_fault_t __vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device *vdev, + struct vm_fault *vmf, + unsigned long pfn, + unsigned int order) { lockdep_assert_held_read(&vdev->memory_lock); - if (vdev->pm_runtime_engaged || !__vfio_pci_memory_enabled(vdev)) - return VM_FAULT_SIGBUS; - if (!order) return vmf_insert_pfn(vmf->vma, vmf->address, pfn); @@ -2040,6 +2121,20 @@ vm_fault_t vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device *vdev, return VM_FAULT_FALLBACK; } +EXPORT_SYMBOL_GPL(__vfio_pci_vmf_insert_pfn); + +vm_fault_t vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device *vdev, + struct vm_fault *vmf, + unsigned long pfn, + unsigned int order) +{ + lockdep_assert_held_read(&vdev->memory_lock); + + if (vdev->pm_runtime_engaged || !__vfio_pci_memory_enabled(vdev)) + return VM_FAULT_SIGBUS; + + return __vfio_pci_vmf_insert_pfn(vdev, vmf, pfn, order); +} EXPORT_SYMBOL_GPL(vfio_pci_vmf_insert_pfn); static vm_fault_t vfio_pci_mmap_huge_fault(struct vm_fault *vmf, diff --git a/drivers/vfio/pci/vfio_pci_dmabuf.c b/drivers/vfio/pci/vfio_pci_dmabuf.c index 8606c15ab2fe..d0f7ec5487de 100644 --- a/drivers/vfio/pci/vfio_pci_dmabuf.c +++ b/drivers/vfio/pci/vfio_pci_dmabuf.c @@ -19,6 +19,7 @@ struct vfio_pci_dma_buf { u32 nr_ranges; struct kref kref; struct completion comp; + u32 region_index; u8 revoked : 1; }; @@ -288,6 +289,7 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags, priv->vdev = vdev; priv->nr_ranges = get_dma_buf.nr_ranges; priv->size = length; + priv->region_index = get_dma_buf.region_index; ret = vdev->pci_ops->get_dmabuf_phys(vdev, &priv->provider, get_dma_buf.region_index, priv->phys_vec, dma_ranges, @@ -347,6 +349,49 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags, return ret; } +static void vfio_pci_dma_buf_move_one(struct vfio_pci_dma_buf *priv, + bool revoked) +{ + if (!get_file_active(&priv->dmabuf->file)) + return; + + if (priv->revoked != revoked) { + dma_resv_lock(priv->dmabuf->resv, NULL); + if (revoked) + priv->revoked = true; + dma_buf_invalidate_mappings(priv->dmabuf); + dma_resv_wait_timeout(priv->dmabuf->resv, + DMA_RESV_USAGE_BOOKKEEP, false, + MAX_SCHEDULE_TIMEOUT); + dma_resv_unlock(priv->dmabuf->resv); + if (revoked) { + kref_put(&priv->kref, vfio_pci_dma_buf_done); + wait_for_completion(&priv->comp); + /* + * Re-arm the registered kref reference and the + * completion so the post-revoke state matches the + * post-creation state. An un-revoke followed by a + * new mapping needs the kref to be non-zero before + * kref_get(), and vfio_pci_dma_buf_cleanup() + * delegates its drain back through this revoke + * path on a possibly-already-revoked dma-buf. + */ + kref_init(&priv->kref); + reinit_completion(&priv->comp); + } else { + dma_resv_lock(priv->dmabuf->resv, NULL); + priv->revoked = false; + dma_resv_unlock(priv->dmabuf->resv); + } + } + fput(priv->dmabuf->file); +} + +/* + * Revoke or re-arm the dma-bufs of the BARs, which follow the memory enable + * bit. A device-specific region is moved by its provider through + * vfio_pci_dma_buf_move_region(). + */ void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, bool revoked) { struct vfio_pci_dma_buf *priv; @@ -354,41 +399,23 @@ void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, bool revoked) lockdep_assert_held_write(&vdev->memory_lock); - list_for_each_entry_safe(priv, tmp, &vdev->dmabufs, dmabufs_elm) { - if (!get_file_active(&priv->dmabuf->file)) - continue; + list_for_each_entry_safe(priv, tmp, &vdev->dmabufs, dmabufs_elm) + if (priv->region_index < VFIO_PCI_NUM_REGIONS) + vfio_pci_dma_buf_move_one(priv, revoked); +} - if (priv->revoked != revoked) { - dma_resv_lock(priv->dmabuf->resv, NULL); - if (revoked) - priv->revoked = true; - dma_buf_invalidate_mappings(priv->dmabuf); - dma_resv_wait_timeout(priv->dmabuf->resv, - DMA_RESV_USAGE_BOOKKEEP, false, - MAX_SCHEDULE_TIMEOUT); - dma_resv_unlock(priv->dmabuf->resv); - if (revoked) { - kref_put(&priv->kref, vfio_pci_dma_buf_done); - wait_for_completion(&priv->comp); - /* - * Re-arm the registered kref reference and the - * completion so the post-revoke state matches the - * post-creation state. An un-revoke followed by a - * new mapping needs the kref to be non-zero before - * kref_get(), and vfio_pci_dma_buf_cleanup() - * delegates its drain back through this revoke - * path on a possibly-already-revoked dma-buf. - */ - kref_init(&priv->kref); - reinit_completion(&priv->comp); - } else { - dma_resv_lock(priv->dmabuf->resv, NULL); - priv->revoked = false; - dma_resv_unlock(priv->dmabuf->resv); - } - } - fput(priv->dmabuf->file); - } +/* Revoke or re-arm the dma-bufs exported from one device-specific region. */ +void vfio_pci_dma_buf_move_region(struct vfio_pci_core_device *vdev, + unsigned int index, bool revoked) +{ + struct vfio_pci_dma_buf *priv; + struct vfio_pci_dma_buf *tmp; + + lockdep_assert_held_write(&vdev->memory_lock); + + list_for_each_entry_safe(priv, tmp, &vdev->dmabufs, dmabufs_elm) + if (priv->region_index == index) + vfio_pci_dma_buf_move_one(priv, revoked); } void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device *vdev) @@ -405,7 +432,8 @@ void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device *vdev) * ready, so cleanup itself does not need to participate in kref * bookkeeping. */ - vfio_pci_dma_buf_move(vdev, true); + list_for_each_entry_safe(priv, tmp, &vdev->dmabufs, dmabufs_elm) + vfio_pci_dma_buf_move_one(priv, true); list_for_each_entry_safe(priv, tmp, &vdev->dmabufs, dmabufs_elm) { if (!get_file_active(&priv->dmabuf->file)) diff --git a/drivers/vfio/pci/vfio_pci_priv.h b/drivers/vfio/pci/vfio_pci_priv.h index dbede6e24fa9..23bccfbb7eb7 100644 --- a/drivers/vfio/pci/vfio_pci_priv.h +++ b/drivers/vfio/pci/vfio_pci_priv.h @@ -141,6 +141,8 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags, size_t argsz); void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device *vdev); void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, bool revoked); +void vfio_pci_dma_buf_move_region(struct vfio_pci_core_device *vdev, + unsigned int index, bool revoked); #else static inline int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags, @@ -156,6 +158,11 @@ static inline void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, bool revoked) { } +static inline void +vfio_pci_dma_buf_move_region(struct vfio_pci_core_device *vdev, + unsigned int index, bool revoked) +{ +} #endif #endif diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h index cb9ba92b696a..7dd93ef103cb 100644 --- a/include/linux/vfio_pci_core.h +++ b/include/linux/vfio_pci_core.h @@ -227,6 +227,11 @@ int vfio_pci_core_register_dev_region(struct vfio_pci_core_device *vdev, unsigned int type, unsigned int subtype, const struct vfio_pci_regops *ops, size_t size, u32 flags, void *data); +void vfio_pci_core_revoke_region(struct vfio_pci_core_device *vdev, + unsigned int index); +void vfio_pci_core_rearm_region(struct vfio_pci_core_device *vdev, + unsigned int index); +void vfio_pci_core_clear_memory_enable(struct vfio_pci_core_device *vdev); void vfio_pci_core_close_device(struct vfio_device *core_vdev); int vfio_pci_core_init_dev(struct vfio_device *core_vdev); void vfio_pci_core_release_dev(struct vfio_device *core_vdev); @@ -250,6 +255,9 @@ ssize_t vfio_pci_core_read(struct vfio_device *core_vdev, char __user *buf, size_t count, loff_t *ppos); ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *buf, size_t count, loff_t *ppos); +vm_fault_t __vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device *vdev, + struct vm_fault *vmf, unsigned long pfn, + unsigned int order); vm_fault_t vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device *vdev, struct vm_fault *vmf, unsigned long pfn, unsigned int order); -- 2.53.0
