On Mon, 22 May 2023 14:46:17 -0600 Alex Williamson <[email protected]> wrote:
> On Sat, 13 May 2023 06:28:17 -0700 > Yi Liu <[email protected]> wrote: > > > this prepares for adding DETACH ioctl for physical VFIO devices. > > > > Reviewed-by: Kevin Tian <[email protected]> > > Tested-by: Terrence Xu <[email protected]> > > Tested-by: Nicolin Chen <[email protected]> > > Tested-by: Matthew Rosato <[email protected]> > > Tested-by: Yanting Jiang <[email protected]> > > Tested-by: Shameer Kolothum <[email protected]> > > Signed-off-by: Yi Liu <[email protected]> > > --- > > Documentation/driver-api/vfio.rst | 8 +++++--- > > drivers/vfio/fsl-mc/vfio_fsl_mc.c | 1 + > > drivers/vfio/iommufd.c | 20 +++++++++++++++++++ > > .../vfio/pci/hisilicon/hisi_acc_vfio_pci.c | 2 ++ > > drivers/vfio/pci/mlx5/main.c | 1 + > > drivers/vfio/pci/vfio_pci.c | 1 + > > drivers/vfio/platform/vfio_amba.c | 1 + > > drivers/vfio/platform/vfio_platform.c | 1 + > > drivers/vfio/vfio_main.c | 3 ++- > > include/linux/vfio.h | 8 +++++++- > > 10 files changed, 41 insertions(+), 5 deletions(-) > > > > diff --git a/Documentation/driver-api/vfio.rst > > b/Documentation/driver-api/vfio.rst > > index 68abc089d6dd..363e12c90b87 100644 > > --- a/Documentation/driver-api/vfio.rst > > +++ b/Documentation/driver-api/vfio.rst > > @@ -279,6 +279,7 @@ similar to a file operations structure:: > > struct iommufd_ctx *ictx, u32 > > *out_device_id); > > void (*unbind_iommufd)(struct vfio_device *vdev); > > int (*attach_ioas)(struct vfio_device *vdev, u32 *pt_id); > > + void (*detach_ioas)(struct vfio_device *vdev); > > int (*open_device)(struct vfio_device *vdev); > > void (*close_device)(struct vfio_device *vdev); > > ssize_t (*read)(struct vfio_device *vdev, char __user *buf, > > @@ -315,9 +316,10 @@ container_of(). > > - The [un]bind_iommufd callbacks are issued when the device is bound to > > and unbound from iommufd. > > > > - - The attach_ioas callback is issued when the device is attached to an > > - IOAS managed by the bound iommufd. The attached IOAS is automatically > > - detached when the device is unbound from iommufd. > > + - The [de]attach_ioas callback is issued when the device is attached to > > + and detached from an IOAS managed by the bound iommufd. However, the > > + attached IOAS can also be automatically detached when the device is > > + unbound from iommufd. > > > > - The read/write/mmap callbacks implement the device region access > > defined > > by the device's own VFIO_DEVICE_GET_REGION_INFO ioctl. > > diff --git a/drivers/vfio/fsl-mc/vfio_fsl_mc.c > > b/drivers/vfio/fsl-mc/vfio_fsl_mc.c > > index c89a047a4cd8..d540cf683d93 100644 > > --- a/drivers/vfio/fsl-mc/vfio_fsl_mc.c > > +++ b/drivers/vfio/fsl-mc/vfio_fsl_mc.c > > @@ -594,6 +594,7 @@ static const struct vfio_device_ops vfio_fsl_mc_ops = { > > .bind_iommufd = vfio_iommufd_physical_bind, > > .unbind_iommufd = vfio_iommufd_physical_unbind, > > .attach_ioas = vfio_iommufd_physical_attach_ioas, > > + .detach_ioas = vfio_iommufd_physical_detach_ioas, > > }; > > > > static struct fsl_mc_driver vfio_fsl_mc_driver = { > > diff --git a/drivers/vfio/iommufd.c b/drivers/vfio/iommufd.c > > index 07b58c4625b5..e3953e1617a5 100644 > > --- a/drivers/vfio/iommufd.c > > +++ b/drivers/vfio/iommufd.c > > @@ -167,6 +167,14 @@ int vfio_iommufd_physical_attach_ioas(struct > > vfio_device *vdev, u32 *pt_id) > > { > > int rc; > > > > + lockdep_assert_held(&vdev->dev_set->lock); > > + > > + if (WARN_ON(!vdev->iommufd_device)) > > + return -EINVAL; > > + > > + if (vdev->iommufd_attached) > > + return -EBUSY; > > + > > rc = iommufd_device_attach(vdev->iommufd_device, pt_id); > > if (rc) > > return rc; > > @@ -175,6 +183,18 @@ int vfio_iommufd_physical_attach_ioas(struct > > vfio_device *vdev, u32 *pt_id) > > } > > EXPORT_SYMBOL_GPL(vfio_iommufd_physical_attach_ioas); > > > > +void vfio_iommufd_physical_detach_ioas(struct vfio_device *vdev) > > +{ > > + lockdep_assert_held(&vdev->dev_set->lock); > > + > > + if (WARN_ON(!vdev->iommufd_device) || !vdev->iommufd_attached) > > + return; > > + > > + iommufd_device_detach(vdev->iommufd_device); > > + vdev->iommufd_attached = false; > > +} > > +EXPORT_SYMBOL_GPL(vfio_iommufd_physical_detach_ioas); > > Can't a user trigger this WARN_ON by simply repeatedly calling the > to-be-added detach ioctl? Thanks, Oops, didn't track the close paren correctly, disregard. Thanks, Alex > > + > > /* > > * The emulated standard ops mean that vfio_device is going to use the > > * "mdev path" and will call vfio_pin_pages()/vfio_dma_rw(). Drivers using > > this > > diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c > > b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c > > index a117eaf21c14..b2f9778c8366 100644 > > --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c > > +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c > > @@ -1373,6 +1373,7 @@ static const struct vfio_device_ops > > hisi_acc_vfio_pci_migrn_ops = { > > .bind_iommufd = vfio_iommufd_physical_bind, > > .unbind_iommufd = vfio_iommufd_physical_unbind, > > .attach_ioas = vfio_iommufd_physical_attach_ioas, > > + .detach_ioas = vfio_iommufd_physical_detach_ioas, > > }; > > > > static const struct vfio_device_ops hisi_acc_vfio_pci_ops = { > > @@ -1391,6 +1392,7 @@ static const struct vfio_device_ops > > hisi_acc_vfio_pci_ops = { > > .bind_iommufd = vfio_iommufd_physical_bind, > > .unbind_iommufd = vfio_iommufd_physical_unbind, > > .attach_ioas = vfio_iommufd_physical_attach_ioas, > > + .detach_ioas = vfio_iommufd_physical_detach_ioas, > > }; > > > > static int hisi_acc_vfio_pci_probe(struct pci_dev *pdev, const struct > > pci_device_id *id) > > diff --git a/drivers/vfio/pci/mlx5/main.c b/drivers/vfio/pci/mlx5/main.c > > index d95fd382814c..42ec574a8622 100644 > > --- a/drivers/vfio/pci/mlx5/main.c > > +++ b/drivers/vfio/pci/mlx5/main.c > > @@ -1320,6 +1320,7 @@ static const struct vfio_device_ops mlx5vf_pci_ops = { > > .bind_iommufd = vfio_iommufd_physical_bind, > > .unbind_iommufd = vfio_iommufd_physical_unbind, > > .attach_ioas = vfio_iommufd_physical_attach_ioas, > > + .detach_ioas = vfio_iommufd_physical_detach_ioas, > > }; > > > > static int mlx5vf_pci_probe(struct pci_dev *pdev, > > diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c > > index 29091ee2e984..cb5b7f865d58 100644 > > --- a/drivers/vfio/pci/vfio_pci.c > > +++ b/drivers/vfio/pci/vfio_pci.c > > @@ -141,6 +141,7 @@ static const struct vfio_device_ops vfio_pci_ops = { > > .bind_iommufd = vfio_iommufd_physical_bind, > > .unbind_iommufd = vfio_iommufd_physical_unbind, > > .attach_ioas = vfio_iommufd_physical_attach_ioas, > > + .detach_ioas = vfio_iommufd_physical_detach_ioas, > > }; > > > > static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id > > *id) > > diff --git a/drivers/vfio/platform/vfio_amba.c > > b/drivers/vfio/platform/vfio_amba.c > > index 83fe54015595..6464b3939ebc 100644 > > --- a/drivers/vfio/platform/vfio_amba.c > > +++ b/drivers/vfio/platform/vfio_amba.c > > @@ -119,6 +119,7 @@ static const struct vfio_device_ops vfio_amba_ops = { > > .bind_iommufd = vfio_iommufd_physical_bind, > > .unbind_iommufd = vfio_iommufd_physical_unbind, > > .attach_ioas = vfio_iommufd_physical_attach_ioas, > > + .detach_ioas = vfio_iommufd_physical_detach_ioas, > > }; > > > > static const struct amba_id pl330_ids[] = { > > diff --git a/drivers/vfio/platform/vfio_platform.c > > b/drivers/vfio/platform/vfio_platform.c > > index 22a1efca32a8..8cf22fa65baa 100644 > > --- a/drivers/vfio/platform/vfio_platform.c > > +++ b/drivers/vfio/platform/vfio_platform.c > > @@ -108,6 +108,7 @@ static const struct vfio_device_ops vfio_platform_ops = > > { > > .bind_iommufd = vfio_iommufd_physical_bind, > > .unbind_iommufd = vfio_iommufd_physical_unbind, > > .attach_ioas = vfio_iommufd_physical_attach_ioas, > > + .detach_ioas = vfio_iommufd_physical_detach_ioas, > > }; > > > > static struct platform_driver vfio_platform_driver = { > > diff --git a/drivers/vfio/vfio_main.c b/drivers/vfio/vfio_main.c > > index 39d552e5160b..599d551fc4b5 100644 > > --- a/drivers/vfio/vfio_main.c > > +++ b/drivers/vfio/vfio_main.c > > @@ -273,7 +273,8 @@ static int __vfio_register_dev(struct vfio_device > > *device, > > if (WARN_ON(IS_ENABLED(CONFIG_IOMMUFD) && > > (!device->ops->bind_iommufd || > > !device->ops->unbind_iommufd || > > - !device->ops->attach_ioas))) > > + !device->ops->attach_ioas || > > + !device->ops->detach_ioas))) > > return -EINVAL; > > > > /* > > diff --git a/include/linux/vfio.h b/include/linux/vfio.h > > index 431eb82e0595..f446f5901e46 100644 > > --- a/include/linux/vfio.h > > +++ b/include/linux/vfio.h > > @@ -74,7 +74,9 @@ struct vfio_device { > > * @bind_iommufd: Called when binding the device to an iommufd > > * @unbind_iommufd: Opposite of bind_iommufd > > * @attach_ioas: Called when attaching device to an IOAS/HWPT managed by > > the > > - * bound iommufd. Undo in unbind_iommufd. > > + * bound iommufd. Undo in unbind_iommufd if @detach_ioas is not > > + * called. > > + * @detach_ioas: Opposite of attach_ioas > > * @open_device: Called when the first file descriptor is opened for this > > device > > * @close_device: Opposite of open_device > > * @read: Perform read(2) on device file descriptor > > @@ -98,6 +100,7 @@ struct vfio_device_ops { > > struct iommufd_ctx *ictx, u32 *out_device_id); > > void (*unbind_iommufd)(struct vfio_device *vdev); > > int (*attach_ioas)(struct vfio_device *vdev, u32 *pt_id); > > + void (*detach_ioas)(struct vfio_device *vdev); > > int (*open_device)(struct vfio_device *vdev); > > void (*close_device)(struct vfio_device *vdev); > > ssize_t (*read)(struct vfio_device *vdev, char __user *buf, > > @@ -121,6 +124,7 @@ int vfio_iommufd_physical_bind(struct vfio_device *vdev, > > struct iommufd_ctx *ictx, u32 *out_device_id); > > void vfio_iommufd_physical_unbind(struct vfio_device *vdev); > > int vfio_iommufd_physical_attach_ioas(struct vfio_device *vdev, u32 > > *pt_id); > > +void vfio_iommufd_physical_detach_ioas(struct vfio_device *vdev); > > int vfio_iommufd_emulated_bind(struct vfio_device *vdev, > > struct iommufd_ctx *ictx, u32 *out_device_id); > > void vfio_iommufd_emulated_unbind(struct vfio_device *vdev); > > @@ -145,6 +149,8 @@ vfio_iommufd_physical_devid(struct vfio_device *vdev) > > ((void (*)(struct vfio_device *vdev)) NULL) > > #define vfio_iommufd_physical_attach_ioas \ > > ((int (*)(struct vfio_device *vdev, u32 *pt_id)) NULL) > > +#define vfio_iommufd_physical_detach_ioas \ > > + ((void (*)(struct vfio_device *vdev)) NULL) > > #define vfio_iommufd_emulated_bind \ > > ((int (*)(struct vfio_device *vdev, struct iommufd_ctx *ictx, \ > > u32 *out_device_id)) NULL) >
