[virtio-dev] [PATCH v6] iommu/virtio: Use page size bitmap supported by endpoint

[Bharat Bhushan]

Different endpoint can support different page size, probe
endpoint if it supports specific page size otherwise use
global page sizes.

Device attached to domain should support a minimum of
domain supported page sizes. If device supports more
than domain supported page sizes then device is limited
to use domain supported page sizes only.

Signed-off-by: Bharat Bhushan 
---
v5->v6
 - property length before dereference
 - Error out on no supported page sizes (page-size-mask is zero)
 - Allow device to attach to domain even it supports
   minimum of domain supported page sizes. In that case device
   will use domain page sizes only.
 - added format of pgsize_bitmap

v4->v5:
 - Rebase to Linux v5.7-rc4

v3->v4:
 - Fix whitespace error

v2->v3:
 - Fixed error return for incompatible endpoint
 - __u64 changed to __le64 in header file

 drivers/iommu/virtio-iommu.c  | 63 ---
 include/uapi/linux/virtio_iommu.h | 14 ++-
 2 files changed, 71 insertions(+), 6 deletions(-)

diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c
index 4e1d11af23c8..cbac3047a781 100644
--- a/drivers/iommu/virtio-iommu.c
+++ b/drivers/iommu/virtio-iommu.c
@@ -78,6 +78,7 @@ struct viommu_endpoint {
struct viommu_dev   *viommu;
struct viommu_domain*vdomain;
struct list_headresv_regions;
+   u64 pgsize_bitmap;
 };
 
 struct viommu_request {
@@ -415,6 +416,23 @@ static int viommu_replay_mappings(struct viommu_domain 
*vdomain)
return ret;
 }
 
+static int viommu_set_pgsize_bitmap(struct viommu_endpoint *vdev,
+   struct virtio_iommu_probe_pgsize_mask *mask,
+   size_t len)
+{
+   u64 pgsize_bitmap;
+
+   if (len < sizeof(*mask))
+   return -EINVAL;
+
+   pgsize_bitmap = le64_to_cpu(mask->pgsize_bitmap);
+   if (!pgsize_bitmap)
+   return -EINVAL;
+
+   vdev->pgsize_bitmap = pgsize_bitmap;
+   return 0;
+}
+
 static int viommu_add_resv_mem(struct viommu_endpoint *vdev,
   struct virtio_iommu_probe_resv_mem *mem,
   size_t len)
@@ -499,6 +517,9 @@ static int viommu_probe_endpoint(struct viommu_dev *viommu, 
struct device *dev)
case VIRTIO_IOMMU_PROBE_T_RESV_MEM:
ret = viommu_add_resv_mem(vdev, (void *)prop, len);
break;
+   case VIRTIO_IOMMU_PROBE_T_PAGE_SIZE_MASK:
+   ret = viommu_set_pgsize_bitmap(vdev, (void *)prop, len);
+   break;
default:
dev_err(dev, "unknown viommu prop 0x%x\n", type);
}
@@ -615,7 +636,7 @@ static int viommu_domain_finalise(struct viommu_endpoint 
*vdev,
struct viommu_dev *viommu = vdev->viommu;
struct viommu_domain *vdomain = to_viommu_domain(domain);
 
-   viommu_page_size = 1UL << __ffs(viommu->pgsize_bitmap);
+   viommu_page_size = 1UL << __ffs(vdev->pgsize_bitmap);
if (viommu_page_size > PAGE_SIZE) {
dev_err(vdev->dev,
"granule 0x%lx larger than system page size 0x%lx\n",
@@ -630,7 +651,7 @@ static int viommu_domain_finalise(struct viommu_endpoint 
*vdev,
 
vdomain->id = (unsigned int)ret;
 
-   domain->pgsize_bitmap   = viommu->pgsize_bitmap;
+   domain->pgsize_bitmap   = vdev->pgsize_bitmap;
domain->geometry= viommu->geometry;
 
vdomain->map_flags  = viommu->map_flags;
@@ -654,6 +675,38 @@ static void viommu_domain_free(struct iommu_domain *domain)
kfree(vdomain);
 }
 
+/*
+ * Check whether the endpoint's capabilities are compatible with other
+ * endpoints in the domain. Report any inconsistency.
+ */
+static bool viommu_endpoint_is_compatible(struct viommu_endpoint *vdev,
+ struct viommu_domain *vdomain)
+{
+   struct device *dev = vdev->dev;
+   u64 pgsize_bitmap;
+
+   if (vdomain->viommu != vdev->viommu) {
+   dev_err(dev, "cannot attach to foreign vIOMMU\n");
+   return false;
+   }
+
+   pgsize_bitmap = vdomain->domain.pgsize_bitmap & vdev->pgsize_bitmap;
+
+   if (pgsize_bitmap != vdomain->domain.pgsize_bitmap) {
+   dev_err(dev, "incompatible domain bitmap 0x%lx != 0x%llx\n",
+   vdomain->domain.pgsize_bitmap, vdev->pgsize_bitmap);
+   return false;
+   }
+
+   /* Domain pagesize bitmap is subset of device pagesize bitmap */
+   if (pgsize_bitmap != vdev->pgsize_bitmap) {
+   dev_info(dev, "page size bitmap used %llx, supported %llx\n",
+pgsize

[virtio-dev] [PATCH v6] iommu/virtio: Use page size bitmap supported by endpoint

[Bharat Bhushan]

Different endpoint can support different page size, probe
endpoint if it supports specific page size otherwise use
global page sizes.

Device attached to domain should support a minimum of
domain supported page sizes. If device supports more
than domain supported page sizes then device is limited
to use domain supported page sizes only.

Signed-off-by: Bharat Bhushan 
---
v5->v6
 - property length before dereference
 - Error out on no supported page sizes (page-size-mask is zero)
 - Allow device to attach to domain even it supports
   minimum of domain supported page sizes. In that case device
   will use domain page sizes only.
 - added format of pgsize_bitmap

v4->v5:
 - Rebase to Linux v5.7-rc4

v3->v4:
 - Fix whitespace error

v2->v3:
 - Fixed error return for incompatible endpoint
 - __u64 changed to __le64 in header file

 drivers/iommu/virtio-iommu.c  | 63 ---
 include/uapi/linux/virtio_iommu.h | 14 ++-
 2 files changed, 71 insertions(+), 6 deletions(-)

diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c
index 4e1d11af23c8..cbac3047a781 100644
--- a/drivers/iommu/virtio-iommu.c
+++ b/drivers/iommu/virtio-iommu.c
@@ -78,6 +78,7 @@ struct viommu_endpoint {
struct viommu_dev   *viommu;
struct viommu_domain*vdomain;
struct list_headresv_regions;
+   u64 pgsize_bitmap;
 };
 
 struct viommu_request {
@@ -415,6 +416,23 @@ static int viommu_replay_mappings(struct viommu_domain 
*vdomain)
return ret;
 }
 
+static int viommu_set_pgsize_bitmap(struct viommu_endpoint *vdev,
+   struct virtio_iommu_probe_pgsize_mask *mask,
+   size_t len)
+{
+   u64 pgsize_bitmap;
+
+   if (len < sizeof(*mask))
+   return -EINVAL;
+
+   pgsize_bitmap = le64_to_cpu(mask->pgsize_bitmap);
+   if (!pgsize_bitmap)
+   return -EINVAL;
+
+   vdev->pgsize_bitmap = pgsize_bitmap;
+   return 0;
+}
+
 static int viommu_add_resv_mem(struct viommu_endpoint *vdev,
   struct virtio_iommu_probe_resv_mem *mem,
   size_t len)
@@ -499,6 +517,9 @@ static int viommu_probe_endpoint(struct viommu_dev *viommu, 
struct device *dev)
case VIRTIO_IOMMU_PROBE_T_RESV_MEM:
ret = viommu_add_resv_mem(vdev, (void *)prop, len);
break;
+   case VIRTIO_IOMMU_PROBE_T_PAGE_SIZE_MASK:
+   ret = viommu_set_pgsize_bitmap(vdev, (void *)prop, len);
+   break;
default:
dev_err(dev, "unknown viommu prop 0x%x\n", type);
}
@@ -615,7 +636,7 @@ static int viommu_domain_finalise(struct viommu_endpoint 
*vdev,
struct viommu_dev *viommu = vdev->viommu;
struct viommu_domain *vdomain = to_viommu_domain(domain);
 
-   viommu_page_size = 1UL << __ffs(viommu->pgsize_bitmap);
+   viommu_page_size = 1UL << __ffs(vdev->pgsize_bitmap);
if (viommu_page_size > PAGE_SIZE) {
dev_err(vdev->dev,
"granule 0x%lx larger than system page size 0x%lx\n",
@@ -630,7 +651,7 @@ static int viommu_domain_finalise(struct viommu_endpoint 
*vdev,
 
vdomain->id = (unsigned int)ret;
 
-   domain->pgsize_bitmap   = viommu->pgsize_bitmap;
+   domain->pgsize_bitmap   = vdev->pgsize_bitmap;
domain->geometry= viommu->geometry;
 
vdomain->map_flags  = viommu->map_flags;
@@ -654,6 +675,38 @@ static void viommu_domain_free(struct iommu_domain *domain)
kfree(vdomain);
 }
 
+/*
+ * Check whether the endpoint's capabilities are compatible with other
+ * endpoints in the domain. Report any inconsistency.
+ */
+static bool viommu_endpoint_is_compatible(struct viommu_endpoint *vdev,
+ struct viommu_domain *vdomain)
+{
+   struct device *dev = vdev->dev;
+   u64 pgsize_bitmap;
+
+   if (vdomain->viommu != vdev->viommu) {
+   dev_err(dev, "cannot attach to foreign vIOMMU\n");
+   return false;
+   }
+
+   pgsize_bitmap = vdomain->domain.pgsize_bitmap & vdev->pgsize_bitmap;
+
+   if (pgsize_bitmap != vdomain->domain.pgsize_bitmap) {
+   dev_err(dev, "incompatible domain bitmap 0x%lx != 0x%llx\n",
+   vdomain->domain.pgsize_bitmap, vdev->pgsize_bitmap);
+   return false;
+   }
+
+   /* Domain pagesize bitmap is subset of device pagesize bitmap */
+   if (pgsize_bitmap != vdev->pgsize_bitmap) {
+   dev_info(dev, "page size bitmap used %llx, supported %llx\n",
+pgsize

[virtio-dev] RE: [PATCH v5 0/7] Add virtio-iommu driver

[Bharat Bhushan]

Hi Jean,

> -Original Message-
> From: Auger Eric 
> Sent: Friday, November 23, 2018 1:59 PM
> To: Jean-Philippe Brucker ;
> io...@lists.linux-foundation.org; linux-...@vger.kernel.org;
> devicet...@vger.kernel.org; virtualizat...@lists.linux-foundation.org; virtio-
> d...@lists.oasis-open.org; j...@8bytes.org; m...@redhat.com
> Cc: jasow...@redhat.com; robh...@kernel.org; mark.rutl...@arm.com;
> bhelg...@google.com; frowand.l...@gmail.com;
> kvm...@lists.cs.columbia.edu; tnowi...@caviumnetworks.com;
> kevin.t...@intel.com; marc.zyng...@arm.com; robin.mur...@arm.com;
> will.dea...@arm.com; lorenzo.pieral...@arm.com; Bharat Bhushan
> 
> Subject: Re: [PATCH v5 0/7] Add virtio-iommu driver
> 
> Hi Jean,
> 
> On 11/22/18 8:37 PM, Jean-Philippe Brucker wrote:
> > Implement the virtio-iommu driver, following specification v0.9 [1].
> >
> > Since v4 [2] I fixed the issues reported by Eric, and added
> > Reviewed-by from Eric and Rob. Thanks!
> >
> > I changed the specification to fix one inconsistency discussed in v4.
> > That the device fills the probe buffer with zeroes is now a "SHOULD"
> > instead of a "MAY", since it's the only way for the driver to know if
> > the device wrote the status. Existing devices already do this. In
> > addition the device now needs to fill the three padding bytes at the
> > tail with zeroes.
> >
> > You can find Linux driver and kvmtool device on branches
> > virtio-iommu/v0.9 [3]. I also lightly tested with Eric's latest QEMU
> > device [4].
> >
> > [1] Virtio-iommu specification v0.9, sources, pdf and diff from v0.8
> > git://linux-arm.org/virtio-iommu.git virtio-iommu/v0.9
> >
> https://emea01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fjpbr
> ucker.net%2Fvirtio-iommu%2Fspec%2Fv0.9%2Fvirtio-iommu-
> v0.9.pdfdata=02%7C01%7Cbharat.bhushan%40nxp.com%7C6e7180e7
> df8e41943d4108d6511db8ed%7C686ea1d3bc2b4c6fa92cd99c5c301635%7C0%
> 7C1%7C636785585424990803sdata=la0tSTLcOI5HkQ65a%2BCHKeI3H5iu
> qZ%2F8r6Q5YF8tfsU%3Dreserved=0
> >
> >
> https://emea01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fjpbr
> > ucker.net%2Fvirtio-iommu%2Fspec%2Fdiffs%2Fvirtio-iommu-pdf-diff-v0.8-
> v
> >
> 0.9.pdfdata=02%7C01%7Cbharat.bhushan%40nxp.com%7C6e7180e7d
> f8e4194
> >
> 3d4108d6511db8ed%7C686ea1d3bc2b4c6fa92cd99c5c301635%7C0%7C1%7C6
> 3678558
> >
> 5424990803sdata=AEXEib4lihcpfE6O6wLf%2BMElPtA7ZLGYE2mj0288PZ
> k%3D&
> > amp;reserved=0
> >
> > [2] [PATCH v4 0/7] Add virtio-iommu driver
> >
> >
> https://emea01.safelinks.protection.outlook.com/?url=https%3A%2F%2Flis
> > ts.linuxfoundation.org%2Fpipermail%2Fiommu%2F2018-
> November%2F031074.ht
> >
> mldata=02%7C01%7Cbharat.bhushan%40nxp.com%7C6e7180e7df8e4
> 1943d410
> >
> 8d6511db8ed%7C686ea1d3bc2b4c6fa92cd99c5c301635%7C0%7C1%7C636785
> 5854249
> >
> 90803sdata=mUUSBQ%2FjEeRGaisGBK20G9WmfXPwlERKDaeeRqHW4
> 08%3Dr
> > eserved=0
> >
> > [3] git://linux-arm.org/linux-jpb.git virtio-iommu/v0.9
> > git://linux-arm.org/kvmtool-jpb.git virtio-iommu/v0.9
> >
> > [4] [RFC v9 00/17] VIRTIO-IOMMU device
> >
> >
> https://emea01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fw
> ww
> > .mail-archive.com%2Fqemu-
> devel%40nongnu.org%2Fmsg575578.htmldata=
> >
> 02%7C01%7Cbharat.bhushan%40nxp.com%7C6e7180e7df8e41943d4108d651
> 1db8ed%
> >
> 7C686ea1d3bc2b4c6fa92cd99c5c301635%7C0%7C1%7C636785585424990803&
> amp;sd
> >
> ata=fo9WKE33Nm%2FdW2C2XcSVmv9itWjEyRN1irgEZgOWtZI%3Drese
> rved=0
> >
> > Jean-Philippe Brucker (7):
> >   dt-bindings: virtio-mmio: Add IOMMU description
> >   dt-bindings: virtio: Add virtio-pci-iommu node
> >   of: Allow the iommu-map property to omit untranslated devices
> >   PCI: OF: Initialize dev->fwnode appropriately
> >   iommu: Add virtio-iommu driver
> >   iommu/virtio: Add probe request
> >   iommu/virtio: Add event queue
> >
> >  .../devicetree/bindings/virtio/iommu.txt  |   66 +
> >  .../devicetree/bindings/virtio/mmio.txt   |   30 +
> >  MAINTAINERS   |7 +
> >  drivers/iommu/Kconfig |   11 +
> >  drivers/iommu/Makefile|1 +
> >  drivers/iommu/virtio-iommu.c  | 1157 +
> >  drivers/of/base.c     |   10 +-
> >  drivers/pci/of.c  |7 +
> >  include/uapi/linux/virtio_ids.h   |1 +
> >  include/uapi/linux/virtio_iommu.h |  161 +++
> >  10 files changed, 1448 insertions(+), 3 deletions(-)  create mode
> > 100644 Documentation/devicetree/bindings/virtio/iommu.txt
> >  create mode 100644 drivers/iommu/virtio-iommu.c  create mode 100644
> > include/uapi/linux/virtio_iommu.h
> >
> for the whole series
> Tested-by: Eric Auger 

I have tested this series with virtio/vfio both
Tested-by: Bharat Bhushan 


Thanks
-Bharat

> 
> Thanks
> 
> Eric

[virtio-dev] RE: [RFC] virtio-iommu version 0.4

[Bharat Bhushan]

Hi Eric,

> -Original Message-
> From: Auger Eric [mailto:eric.au...@redhat.com]
> Sent: Tuesday, September 12, 2017 10:43 PM
> To: Jean-Philippe Brucker <jean-philippe.bruc...@arm.com>;
> io...@lists.linux-foundation.org; k...@vger.kernel.org;
> virtualizat...@lists.linux-foundation.org; virtio-dev@lists.oasis-open.org
> Cc: will.dea...@arm.com; robin.mur...@arm.com;
> lorenzo.pieral...@arm.com; m...@redhat.com; jasow...@redhat.com;
> marc.zyng...@arm.com; eric.auger@gmail.com; Bharat Bhushan
> <bharat.bhus...@nxp.com>; pet...@redhat.com; kevin.t...@intel.com
> Subject: Re: [RFC] virtio-iommu version 0.4
> 
> Hi jean,
> 
> On 04/08/2017 20:19, Jean-Philippe Brucker wrote:
> > This is the continuation of my proposal for virtio-iommu, the para-
> > virtualized IOMMU. Here is a summary of the changes since last time [1]:
> >
> > * The virtio-iommu document now resembles an actual specification. It is
> >   split into a formal description of the virtio device, and implementation
> >   notes. Please find sources and binaries at [2].
> >
> > * Added a probe request to describe to the guest different properties that
> >   do not fit in firmware or in the virtio config space. This is a
> >   necessary stepping stone for extending the virtio-iommu.
> >
> > * There is a working Qemu prototype [3], thanks to Eric Auger and Bharat
> >   Bhushan.
> >
> > You can find the Linux driver and kvmtool device at [4] and [5]. I
> > plan to rework driver and kvmtool device slightly before sending the
> > patches.
> >
> > To understand the virtio-iommu, I advise to first read introduction
> > and motivation, then skim through implementation notes and finally
> > look at the device specification.
> >
> > I wasn't sure how to organize the review. For those who prefer to
> > comment inline, I attached v0.4 of device-operations.tex and
> > topology.tex+MSI.tex to this thread. They are the biggest chunks of
> > the document. But LaTeX isn't very pleasant to read, so you can simply
> > send a list of comments in relation to section numbers and a few words of
> context, we'll manage.
> >
> > ---
> > Version numbers 0.1-0.4 are arbitrary. I'm hoping they allow to
> > compare more easily differences since the RFC (see [6]), but haven't
> > been made public so far. This is the first public posting since
> > initial proposal [1], and the following describes all changes.
> >
> > ## v0.1 ##
> >
> > Content is the same as the RFC, but formatted to LaTeX. 'make'
> > generates one PDF and one HTML document.
> >
> > ## v0.2 ##
> >
> > Add introductions, improve topology example and firmware description
> > based on feedback and a number of useful discussions.
> >
> > ## v0.3 ##
> >
> > Add normative sections (MUST, SHOULD, etc). Clarify some things,
> > tighten the device and driver behaviour. Unmap semantics are
> > consolidated; they are now closer to VFIO Type1 v2 semantics.
> >
> > ## v0.4 ##
> >
> > Introduce PROBE requests. They provide per-endpoint information to the
> > driver that couldn't be described otherwise.
> >
> > For the moment, they allow to handle MSIs on x86 virtual platforms
> > (see 3.2). To do that we communicate reserved IOVA regions, that will
> > also be useful for describing regions that cannot be mapped for a
> > given endpoint, for instance addresses that correspond to a PCI bridge
> window.
> >
> > Introducing such a large framework for this tiny feature may seem
> > overkill, but it is needed for future extensions of the virtio-iommu
> > and I believe it really is worth the effort.
> 
> 2.6.7
> - As I am currently integrating v0.4 in QEMU here are some other comments:
> At the moment struct virtio_iommu_req_probe flags is missing in your
> header. As such I understood the ACK protocol was not implemented by the
> driver in your branch.
> - VIRTIO_IOMMU_PROBE_PROPERTY_TYPE_MASK is
> VIRTIO_IOMMU_T_MASK in your header too.
> 2.6.8.2:
> - I am really confused about what the device should report as resv regions
> depending on the PE nature (VFIO or not VFIO)
> 
> In other iommu drivers, the resv regions are populated by the iommu driver
> through its get_resv_regions callback. They are usually composed of an
> iommu specific MSI region (mapped or bypassed) and non IOMMU specific
> (device specific) reserved regions:
> iommu_dma_get_resv_regions(). In the case of virtio-iommu driver, those
> are the guest reserved regions.
> 
> First in the current virtio-iommu driver I don't see the
> iommu_dma_get_resv_r

RE: [virtio-dev] [RFC] virtio-iommu version 0.4

[Bharat Bhushan]

Hi,

> -Original Message-
> From: Jean-Philippe Brucker [mailto:jean-philippe.bruc...@arm.com]
> Sent: Thursday, August 17, 2017 3:42 PM
> To: Adam Tao <taoz...@huawei.com>
> Cc: io...@lists.linux-foundation.org; k...@vger.kernel.org;
> virtualizat...@lists.linux-foundation.org; virtio-dev@lists.oasis-open.org;
> will.dea...@arm.com; robin.mur...@arm.com; lorenzo.pieral...@arm.com;
> m...@redhat.com; jasow...@redhat.com; marc.zyng...@arm.com;
> eric.au...@redhat.com; eric.auger@gmail.com; Bharat Bhushan
> <bharat.bhus...@nxp.com>; pet...@redhat.com; kevin.t...@intel.com
> Subject: Re: [virtio-dev] [RFC] virtio-iommu version 0.4
> 
> Hi Adam,
> 
> On 16/08/17 05:08, Adam Tao wrote:
> >> * There is a working Qemu prototype [3], thanks to Eric Auger and Bharat
> >>   Bhushan.
> > Hi, Brucker
> > I read the related spec for virtio IOMMU, I am wondering if we support
> > both the virtual and physical devices in the guest to use the virtio
> > IOMMU, how can we config the related address translation for the
> > different type of devices.
> > e.g.
> > 1. for Virtio devs(e.g. virtio-net), we can change the memmap table
> > used by the vhost device.
> > 2. for physical devices we can directly config the IOMMU/SMMU on the
> > host.
> >
> > So do you know are there any RFCs for the back-end implementation for
> > the virtio-IOMMU thanks Adam
> 
> Eric's series [3] adds support for virtual devices (I'v only had time to test
> virtio-net so far, but as vhost-iotlb seems to be supported by Qemu, it should
> work with vhost-net as well). Bharat is working on the VFIO backend for
> physical devices. As far as I'm aware, his latest version is:
> 
> [7] https://lists.gnu.org/archive/html/qemu-devel/2017-07/msg04094.html

I am working on newer version which will be based on Eric-v3.
Actually I wanted to include the fix for the issue reported by Eric, it fails 
if we assign more than one interface.
I reproduced the issues on my side and know the root cause. I am working on 
fixing this.

Thanks
-Bharat

> 
> Thanks,
> Jean
> 
> 
> >> [1] http://www.spinics.net/lists/kvm/msg147990.html
> >> [2] git://linux-arm.org/virtio-iommu.git branch viommu/v0.4
> >> http://www.linux-arm.org/git?p=virtio-
> iommu.git;a=blob;f=dist/v0.4/virtio-iommu-v0.4.pdf
> >> I reiterate the disclaimers: don't use this document as a reference,
> >> it's a draft. It's also not an OASIS document yet. It may be riddled
> >> with mistakes. As this is a working draft, it is unstable and I do not
> >> guarantee backward compatibility of future versions.
> >> [3] https://lists.gnu.org/archive/html/qemu-arm/2017-08/msg4.html
> >> [4] git://linux-arm.org/linux-jpb.git virtio-iommu/v0.4
> >> Warning: UAPI headers have changed! They didn't follow the spec,
> >> please update. (Use branch v0.1, that has the old headers, for the
> >> Qemu prototype [3])
> >> [5] git://linux-arm.org/kvmtool-jpb.git virtio-iommu/v0.4
> >> Warning: command-line has changed! Use --viommu vfio[,opts] and
> >> --viommu virtio[,opts] to instantiate a device.
> >> [6]
> >> http://www.linux-arm.org/git?p=virtio-iommu.git;a=tree;f=dist/diffs

RE: [virtio-dev] [RFC PATCH linux] iommu: Add virtio-iommu driver

[Bharat Bhushan]

Hi Jean

> -Original Message-
> From: virtio-dev@lists.oasis-open.org [mailto:virtio-dev@lists.oasis-
> open.org] On Behalf Of Jean-Philippe Brucker
> Sent: Saturday, April 08, 2017 12:53 AM
> To: io...@lists.linux-foundation.org; k...@vger.kernel.org;
> virtualizat...@lists.linux-foundation.org; virtio-dev@lists.oasis-open.org
> Cc: cd...@linaro.org; will.dea...@arm.com; robin.mur...@arm.com;
> lorenzo.pieral...@arm.com; j...@8bytes.org; m...@redhat.com;
> jasow...@redhat.com; alex.william...@redhat.com;
> marc.zyng...@arm.com
> Subject: [virtio-dev] [RFC PATCH linux] iommu: Add virtio-iommu driver
> 
> The virtio IOMMU is a para-virtualized device, allowing to send IOMMU
> requests such as map/unmap over virtio-mmio transport. This driver should
> illustrate the initial proposal for virtio-iommu, that you hopefully received
> with it. It handle attach, detach, map and unmap requests.
> 
> The bulk of the code is to create requests and send them through virtio.
> Implementing the IOMMU API is fairly straightforward since the virtio-iommu
> MAP/UNMAP interface is almost identical. I threw in a custom
> map_sg() function which takes up some space, but is optional. The core
> function would send a sequence of map requests, waiting for a reply
> between each mapping. This optimization avoids yielding to the host after
> each map, and instead prepares a batch of requests in the virtio ring and
> kicks the host once.
> 
> It must be applied on top of the probe deferral work for IOMMU, currently
> under discussion. This allows to dissociate early driver detection and device
> probing: device-tree or ACPI is parsed early to find which devices are
> translated by the IOMMU, but the IOMMU itself cannot be probed until the
> core virtio module is loaded.
> 
> Enabling DEBUG makes it extremely verbose at the moment, but it should be
> calmer in next versions.
> 
> Signed-off-by: Jean-Philippe Brucker 
> ---
>  drivers/iommu/Kconfig |  11 +
>  drivers/iommu/Makefile|   1 +
>  drivers/iommu/virtio-iommu.c  | 980
> ++
>  include/uapi/linux/Kbuild |   1 +
>  include/uapi/linux/virtio_ids.h   |   1 +
>  include/uapi/linux/virtio_iommu.h | 142 ++
>  6 files changed, 1136 insertions(+)
>  create mode 100644 drivers/iommu/virtio-iommu.c  create mode 100644
> include/uapi/linux/virtio_iommu.h
> 
> diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index
> 37e204f3d9be..8cd56ee9a93a 100644
> --- a/drivers/iommu/Kconfig
> +++ b/drivers/iommu/Kconfig
> @@ -359,4 +359,15 @@ config MTK_IOMMU_V1
> 
> if unsure, say N here.
> 
> +config VIRTIO_IOMMU
> + tristate "Virtio IOMMU driver"
> + depends on VIRTIO_MMIO
> + select IOMMU_API
> + select INTERVAL_TREE
> + select ARM_DMA_USE_IOMMU if ARM
> + help
> +   Para-virtualised IOMMU driver with virtio.
> +
> +   Say Y here if you intend to run this kernel as a guest.
> +
>  endif # IOMMU_SUPPORT
> diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index
> 195f7b997d8e..1199d8475802 100644
> --- a/drivers/iommu/Makefile
> +++ b/drivers/iommu/Makefile
> @@ -27,3 +27,4 @@ obj-$(CONFIG_TEGRA_IOMMU_SMMU) += tegra-
> smmu.o
>  obj-$(CONFIG_EXYNOS_IOMMU) += exynos-iommu.o
>  obj-$(CONFIG_FSL_PAMU) += fsl_pamu.o fsl_pamu_domain.o
>  obj-$(CONFIG_S390_IOMMU) += s390-iommu.o
> +obj-$(CONFIG_VIRTIO_IOMMU) += virtio-iommu.o
> diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c
> new file mode 100644 index ..1cf4f57b7817
> --- /dev/null
> +++ b/drivers/iommu/virtio-iommu.c
> @@ -0,0 +1,980 @@
> +/*
> + * Virtio driver for the paravirtualized IOMMU
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License version 2 as
> + * published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License
> + * along with this program; if not, write to the Free Software
> + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
> USA.
> + *
> + * Copyright (C) 2017 ARM Limited
> + *
> + * Author: Jean-Philippe Brucker   */
> +
> +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
> +
> +#include 
> +#include 
> +#include 
> +#include 
> +#include 
> +#include 
> +#include 
> +#include 
> +#include 
> +#include 
> +#include 
> +#include 
> +#include 
> +#include 
> +
> +#include 
> +
> +struct viommu_dev {
> + struct iommu_device iommu;
> + struct device   *dev;
> + struct virtio_device*vdev;
> +
> + struct virtqueue

[virtio-dev] RE: [RFC PATCH kvmtool 00/15] Add virtio-iommu

[Bharat Bhushan]

Hi Jean,

I am trying to run and review on my side but I see Linux patches are not with 
latest kernel version.
Will it be possible for you to share your Linux and kvmtool git repository 
reference?

Thanks
-Bharat

> -Original Message-
> From: virtualization-boun...@lists.linux-foundation.org
> [mailto:virtualization-boun...@lists.linux-foundation.org] On Behalf Of Jean-
> Philippe Brucker
> Sent: Saturday, April 08, 2017 12:55 AM
> To: io...@lists.linux-foundation.org; k...@vger.kernel.org;
> virtualizat...@lists.linux-foundation.org; virtio-dev@lists.oasis-open.org
> Cc: cd...@linaro.org; lorenzo.pieral...@arm.com; m...@redhat.com;
> marc.zyng...@arm.com; j...@8bytes.org; will.dea...@arm.com;
> robin.mur...@arm.com
> Subject: [RFC PATCH kvmtool 00/15] Add virtio-iommu
> 
> Implement a virtio-iommu device and translate DMA traffic from vfio and
> virtio devices. Virtio needed some rework to support scatter-gather accesses
> to vring and buffers at page granularity. Patch 3 implements the actual 
> virtio-
> iommu device.
> 
> Adding --viommu on the command-line now inserts a virtual IOMMU in front
> of all virtio and vfio devices:
> 
>   $ lkvm run -k Image --console virtio -p console=hvc0 \
>  --viommu --vfio 0 --vfio 4 --irqchip gicv3-its
>   ...
>   [2.998949] virtio_iommu virtio0: probe successful
>   [3.007739] virtio_iommu virtio1: probe successful
>   ...
>   [3.165023] iommu: Adding device :00:00.0 to group 0
>   [3.536480] iommu: Adding device 10200.virtio to group 1
>   [3.553643] iommu: Adding device 10600.virtio to group 2
>   [3.570687] iommu: Adding device 10800.virtio to group 3
>   [3.627425] iommu: Adding device 10a00.virtio to group 4
>   [7.823689] iommu: Adding device :00:01.0 to group 5
>   ...
> 
> Patches 13 and 14 add debug facilities. Some statistics are gathered for each
> address space and can be queried via the debug builtin:
> 
>   $ lkvm debug -n guest-1210 --iommu stats
>   iommu 0 "viommu-vfio"
> kicks 1255
> requests  1256
> ioas 1
>   maps7
>   unmaps  4
>   resident2101248
> ioas 6
>   maps623
>   unmaps  620
>   resident16384
>   iommu 1 "viommu-virtio"
> kicks 11426
> requests  11431
> ioas 2
>   maps2836
>   unmaps  2835
>   resident8192
>   accesses2836
>   ...
> 
> This is based on the VFIO patchset[1], itself based on Andre's ITS work.
> The VFIO bits have only been tested on a software model and are unlikely to
> work on actual hardware, but I also tested virtio on an ARM Juno.
> 
> [1] http://www.spinics.net/lists/kvm/msg147624.html
> 
> Jean-Philippe Brucker (15):
>   virtio: synchronize virtio-iommu headers with Linux
>   FDT: (re)introduce a dynamic phandle allocator
>   virtio: add virtio-iommu
>   Add a simple IOMMU
>   iommu: describe IOMMU topology in device-trees
>   irq: register MSI doorbell addresses
>   virtio: factor virtqueue initialization
>   virtio: add vIOMMU instance for virtio devices
>   virtio: access vring and buffers through IOMMU mappings
>   virtio-pci: translate MSIs with the virtual IOMMU
>   virtio: set VIRTIO_F_IOMMU_PLATFORM when necessary
>   vfio: add support for virtual IOMMU
>   virtio-iommu: debug via IPC
>   virtio-iommu: implement basic debug commands
>   virtio: use virtio-iommu when available
> 
>  Makefile  |   3 +
>  arm/gic.c |   4 +
>  arm/include/arm-common/fdt-arch.h |   2 +-
>  arm/pci.c |  49 ++-
>  builtin-debug.c   |   8 +-
>  builtin-run.c |   2 +
>  fdt.c |  35 ++
>  include/kvm/builtin-debug.h   |   6 +
>  include/kvm/devices.h |   4 +
>  include/kvm/fdt.h |  20 +
>  include/kvm/iommu.h   | 105 +
>  include/kvm/irq.h |   3 +
>  include/kvm/kvm-config.h  |   1 +
>  include/kvm/vfio.h|   2 +
>  include/kvm/virtio-iommu.h|  15 +
>  include/kvm/virtio-mmio.h |   1 +
>  include/kvm/virtio-pci.h  |   2 +
>  include/kvm/virtio.h  | 137 +-
>  include/linux/virtio_config.h |  74 
>  include/linux/virtio_ids.h|   4 +
>  include/linux/virtio_iommu.h  | 142 ++
>  iommu.c   | 240 ++
>  irq.c |  35 ++
>  kvm-ipc.c |  43 +-
>  mips/include/kvm/fdt-arch.h   |   2 +-
>  powerpc/include/kvm/fdt-arch.h|   2 +-
>  vfio.c| 281 +++-
>  virtio/9p.c   |   7 +-
>