[PATCH 4/4] iommu/vt-d: Hooks to invalidate iotlb/devtlb when using supervisor PASID's.

[Ashok Raj]

When a kernel client uses intel_svm_bind_mm() and requests a supervisor
PASID, IOMMU needs to track changes to these addresses. Otherwise the device
tlb will be stale compared to what's on the cpu for kernel mappings. This
is similar to what's done for user space registrations via
mmu_notifier_register() api's.

To: linux-kernel@vger.kernel.org
To: Joerg Roedel 
Cc: Ashok Raj 
Cc: Dave Hansen 
Cc: Huang Ying 
Cc: CQ Tang 
Cc: Thomas Gleixner 
Cc: Ingo Molnar 
Cc: H. Peter Anvin 
Cc: Andy Lutomirski 
Cc: Rik van Riel 
Cc: Kees Cook 
Cc: Andrew Morton 
Cc: Michal Hocko 
Cc: "Paul E. McKenney" 
Cc: Vegard Nossum 
Cc: x...@kernel.org
Cc: linux...@kvack.org
Cc: io...@lists.linux-foundation.org
Cc: David Woodhouse 
CC: Jean-Phillipe Brucker 

Signed-off-by: Ashok Raj 
---
 drivers/iommu/intel-svm.c   | 29 +++--
 include/linux/intel-iommu.h |  5 -
 2 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c
index 0c9f077..1758814 100644
--- a/drivers/iommu/intel-svm.c
+++ b/drivers/iommu/intel-svm.c
@@ -292,6 +292,26 @@ static const struct mmu_notifier_ops intel_mmuops = {
 
 static DEFINE_MUTEX(pasid_mutex);
 
+static int intel_init_mm_inval_range(struct notifier_block *nb,
+   unsigned long action, void *data)
+{
+   struct kernel_mmu_address_range *range;
+   struct intel_svm *svm = container_of(nb, struct intel_svm, init_mm_nb);
+   unsigned long start, end;
+   struct intel_iommu *iommu;
+
+   if (action == KERNEL_MMU_INVALIDATE_RANGE) {
+   range = data;
+   start = range->start;
+   end = range->end;
+   iommu = svm->iommu;
+
+   intel_flush_svm_range(svm, start,
+   (end - start + PAGE_SIZE - 1) >> VTD_PAGE_SHIFT, 0, 0);
+   }
+   return 0;
+}
+
 int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct 
svm_dev_ops *ops)
 {
struct intel_iommu *iommu = intel_svm_device_to_iommu(dev);
@@ -391,12 +411,12 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int 
flags, struct svm_dev_
goto out;
}
svm->pasid = ret;
-   svm->notifier.ops = _mmuops;
svm->mm = mm;
svm->flags = flags;
INIT_LIST_HEAD_RCU(>devs);
ret = -ENOMEM;
if (mm) {
+   svm->notifier.ops = _mmuops;
ret = mmu_notifier_register(>notifier, mm);
if (ret) {
idr_remove(>iommu->pasid_idr, svm->pasid);
@@ -405,8 +425,11 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int 
flags, struct svm_dev_
goto out;
}
iommu->pasid_table[svm->pasid].val = (u64)__pa(mm->pgd) 
| 1;
-   } else
+   } else {
+   svm->init_mm_nb.notifier_call = 
intel_init_mm_inval_range;
+   kernel_mmu_notifier_register(>init_mm_nb);
iommu->pasid_table[svm->pasid].val = 
(u64)__pa(init_mm.pgd) | 1 | (1ULL << 11);
+   }
wmb();
/* In caching mode, we still have to flush with PASID 0 when
 * a PASID table entry becomes present. Not entirely clear
@@ -471,6 +494,8 @@ int intel_svm_unbind_mm(struct device *dev, int pasid)
idr_remove(>iommu->pasid_idr, 
svm->pasid);
if (svm->mm)

mmu_notifier_unregister(>notifier, svm->mm);
+   else
+   
kernel_mmu_notifier_unregister(>init_mm_nb);
 
/* We mandate that no page faults may 
be outstanding
 * for the PASID when 
intel_svm_unbind_mm() is called.
diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h
index 485a5b4..d6019b4 100644
--- a/include/linux/intel-iommu.h
+++ b/include/linux/intel-iommu.h
@@ -477,7 +477,10 @@ struct intel_svm_dev {
 };
 
 struct intel_svm {
-   struct mmu_notifier notifier;
+   union {
+   struct mmu_notifier notifier;
+   struct notifier_block init_mm_nb;
+   };
struct mm_struct *mm;
struct intel_iommu *iommu;
int flags;
-- 
2.7.4

[PATCH 4/4] iommu/vt-d: Hooks to invalidate iotlb/devtlb when using supervisor PASID's.

[Ashok Raj]

When a kernel client uses intel_svm_bind_mm() and requests a supervisor
PASID, IOMMU needs to track changes to these addresses. Otherwise the device
tlb will be stale compared to what's on the cpu for kernel mappings. This
is similar to what's done for user space registrations via
mmu_notifier_register() api's.

To: linux-kernel@vger.kernel.org
To: Joerg Roedel 
Cc: Ashok Raj 
Cc: Dave Hansen 
Cc: Huang Ying 
Cc: CQ Tang 
Cc: Thomas Gleixner 
Cc: Ingo Molnar 
Cc: H. Peter Anvin 
Cc: Andy Lutomirski 
Cc: Rik van Riel 
Cc: Kees Cook 
Cc: Andrew Morton 
Cc: Michal Hocko 
Cc: "Paul E. McKenney" 
Cc: Vegard Nossum 
Cc: x...@kernel.org
Cc: linux...@kvack.org
Cc: io...@lists.linux-foundation.org
Cc: David Woodhouse 
CC: Jean-Phillipe Brucker 

Signed-off-by: Ashok Raj 
---
 drivers/iommu/intel-svm.c   | 29 +++--
 include/linux/intel-iommu.h |  5 -
 2 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c
index 0c9f077..1758814 100644
--- a/drivers/iommu/intel-svm.c
+++ b/drivers/iommu/intel-svm.c
@@ -292,6 +292,26 @@ static const struct mmu_notifier_ops intel_mmuops = {
 
 static DEFINE_MUTEX(pasid_mutex);
 
+static int intel_init_mm_inval_range(struct notifier_block *nb,
+   unsigned long action, void *data)
+{
+   struct kernel_mmu_address_range *range;
+   struct intel_svm *svm = container_of(nb, struct intel_svm, init_mm_nb);
+   unsigned long start, end;
+   struct intel_iommu *iommu;
+
+   if (action == KERNEL_MMU_INVALIDATE_RANGE) {
+   range = data;
+   start = range->start;
+   end = range->end;
+   iommu = svm->iommu;
+
+   intel_flush_svm_range(svm, start,
+   (end - start + PAGE_SIZE - 1) >> VTD_PAGE_SHIFT, 0, 0);
+   }
+   return 0;
+}
+
 int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct 
svm_dev_ops *ops)
 {
struct intel_iommu *iommu = intel_svm_device_to_iommu(dev);
@@ -391,12 +411,12 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int 
flags, struct svm_dev_
goto out;
}
svm->pasid = ret;
-   svm->notifier.ops = _mmuops;
svm->mm = mm;
svm->flags = flags;
INIT_LIST_HEAD_RCU(>devs);
ret = -ENOMEM;
if (mm) {
+   svm->notifier.ops = _mmuops;
ret = mmu_notifier_register(>notifier, mm);
if (ret) {
idr_remove(>iommu->pasid_idr, svm->pasid);
@@ -405,8 +425,11 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int 
flags, struct svm_dev_
goto out;
}
iommu->pasid_table[svm->pasid].val = (u64)__pa(mm->pgd) 
| 1;
-   } else
+   } else {
+   svm->init_mm_nb.notifier_call = 
intel_init_mm_inval_range;
+   kernel_mmu_notifier_register(>init_mm_nb);
iommu->pasid_table[svm->pasid].val = 
(u64)__pa(init_mm.pgd) | 1 | (1ULL << 11);
+   }
wmb();
/* In caching mode, we still have to flush with PASID 0 when
 * a PASID table entry becomes present. Not entirely clear
@@ -471,6 +494,8 @@ int intel_svm_unbind_mm(struct device *dev, int pasid)
idr_remove(>iommu->pasid_idr, 
svm->pasid);
if (svm->mm)

mmu_notifier_unregister(>notifier, svm->mm);
+   else
+   
kernel_mmu_notifier_unregister(>init_mm_nb);
 
/* We mandate that no page faults may 
be outstanding
 * for the PASID when 
intel_svm_unbind_mm() is called.
diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h
index 485a5b4..d6019b4 100644
--- a/include/linux/intel-iommu.h
+++ b/include/linux/intel-iommu.h
@@ -477,7 +477,10 @@ struct intel_svm_dev {
 };
 
 struct intel_svm {
-   struct mmu_notifier notifier;
+   union {
+   struct mmu_notifier notifier;
+   struct notifier_block init_mm_nb;
+   };
struct mm_struct *mm;
struct intel_iommu *iommu;
int flags;
-- 
2.7.4

[PATCH 4/4] iommu/vt-d: Hooks to invalidate iotlb/devtlb when using supervisor PASID's.

[Ashok Raj]

When a kernel client uses intel_svm_bind_mm() and requests a supervisor
PASID, IOMMU needs to track changes to these addresses. Otherwise the device
tlb will be stale compared to what's on the cpu for kernel mappings. This
is similar to what's done for user space registrations via
mmu_notifier_register() api's.

To: linux-kernel@vger.kernel.org
To: Joerg Roedel 
Cc: Ashok Raj 
Cc: Dave Hansen 
Cc: Huang Ying 
Cc: CQ Tang 
Cc: Thomas Gleixner 
Cc: Ingo Molnar 
Cc: H. Peter Anvin 
Cc: Andy Lutomirski 
Cc: Rik van Riel 
Cc: Kees Cook 
Cc: Andrew Morton 
Cc: Michal Hocko 
Cc: "Paul E. McKenney" 
Cc: Vegard Nossum 
Cc: x...@kernel.org
Cc: linux...@kvack.org
Cc: io...@lists-foundation.org
Cc: David Woodhouse 
CC: Jean-Phillipe Brucker 

Signed-off-by: Ashok Raj 
---
 drivers/iommu/intel-svm.c   | 29 +++--
 include/linux/intel-iommu.h |  5 -
 2 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c
index 0c9f077..1758814 100644
--- a/drivers/iommu/intel-svm.c
+++ b/drivers/iommu/intel-svm.c
@@ -292,6 +292,26 @@ static const struct mmu_notifier_ops intel_mmuops = {
 
 static DEFINE_MUTEX(pasid_mutex);
 
+static int intel_init_mm_inval_range(struct notifier_block *nb,
+   unsigned long action, void *data)
+{
+   struct kernel_mmu_address_range *range;
+   struct intel_svm *svm = container_of(nb, struct intel_svm, init_mm_nb);
+   unsigned long start, end;
+   struct intel_iommu *iommu;
+
+   if (action == KERNEL_MMU_INVALIDATE_RANGE) {
+   range = data;
+   start = range->start;
+   end = range->end;
+   iommu = svm->iommu;
+
+   intel_flush_svm_range(svm, start,
+   (end - start + PAGE_SIZE - 1) >> VTD_PAGE_SHIFT, 0, 0);
+   }
+   return 0;
+}
+
 int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct 
svm_dev_ops *ops)
 {
struct intel_iommu *iommu = intel_svm_device_to_iommu(dev);
@@ -391,12 +411,12 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int 
flags, struct svm_dev_
goto out;
}
svm->pasid = ret;
-   svm->notifier.ops = _mmuops;
svm->mm = mm;
svm->flags = flags;
INIT_LIST_HEAD_RCU(>devs);
ret = -ENOMEM;
if (mm) {
+   svm->notifier.ops = _mmuops;
ret = mmu_notifier_register(>notifier, mm);
if (ret) {
idr_remove(>iommu->pasid_idr, svm->pasid);
@@ -405,8 +425,11 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int 
flags, struct svm_dev_
goto out;
}
iommu->pasid_table[svm->pasid].val = (u64)__pa(mm->pgd) 
| 1;
-   } else
+   } else {
+   svm->init_mm_nb.notifier_call = 
intel_init_mm_inval_range;
+   kernel_mmu_notifier_register(>init_mm_nb);
iommu->pasid_table[svm->pasid].val = 
(u64)__pa(init_mm.pgd) | 1 | (1ULL << 11);
+   }
wmb();
/* In caching mode, we still have to flush with PASID 0 when
 * a PASID table entry becomes present. Not entirely clear
@@ -471,6 +494,8 @@ int intel_svm_unbind_mm(struct device *dev, int pasid)
idr_remove(>iommu->pasid_idr, 
svm->pasid);
if (svm->mm)

mmu_notifier_unregister(>notifier, svm->mm);
+   else
+   
kernel_mmu_notifier_unregister(>init_mm_nb);
 
/* We mandate that no page faults may 
be outstanding
 * for the PASID when 
intel_svm_unbind_mm() is called.
diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h
index 485a5b4..d6019b4 100644
--- a/include/linux/intel-iommu.h
+++ b/include/linux/intel-iommu.h
@@ -477,7 +477,10 @@ struct intel_svm_dev {
 };
 
 struct intel_svm {
-   struct mmu_notifier notifier;
+   union {
+   struct mmu_notifier notifier;
+   struct notifier_block init_mm_nb;
+   };
struct mm_struct *mm;
struct intel_iommu *iommu;
int flags;
-- 
2.7.4

[PATCH 4/4] iommu/vt-d: Hooks to invalidate iotlb/devtlb when using supervisor PASID's.

[Ashok Raj]

When a kernel client uses intel_svm_bind_mm() and requests a supervisor
PASID, IOMMU needs to track changes to these addresses. Otherwise the device
tlb will be stale compared to what's on the cpu for kernel mappings. This
is similar to what's done for user space registrations via
mmu_notifier_register() api's.

To: linux-kernel@vger.kernel.org
To: Joerg Roedel 
Cc: Ashok Raj 
Cc: Dave Hansen 
Cc: Huang Ying 
Cc: CQ Tang 
Cc: Thomas Gleixner 
Cc: Ingo Molnar 
Cc: H. Peter Anvin 
Cc: Andy Lutomirski 
Cc: Rik van Riel 
Cc: Kees Cook 
Cc: Andrew Morton 
Cc: Michal Hocko 
Cc: "Paul E. McKenney" 
Cc: Vegard Nossum 
Cc: x...@kernel.org
Cc: linux...@kvack.org
Cc: io...@lists-foundation.org
Cc: David Woodhouse 
CC: Jean-Phillipe Brucker 

Signed-off-by: Ashok Raj 
---
 drivers/iommu/intel-svm.c   | 29 +++--
 include/linux/intel-iommu.h |  5 -
 2 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c
index 0c9f077..1758814 100644
--- a/drivers/iommu/intel-svm.c
+++ b/drivers/iommu/intel-svm.c
@@ -292,6 +292,26 @@ static const struct mmu_notifier_ops intel_mmuops = {
 
 static DEFINE_MUTEX(pasid_mutex);
 
+static int intel_init_mm_inval_range(struct notifier_block *nb,
+   unsigned long action, void *data)
+{
+   struct kernel_mmu_address_range *range;
+   struct intel_svm *svm = container_of(nb, struct intel_svm, init_mm_nb);
+   unsigned long start, end;
+   struct intel_iommu *iommu;
+
+   if (action == KERNEL_MMU_INVALIDATE_RANGE) {
+   range = data;
+   start = range->start;
+   end = range->end;
+   iommu = svm->iommu;
+
+   intel_flush_svm_range(svm, start,
+   (end - start + PAGE_SIZE - 1) >> VTD_PAGE_SHIFT, 0, 0);
+   }
+   return 0;
+}
+
 int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct 
svm_dev_ops *ops)
 {
struct intel_iommu *iommu = intel_svm_device_to_iommu(dev);
@@ -391,12 +411,12 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int 
flags, struct svm_dev_
goto out;
}
svm->pasid = ret;
-   svm->notifier.ops = _mmuops;
svm->mm = mm;
svm->flags = flags;
INIT_LIST_HEAD_RCU(>devs);
ret = -ENOMEM;
if (mm) {
+   svm->notifier.ops = _mmuops;
ret = mmu_notifier_register(>notifier, mm);
if (ret) {
idr_remove(>iommu->pasid_idr, svm->pasid);
@@ -405,8 +425,11 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int 
flags, struct svm_dev_
goto out;
}
iommu->pasid_table[svm->pasid].val = (u64)__pa(mm->pgd) 
| 1;
-   } else
+   } else {
+   svm->init_mm_nb.notifier_call = 
intel_init_mm_inval_range;
+   kernel_mmu_notifier_register(>init_mm_nb);
iommu->pasid_table[svm->pasid].val = 
(u64)__pa(init_mm.pgd) | 1 | (1ULL << 11);
+   }
wmb();
/* In caching mode, we still have to flush with PASID 0 when
 * a PASID table entry becomes present. Not entirely clear
@@ -471,6 +494,8 @@ int intel_svm_unbind_mm(struct device *dev, int pasid)
idr_remove(>iommu->pasid_idr, 
svm->pasid);
if (svm->mm)

mmu_notifier_unregister(>notifier, svm->mm);
+   else
+   
kernel_mmu_notifier_unregister(>init_mm_nb);
 
/* We mandate that no page faults may 
be outstanding
 * for the PASID when 
intel_svm_unbind_mm() is called.
diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h
index 485a5b4..d6019b4 100644
--- a/include/linux/intel-iommu.h
+++ b/include/linux/intel-iommu.h
@@ -477,7 +477,10 @@ struct intel_svm_dev {
 };
 
 struct intel_svm {
-   struct mmu_notifier notifier;
+   union {
+   struct mmu_notifier notifier;
+   struct notifier_block init_mm_nb;
+   };
struct mm_struct *mm;
struct intel_iommu *iommu;
int flags;
-- 
2.7.4