Preparing to migrate to use IO page table framework.
There is no functional change.

Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpa...@amd.com>
---
 drivers/iommu/amd/amd_iommu.h  |  18 ++
 drivers/iommu/amd/io_pgtable.c | 473 ++++++++++++++++++++++++++++++++
 drivers/iommu/amd/iommu.c      | 476 +--------------------------------
 3 files changed, 493 insertions(+), 474 deletions(-)

diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h
index 8b7be9171030..ee7ff4d827e1 100644
--- a/drivers/iommu/amd/amd_iommu.h
+++ b/drivers/iommu/amd/amd_iommu.h
@@ -122,4 +122,22 @@ void amd_iommu_apply_ivrs_quirks(void);
 static inline void amd_iommu_apply_ivrs_quirks(void) { }
 #endif
 
+/* TODO: These are temporary and will be removed once fully transition */
+extern void free_pagetable(struct domain_pgtable *pgtable);
+extern int iommu_map_page(struct protection_domain *dom,
+                         unsigned long bus_addr,
+                         unsigned long phys_addr,
+                         unsigned long page_size,
+                         int prot,
+                         gfp_t gfp);
+extern unsigned long iommu_unmap_page(struct protection_domain *dom,
+                                     unsigned long bus_addr,
+                                     unsigned long page_size);
+extern u64 *fetch_pte(struct protection_domain *domain,
+                     unsigned long address,
+                     unsigned long *page_size);
+extern void amd_iommu_domain_get_pgtable(struct protection_domain *domain,
+                                        struct domain_pgtable *pgtable);
+extern void amd_iommu_domain_set_pgtable(struct protection_domain *domain,
+                                        u64 *root, int mode);
 #endif
diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c
index 452cad26a2b3..7e4154e26757 100644
--- a/drivers/iommu/amd/io_pgtable.c
+++ b/drivers/iommu/amd/io_pgtable.c
@@ -54,6 +54,479 @@ static const struct iommu_flush_ops amd_flush_ops = {
        .tlb_add_page   = v1_tlb_add_page,
 };
 
+/*
+ * Helper function to get the first pte of a large mapping
+ */
+static u64 *first_pte_l7(u64 *pte, unsigned long *page_size,
+                        unsigned long *count)
+{
+       unsigned long pte_mask, pg_size, cnt;
+       u64 *fpte;
+
+       pg_size  = PTE_PAGE_SIZE(*pte);
+       cnt      = PAGE_SIZE_PTE_COUNT(pg_size);
+       pte_mask = ~((cnt << 3) - 1);
+       fpte     = (u64 *)(((unsigned long)pte) & pte_mask);
+
+       if (page_size)
+               *page_size = pg_size;
+
+       if (count)
+               *count = cnt;
+
+       return fpte;
+}
+
+/****************************************************************************
+ *
+ * The functions below are used the create the page table mappings for
+ * unity mapped regions.
+ *
+ ****************************************************************************/
+
+static void free_page_list(struct page *freelist)
+{
+       while (freelist != NULL) {
+               unsigned long p = (unsigned long)page_address(freelist);
+
+               freelist = freelist->freelist;
+               free_page(p);
+       }
+}
+
+static struct page *free_pt_page(unsigned long pt, struct page *freelist)
+{
+       struct page *p = virt_to_page((void *)pt);
+
+       p->freelist = freelist;
+
+       return p;
+}
+
+#define DEFINE_FREE_PT_FN(LVL, FN)                                             
\
+static struct page *free_pt_##LVL (unsigned long __pt, struct page *freelist)  
\
+{                                                                              
\
+       unsigned long p;                                                        
\
+       u64 *pt;                                                                
\
+       int i;                                                                  
\
+                                                                               
\
+       pt = (u64 *)__pt;                                                       
\
+                                                                               
\
+       for (i = 0; i < 512; ++i) {                                             
\
+               /* PTE present? */                                              
\
+               if (!IOMMU_PTE_PRESENT(pt[i]))                                  
\
+                       continue;                                               
\
+                                                                               
\
+               /* Large PTE? */                                                
\
+               if (PM_PTE_LEVEL(pt[i]) == 0 ||                                 
\
+                   PM_PTE_LEVEL(pt[i]) == 7)                                   
\
+                       continue;                                               
\
+                                                                               
\
+               p = (unsigned long)IOMMU_PTE_PAGE(pt[i]);                       
\
+               freelist = FN(p, freelist);                                     
\
+       }                                                                       
\
+                                                                               
\
+       return free_pt_page((unsigned long)pt, freelist);                       
\
+}
+
+DEFINE_FREE_PT_FN(l2, free_pt_page)
+DEFINE_FREE_PT_FN(l3, free_pt_l2)
+DEFINE_FREE_PT_FN(l4, free_pt_l3)
+DEFINE_FREE_PT_FN(l5, free_pt_l4)
+DEFINE_FREE_PT_FN(l6, free_pt_l5)
+
+static struct page *free_sub_pt(unsigned long root, int mode,
+                               struct page *freelist)
+{
+       switch (mode) {
+       case PAGE_MODE_NONE:
+       case PAGE_MODE_7_LEVEL:
+               break;
+       case PAGE_MODE_1_LEVEL:
+               freelist = free_pt_page(root, freelist);
+               break;
+       case PAGE_MODE_2_LEVEL:
+               freelist = free_pt_l2(root, freelist);
+               break;
+       case PAGE_MODE_3_LEVEL:
+               freelist = free_pt_l3(root, freelist);
+               break;
+       case PAGE_MODE_4_LEVEL:
+               freelist = free_pt_l4(root, freelist);
+               break;
+       case PAGE_MODE_5_LEVEL:
+               freelist = free_pt_l5(root, freelist);
+               break;
+       case PAGE_MODE_6_LEVEL:
+               freelist = free_pt_l6(root, freelist);
+               break;
+       default:
+               BUG();
+       }
+
+       return freelist;
+}
+
+void free_pagetable(struct domain_pgtable *pgtable)
+{
+       struct page *freelist = NULL;
+       unsigned long root;
+
+       if (pgtable->mode == PAGE_MODE_NONE)
+               return;
+
+       BUG_ON(pgtable->mode < PAGE_MODE_NONE ||
+              pgtable->mode > PAGE_MODE_6_LEVEL);
+
+       root = (unsigned long)pgtable->root;
+       freelist = free_sub_pt(root, pgtable->mode, freelist);
+
+       free_page_list(freelist);
+}
+
+void amd_iommu_domain_set_pgtable(struct protection_domain *domain,
+                                 u64 *root, int mode)
+{
+       u64 pt_root;
+
+       /* lowest 3 bits encode pgtable mode */
+       pt_root = mode & 7;
+       pt_root |= (u64)root;
+
+       amd_iommu_domain_set_pt_root(domain, pt_root);
+}
+
+/*
+ * This function is used to add another level to an IO page table. Adding
+ * another level increases the size of the address space by 9 bits to a size up
+ * to 64 bits.
+ */
+static bool increase_address_space(struct protection_domain *domain,
+                                  unsigned long address,
+                                  gfp_t gfp)
+{
+       struct domain_pgtable pgtable;
+       unsigned long flags;
+       bool ret = true;
+       u64 *pte;
+
+       spin_lock_irqsave(&domain->lock, flags);
+
+       amd_iommu_domain_get_pgtable(domain, &pgtable);
+
+       if (address <= PM_LEVEL_SIZE(pgtable.mode))
+               goto out;
+
+       ret = false;
+       if (WARN_ON_ONCE(pgtable.mode == PAGE_MODE_6_LEVEL))
+               goto out;
+
+       pte = (void *)get_zeroed_page(gfp);
+       if (!pte)
+               goto out;
+
+       *pte = PM_LEVEL_PDE(pgtable.mode, iommu_virt_to_phys(pgtable.root));
+
+       pgtable.root  = pte;
+       pgtable.mode += 1;
+       amd_iommu_update_and_flush_device_table(domain);
+       amd_iommu_domain_flush_complete(domain);
+
+       /*
+        * Device Table needs to be updated and flushed before the new root can
+        * be published.
+        */
+       amd_iommu_domain_set_pgtable(domain, pte, pgtable.mode);
+
+       ret = true;
+
+out:
+       spin_unlock_irqrestore(&domain->lock, flags);
+
+       return ret;
+}
+
+static u64 *alloc_pte(struct protection_domain *domain,
+                     unsigned long address,
+                     unsigned long page_size,
+                     u64 **pte_page,
+                     gfp_t gfp,
+                     bool *updated)
+{
+       struct domain_pgtable pgtable;
+       int level, end_lvl;
+       u64 *pte, *page;
+
+       BUG_ON(!is_power_of_2(page_size));
+
+       amd_iommu_domain_get_pgtable(domain, &pgtable);
+
+       while (address > PM_LEVEL_SIZE(pgtable.mode)) {
+               /*
+                * Return an error if there is no memory to update the
+                * page-table.
+                */
+               if (!increase_address_space(domain, address, gfp))
+                       return NULL;
+
+               /* Read new values to check if update was successful */
+               amd_iommu_domain_get_pgtable(domain, &pgtable);
+       }
+
+
+       level   = pgtable.mode - 1;
+       pte     = &pgtable.root[PM_LEVEL_INDEX(level, address)];
+       address = PAGE_SIZE_ALIGN(address, page_size);
+       end_lvl = PAGE_SIZE_LEVEL(page_size);
+
+       while (level > end_lvl) {
+               u64 __pte, __npte;
+               int pte_level;
+
+               __pte     = *pte;
+               pte_level = PM_PTE_LEVEL(__pte);
+
+               /*
+                * If we replace a series of large PTEs, we need
+                * to tear down all of them.
+                */
+               if (IOMMU_PTE_PRESENT(__pte) &&
+                   pte_level == PAGE_MODE_7_LEVEL) {
+                       unsigned long count, i;
+                       u64 *lpte;
+
+                       lpte = first_pte_l7(pte, NULL, &count);
+
+                       /*
+                        * Unmap the replicated PTEs that still match the
+                        * original large mapping
+                        */
+                       for (i = 0; i < count; ++i)
+                               cmpxchg64(&lpte[i], __pte, 0ULL);
+
+                       *updated = true;
+                       continue;
+               }
+
+               if (!IOMMU_PTE_PRESENT(__pte) ||
+                   pte_level == PAGE_MODE_NONE) {
+                       page = (u64 *)get_zeroed_page(gfp);
+
+                       if (!page)
+                               return NULL;
+
+                       __npte = PM_LEVEL_PDE(level, iommu_virt_to_phys(page));
+
+                       /* pte could have been changed somewhere. */
+                       if (cmpxchg64(pte, __pte, __npte) != __pte)
+                               free_page((unsigned long)page);
+                       else if (IOMMU_PTE_PRESENT(__pte))
+                               *updated = true;
+
+                       continue;
+               }
+
+               /* No level skipping support yet */
+               if (pte_level != level)
+                       return NULL;
+
+               level -= 1;
+
+               pte = IOMMU_PTE_PAGE(__pte);
+
+               if (pte_page && level == end_lvl)
+                       *pte_page = pte;
+
+               pte = &pte[PM_LEVEL_INDEX(level, address)];
+       }
+
+       return pte;
+}
+
+/*
+ * This function checks if there is a PTE for a given dma address. If
+ * there is one, it returns the pointer to it.
+ */
+u64 *fetch_pte(struct protection_domain *domain,
+              unsigned long address,
+              unsigned long *page_size)
+{
+       struct domain_pgtable pgtable;
+       int level;
+       u64 *pte;
+
+       *page_size = 0;
+
+       amd_iommu_domain_get_pgtable(domain, &pgtable);
+
+       if (address > PM_LEVEL_SIZE(pgtable.mode))
+               return NULL;
+
+       level      =  pgtable.mode - 1;
+       pte        = &pgtable.root[PM_LEVEL_INDEX(level, address)];
+       *page_size =  PTE_LEVEL_PAGE_SIZE(level);
+
+       while (level > 0) {
+
+               /* Not Present */
+               if (!IOMMU_PTE_PRESENT(*pte))
+                       return NULL;
+
+               /* Large PTE */
+               if (PM_PTE_LEVEL(*pte) == 7 ||
+                   PM_PTE_LEVEL(*pte) == 0)
+                       break;
+
+               /* No level skipping support yet */
+               if (PM_PTE_LEVEL(*pte) != level)
+                       return NULL;
+
+               level -= 1;
+
+               /* Walk to the next level */
+               pte        = IOMMU_PTE_PAGE(*pte);
+               pte        = &pte[PM_LEVEL_INDEX(level, address)];
+               *page_size = PTE_LEVEL_PAGE_SIZE(level);
+       }
+
+       /*
+        * If we have a series of large PTEs, make
+        * sure to return a pointer to the first one.
+        */
+       if (PM_PTE_LEVEL(*pte) == PAGE_MODE_7_LEVEL)
+               pte = first_pte_l7(pte, page_size, NULL);
+
+       return pte;
+}
+
+static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist)
+{
+       unsigned long pt;
+       int mode;
+
+       while (cmpxchg64(pte, pteval, 0) != pteval) {
+               pr_warn("AMD-Vi: IOMMU pte changed since we read it\n");
+               pteval = *pte;
+       }
+
+       if (!IOMMU_PTE_PRESENT(pteval))
+               return freelist;
+
+       pt   = (unsigned long)IOMMU_PTE_PAGE(pteval);
+       mode = IOMMU_PTE_MODE(pteval);
+
+       return free_sub_pt(pt, mode, freelist);
+}
+
+/*
+ * Generic mapping functions. It maps a physical address into a DMA
+ * address space. It allocates the page table pages if necessary.
+ * In the future it can be extended to a generic mapping function
+ * supporting all features of AMD IOMMU page tables like level skipping
+ * and full 64 bit address spaces.
+ */
+int iommu_map_page(struct protection_domain *dom,
+                  unsigned long bus_addr,
+                  unsigned long phys_addr,
+                  unsigned long page_size,
+                  int prot,
+                  gfp_t gfp)
+{
+       struct page *freelist = NULL;
+       bool updated = false;
+       u64 __pte, *pte;
+       int ret, i, count;
+
+       BUG_ON(!IS_ALIGNED(bus_addr, page_size));
+       BUG_ON(!IS_ALIGNED(phys_addr, page_size));
+
+       ret = -EINVAL;
+       if (!(prot & IOMMU_PROT_MASK))
+               goto out;
+
+       count = PAGE_SIZE_PTE_COUNT(page_size);
+       pte   = alloc_pte(dom, bus_addr, page_size, NULL, gfp, &updated);
+
+       ret = -ENOMEM;
+       if (!pte)
+               goto out;
+
+       for (i = 0; i < count; ++i)
+               freelist = free_clear_pte(&pte[i], pte[i], freelist);
+
+       if (freelist != NULL)
+               updated = true;
+
+       if (count > 1) {
+               __pte = PAGE_SIZE_PTE(__sme_set(phys_addr), page_size);
+               __pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_PR | IOMMU_PTE_FC;
+       } else
+               __pte = __sme_set(phys_addr) | IOMMU_PTE_PR | IOMMU_PTE_FC;
+
+       if (prot & IOMMU_PROT_IR)
+               __pte |= IOMMU_PTE_IR;
+       if (prot & IOMMU_PROT_IW)
+               __pte |= IOMMU_PTE_IW;
+
+       for (i = 0; i < count; ++i)
+               pte[i] = __pte;
+
+       ret = 0;
+
+out:
+       if (updated) {
+               unsigned long flags;
+
+               spin_lock_irqsave(&dom->lock, flags);
+               /*
+                * Flush domain TLB(s) and wait for completion. Any Device-Table
+                * Updates and flushing already happened in
+                * increase_address_space().
+                */
+               amd_iommu_domain_flush_tlb_pde(dom);
+               amd_iommu_domain_flush_complete(dom);
+               spin_unlock_irqrestore(&dom->lock, flags);
+       }
+
+       /* Everything flushed out, free pages now */
+       free_page_list(freelist);
+
+       return ret;
+}
+
+unsigned long iommu_unmap_page(struct protection_domain *dom,
+                              unsigned long bus_addr,
+                              unsigned long page_size)
+{
+       unsigned long long unmapped;
+       unsigned long unmap_size;
+       u64 *pte;
+
+       BUG_ON(!is_power_of_2(page_size));
+
+       unmapped = 0;
+
+       while (unmapped < page_size) {
+
+               pte = fetch_pte(dom, bus_addr, &unmap_size);
+
+               if (pte) {
+                       int i, count;
+
+                       count = PAGE_SIZE_PTE_COUNT(unmap_size);
+                       for (i = 0; i < count; i++)
+                               pte[i] = 0ULL;
+               }
+
+               bus_addr  = (bus_addr & ~(unmap_size - 1)) + unmap_size;
+               unmapped += unmap_size;
+       }
+
+       BUG_ON(unmapped && !is_power_of_2(unmapped));
+
+       return unmapped;
+}
+
 struct io_pgtable_ops *amd_iommu_setup_io_pgtable_ops(struct iommu_dev_data 
*dev_data,
                                             struct protection_domain *domain)
 {
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index f91f35edb7ba..4d65f64236b6 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -140,8 +140,8 @@ static struct protection_domain *to_pdomain(struct 
iommu_domain *dom)
        return container_of(dom, struct protection_domain, domain);
 }
 
-static void amd_iommu_domain_get_pgtable(struct protection_domain *domain,
-                                        struct domain_pgtable *pgtable)
+void amd_iommu_domain_get_pgtable(struct protection_domain *domain,
+                                 struct domain_pgtable *pgtable)
 {
        u64 pt_root = atomic64_read(&domain->iop.pt_root);
 
@@ -149,18 +149,6 @@ static void amd_iommu_domain_get_pgtable(struct 
protection_domain *domain,
        pgtable->mode = pt_root & 7; /* lowest 3 bits encode pgtable mode */
 }
 
-static void amd_iommu_domain_set_pgtable(struct protection_domain *domain,
-                                        u64 *root, int mode)
-{
-       u64 pt_root;
-
-       /* lowest 3 bits encode pgtable mode */
-       pt_root = mode & 7;
-       pt_root |= (u64)root;
-
-       amd_iommu_domain_set_pt_root(domain, pt_root);
-}
-
 static struct iommu_dev_data *alloc_dev_data(u16 devid)
 {
        struct iommu_dev_data *dev_data;
@@ -416,29 +404,6 @@ static void amd_iommu_uninit_device(struct device *dev)
         */
 }
 
-/*
- * Helper function to get the first pte of a large mapping
- */
-static u64 *first_pte_l7(u64 *pte, unsigned long *page_size,
-                        unsigned long *count)
-{
-       unsigned long pte_mask, pg_size, cnt;
-       u64 *fpte;
-
-       pg_size  = PTE_PAGE_SIZE(*pte);
-       cnt      = PAGE_SIZE_PTE_COUNT(pg_size);
-       pte_mask = ~((cnt << 3) - 1);
-       fpte     = (u64 *)(((unsigned long)pte) & pte_mask);
-
-       if (page_size)
-               *page_size = pg_size;
-
-       if (count)
-               *count = cnt;
-
-       return fpte;
-}
-
 /****************************************************************************
  *
  * Interrupt handling functions
@@ -1340,443 +1305,6 @@ static void domain_flush_devices(struct 
protection_domain *domain)
                device_flush_dte(dev_data);
 }
 
-/****************************************************************************
- *
- * The functions below are used the create the page table mappings for
- * unity mapped regions.
- *
- ****************************************************************************/
-
-static void free_page_list(struct page *freelist)
-{
-       while (freelist != NULL) {
-               unsigned long p = (unsigned long)page_address(freelist);
-               freelist = freelist->freelist;
-               free_page(p);
-       }
-}
-
-static struct page *free_pt_page(unsigned long pt, struct page *freelist)
-{
-       struct page *p = virt_to_page((void *)pt);
-
-       p->freelist = freelist;
-
-       return p;
-}
-
-#define DEFINE_FREE_PT_FN(LVL, FN)                                             
\
-static struct page *free_pt_##LVL (unsigned long __pt, struct page *freelist)  
\
-{                                                                              
\
-       unsigned long p;                                                        
\
-       u64 *pt;                                                                
\
-       int i;                                                                  
\
-                                                                               
\
-       pt = (u64 *)__pt;                                                       
\
-                                                                               
\
-       for (i = 0; i < 512; ++i) {                                             
\
-               /* PTE present? */                                              
\
-               if (!IOMMU_PTE_PRESENT(pt[i]))                                  
\
-                       continue;                                               
\
-                                                                               
\
-               /* Large PTE? */                                                
\
-               if (PM_PTE_LEVEL(pt[i]) == 0 ||                                 
\
-                   PM_PTE_LEVEL(pt[i]) == 7)                                   
\
-                       continue;                                               
\
-                                                                               
\
-               p = (unsigned long)IOMMU_PTE_PAGE(pt[i]);                       
\
-               freelist = FN(p, freelist);                                     
\
-       }                                                                       
\
-                                                                               
\
-       return free_pt_page((unsigned long)pt, freelist);                       
\
-}
-
-DEFINE_FREE_PT_FN(l2, free_pt_page)
-DEFINE_FREE_PT_FN(l3, free_pt_l2)
-DEFINE_FREE_PT_FN(l4, free_pt_l3)
-DEFINE_FREE_PT_FN(l5, free_pt_l4)
-DEFINE_FREE_PT_FN(l6, free_pt_l5)
-
-static struct page *free_sub_pt(unsigned long root, int mode,
-                               struct page *freelist)
-{
-       switch (mode) {
-       case PAGE_MODE_NONE:
-       case PAGE_MODE_7_LEVEL:
-               break;
-       case PAGE_MODE_1_LEVEL:
-               freelist = free_pt_page(root, freelist);
-               break;
-       case PAGE_MODE_2_LEVEL:
-               freelist = free_pt_l2(root, freelist);
-               break;
-       case PAGE_MODE_3_LEVEL:
-               freelist = free_pt_l3(root, freelist);
-               break;
-       case PAGE_MODE_4_LEVEL:
-               freelist = free_pt_l4(root, freelist);
-               break;
-       case PAGE_MODE_5_LEVEL:
-               freelist = free_pt_l5(root, freelist);
-               break;
-       case PAGE_MODE_6_LEVEL:
-               freelist = free_pt_l6(root, freelist);
-               break;
-       default:
-               BUG();
-       }
-
-       return freelist;
-}
-
-static void free_pagetable(struct domain_pgtable *pgtable)
-{
-       struct page *freelist = NULL;
-       unsigned long root;
-
-       if (pgtable->mode == PAGE_MODE_NONE)
-               return;
-
-       BUG_ON(pgtable->mode < PAGE_MODE_NONE ||
-              pgtable->mode > PAGE_MODE_6_LEVEL);
-
-       root = (unsigned long)pgtable->root;
-       freelist = free_sub_pt(root, pgtable->mode, freelist);
-
-       free_page_list(freelist);
-}
-
-/*
- * This function is used to add another level to an IO page table. Adding
- * another level increases the size of the address space by 9 bits to a size up
- * to 64 bits.
- */
-static bool increase_address_space(struct protection_domain *domain,
-                                  unsigned long address,
-                                  gfp_t gfp)
-{
-       struct domain_pgtable pgtable;
-       unsigned long flags;
-       bool ret = true;
-       u64 *pte;
-
-       spin_lock_irqsave(&domain->lock, flags);
-
-       amd_iommu_domain_get_pgtable(domain, &pgtable);
-
-       if (address <= PM_LEVEL_SIZE(pgtable.mode))
-               goto out;
-
-       ret = false;
-       if (WARN_ON_ONCE(pgtable.mode == PAGE_MODE_6_LEVEL))
-               goto out;
-
-       pte = (void *)get_zeroed_page(gfp);
-       if (!pte)
-               goto out;
-
-       *pte = PM_LEVEL_PDE(pgtable.mode, iommu_virt_to_phys(pgtable.root));
-
-       pgtable.root  = pte;
-       pgtable.mode += 1;
-       amd_iommu_update_and_flush_device_table(domain);
-       amd_iommu_domain_flush_complete(domain);
-
-       /*
-        * Device Table needs to be updated and flushed before the new root can
-        * be published.
-        */
-       amd_iommu_domain_set_pgtable(domain, pte, pgtable.mode);
-
-       ret = true;
-
-out:
-       spin_unlock_irqrestore(&domain->lock, flags);
-
-       return ret;
-}
-
-static u64 *alloc_pte(struct protection_domain *domain,
-                     unsigned long address,
-                     unsigned long page_size,
-                     u64 **pte_page,
-                     gfp_t gfp,
-                     bool *updated)
-{
-       struct domain_pgtable pgtable;
-       int level, end_lvl;
-       u64 *pte, *page;
-
-       BUG_ON(!is_power_of_2(page_size));
-
-       amd_iommu_domain_get_pgtable(domain, &pgtable);
-
-       while (address > PM_LEVEL_SIZE(pgtable.mode)) {
-               /*
-                * Return an error if there is no memory to update the
-                * page-table.
-                */
-               if (!increase_address_space(domain, address, gfp))
-                       return NULL;
-
-               /* Read new values to check if update was successful */
-               amd_iommu_domain_get_pgtable(domain, &pgtable);
-       }
-
-
-       level   = pgtable.mode - 1;
-       pte     = &pgtable.root[PM_LEVEL_INDEX(level, address)];
-       address = PAGE_SIZE_ALIGN(address, page_size);
-       end_lvl = PAGE_SIZE_LEVEL(page_size);
-
-       while (level > end_lvl) {
-               u64 __pte, __npte;
-               int pte_level;
-
-               __pte     = *pte;
-               pte_level = PM_PTE_LEVEL(__pte);
-
-               /*
-                * If we replace a series of large PTEs, we need
-                * to tear down all of them.
-                */
-               if (IOMMU_PTE_PRESENT(__pte) &&
-                   pte_level == PAGE_MODE_7_LEVEL) {
-                       unsigned long count, i;
-                       u64 *lpte;
-
-                       lpte = first_pte_l7(pte, NULL, &count);
-
-                       /*
-                        * Unmap the replicated PTEs that still match the
-                        * original large mapping
-                        */
-                       for (i = 0; i < count; ++i)
-                               cmpxchg64(&lpte[i], __pte, 0ULL);
-
-                       *updated = true;
-                       continue;
-               }
-
-               if (!IOMMU_PTE_PRESENT(__pte) ||
-                   pte_level == PAGE_MODE_NONE) {
-                       page = (u64 *)get_zeroed_page(gfp);
-
-                       if (!page)
-                               return NULL;
-
-                       __npte = PM_LEVEL_PDE(level, iommu_virt_to_phys(page));
-
-                       /* pte could have been changed somewhere. */
-                       if (cmpxchg64(pte, __pte, __npte) != __pte)
-                               free_page((unsigned long)page);
-                       else if (IOMMU_PTE_PRESENT(__pte))
-                               *updated = true;
-
-                       continue;
-               }
-
-               /* No level skipping support yet */
-               if (pte_level != level)
-                       return NULL;
-
-               level -= 1;
-
-               pte = IOMMU_PTE_PAGE(__pte);
-
-               if (pte_page && level == end_lvl)
-                       *pte_page = pte;
-
-               pte = &pte[PM_LEVEL_INDEX(level, address)];
-       }
-
-       return pte;
-}
-
-/*
- * This function checks if there is a PTE for a given dma address. If
- * there is one, it returns the pointer to it.
- */
-static u64 *fetch_pte(struct protection_domain *domain,
-                     unsigned long address,
-                     unsigned long *page_size)
-{
-       struct domain_pgtable pgtable;
-       int level;
-       u64 *pte;
-
-       *page_size = 0;
-
-       amd_iommu_domain_get_pgtable(domain, &pgtable);
-
-       if (address > PM_LEVEL_SIZE(pgtable.mode))
-               return NULL;
-
-       level      =  pgtable.mode - 1;
-       pte        = &pgtable.root[PM_LEVEL_INDEX(level, address)];
-       *page_size =  PTE_LEVEL_PAGE_SIZE(level);
-
-       while (level > 0) {
-
-               /* Not Present */
-               if (!IOMMU_PTE_PRESENT(*pte))
-                       return NULL;
-
-               /* Large PTE */
-               if (PM_PTE_LEVEL(*pte) == 7 ||
-                   PM_PTE_LEVEL(*pte) == 0)
-                       break;
-
-               /* No level skipping support yet */
-               if (PM_PTE_LEVEL(*pte) != level)
-                       return NULL;
-
-               level -= 1;
-
-               /* Walk to the next level */
-               pte        = IOMMU_PTE_PAGE(*pte);
-               pte        = &pte[PM_LEVEL_INDEX(level, address)];
-               *page_size = PTE_LEVEL_PAGE_SIZE(level);
-       }
-
-       /*
-        * If we have a series of large PTEs, make
-        * sure to return a pointer to the first one.
-        */
-       if (PM_PTE_LEVEL(*pte) == PAGE_MODE_7_LEVEL)
-               pte = first_pte_l7(pte, page_size, NULL);
-
-       return pte;
-}
-
-static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist)
-{
-       unsigned long pt;
-       int mode;
-
-       while (cmpxchg64(pte, pteval, 0) != pteval) {
-               pr_warn("AMD-Vi: IOMMU pte changed since we read it\n");
-               pteval = *pte;
-       }
-
-       if (!IOMMU_PTE_PRESENT(pteval))
-               return freelist;
-
-       pt   = (unsigned long)IOMMU_PTE_PAGE(pteval);
-       mode = IOMMU_PTE_MODE(pteval);
-
-       return free_sub_pt(pt, mode, freelist);
-}
-
-/*
- * Generic mapping functions. It maps a physical address into a DMA
- * address space. It allocates the page table pages if necessary.
- * In the future it can be extended to a generic mapping function
- * supporting all features of AMD IOMMU page tables like level skipping
- * and full 64 bit address spaces.
- */
-static int iommu_map_page(struct protection_domain *dom,
-                         unsigned long bus_addr,
-                         unsigned long phys_addr,
-                         unsigned long page_size,
-                         int prot,
-                         gfp_t gfp)
-{
-       struct page *freelist = NULL;
-       bool updated = false;
-       u64 __pte, *pte;
-       int ret, i, count;
-
-       BUG_ON(!IS_ALIGNED(bus_addr, page_size));
-       BUG_ON(!IS_ALIGNED(phys_addr, page_size));
-
-       ret = -EINVAL;
-       if (!(prot & IOMMU_PROT_MASK))
-               goto out;
-
-       count = PAGE_SIZE_PTE_COUNT(page_size);
-       pte   = alloc_pte(dom, bus_addr, page_size, NULL, gfp, &updated);
-
-       ret = -ENOMEM;
-       if (!pte)
-               goto out;
-
-       for (i = 0; i < count; ++i)
-               freelist = free_clear_pte(&pte[i], pte[i], freelist);
-
-       if (freelist != NULL)
-               updated = true;
-
-       if (count > 1) {
-               __pte = PAGE_SIZE_PTE(__sme_set(phys_addr), page_size);
-               __pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_PR | IOMMU_PTE_FC;
-       } else
-               __pte = __sme_set(phys_addr) | IOMMU_PTE_PR | IOMMU_PTE_FC;
-
-       if (prot & IOMMU_PROT_IR)
-               __pte |= IOMMU_PTE_IR;
-       if (prot & IOMMU_PROT_IW)
-               __pte |= IOMMU_PTE_IW;
-
-       for (i = 0; i < count; ++i)
-               pte[i] = __pte;
-
-       ret = 0;
-
-out:
-       if (updated) {
-               unsigned long flags;
-
-               spin_lock_irqsave(&dom->lock, flags);
-               /*
-                * Flush domain TLB(s) and wait for completion. Any Device-Table
-                * Updates and flushing already happened in
-                * increase_address_space().
-                */
-               amd_iommu_domain_flush_tlb_pde(dom);
-               amd_iommu_domain_flush_complete(dom);
-               spin_unlock_irqrestore(&dom->lock, flags);
-       }
-
-       /* Everything flushed out, free pages now */
-       free_page_list(freelist);
-
-       return ret;
-}
-
-static unsigned long iommu_unmap_page(struct protection_domain *dom,
-                                     unsigned long bus_addr,
-                                     unsigned long page_size)
-{
-       unsigned long long unmapped;
-       unsigned long unmap_size;
-       u64 *pte;
-
-       BUG_ON(!is_power_of_2(page_size));
-
-       unmapped = 0;
-
-       while (unmapped < page_size) {
-
-               pte = fetch_pte(dom, bus_addr, &unmap_size);
-
-               if (pte) {
-                       int i, count;
-
-                       count = PAGE_SIZE_PTE_COUNT(unmap_size);
-                       for (i = 0; i < count; i++)
-                               pte[i] = 0ULL;
-               }
-
-               bus_addr  = (bus_addr & ~(unmap_size - 1)) + unmap_size;
-               unmapped += unmap_size;
-       }
-
-       BUG_ON(unmapped && !is_power_of_2(unmapped));
-
-       return unmapped;
-}
-
 /****************************************************************************
  *
  * The next functions belong to the domain allocation. A domain is
-- 
2.17.1

_______________________________________________
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu

Reply via email to