From: Peter Zijlstra <pet...@infradead.org>

commit 22a61c3c4f1379ef8b0ce0d5cb78baf3178950e2 upstream

Some architectures require different TLB invalidation instructions
depending on whether it is only the last-level of page table being
changed, or whether there are also changes to the intermediate
(directory) entries higher up the tree.

Add a new bit to the flags bitfield in struct mmu_gather so that the
architecture code can operate accordingly if it's the intermediate
levels being invalidated.

Signed-off-by: Peter Zijlstra <pet...@infradead.org>
Signed-off-by: Will Deacon <will.dea...@arm.com>
Cc: <sta...@vger.kernel.org> # 4.19
Signed-off-by: Santosh Sivaraj <sant...@fossix.org>
[santosh: prerequisite for tlbflush backports]
---
 include/asm-generic/tlb.h | 31 +++++++++++++++++++++++--------
 1 file changed, 23 insertions(+), 8 deletions(-)

diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index b3353e21f3b3..97306b32d8d2 100644
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -97,12 +97,22 @@ struct mmu_gather {
 #endif
        unsigned long           start;
        unsigned long           end;
-       /* we are in the middle of an operation to clear
-        * a full mm and can make some optimizations */
-       unsigned int            fullmm : 1,
-       /* we have performed an operation which
-        * requires a complete flush of the tlb */
-                               need_flush_all : 1;
+       /*
+        * we are in the middle of an operation to clear
+        * a full mm and can make some optimizations
+        */
+       unsigned int            fullmm : 1;
+
+       /*
+        * we have performed an operation which
+        * requires a complete flush of the tlb
+        */
+       unsigned int            need_flush_all : 1;
+
+       /*
+        * we have removed page directories
+        */
+       unsigned int            freed_tables : 1;
 
        struct mmu_gather_batch *active;
        struct mmu_gather_batch local;
@@ -137,6 +147,7 @@ static inline void __tlb_reset_range(struct mmu_gather *tlb)
                tlb->start = TASK_SIZE;
                tlb->end = 0;
        }
+       tlb->freed_tables = 0;
 }
 
 static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
@@ -278,6 +289,7 @@ static inline void tlb_remove_check_page_size_change(struct 
mmu_gather *tlb,
 #define pte_free_tlb(tlb, ptep, address)                       \
        do {                                                    \
                __tlb_adjust_range(tlb, address, PAGE_SIZE);    \
+               tlb->freed_tables = 1;                  \
                __pte_free_tlb(tlb, ptep, address);             \
        } while (0)
 #endif
@@ -285,7 +297,8 @@ static inline void tlb_remove_check_page_size_change(struct 
mmu_gather *tlb,
 #ifndef pmd_free_tlb
 #define pmd_free_tlb(tlb, pmdp, address)                       \
        do {                                                    \
-               __tlb_adjust_range(tlb, address, PAGE_SIZE);            \
+               __tlb_adjust_range(tlb, address, PAGE_SIZE);    \
+               tlb->freed_tables = 1;                  \
                __pmd_free_tlb(tlb, pmdp, address);             \
        } while (0)
 #endif
@@ -295,6 +308,7 @@ static inline void tlb_remove_check_page_size_change(struct 
mmu_gather *tlb,
 #define pud_free_tlb(tlb, pudp, address)                       \
        do {                                                    \
                __tlb_adjust_range(tlb, address, PAGE_SIZE);    \
+               tlb->freed_tables = 1;                  \
                __pud_free_tlb(tlb, pudp, address);             \
        } while (0)
 #endif
@@ -304,7 +318,8 @@ static inline void tlb_remove_check_page_size_change(struct 
mmu_gather *tlb,
 #ifndef p4d_free_tlb
 #define p4d_free_tlb(tlb, pudp, address)                       \
        do {                                                    \
-               __tlb_adjust_range(tlb, address, PAGE_SIZE);            \
+               __tlb_adjust_range(tlb, address, PAGE_SIZE);    \
+               tlb->freed_tables = 1;                  \
                __p4d_free_tlb(tlb, pudp, address);             \
        } while (0)
 #endif
-- 
2.25.4

Reply via email to