Re: [PATCH 09/12] asm-generic/tlb: Track which levels of the page tables have been cleared
On Thu, 30 Aug 2018 17:15:43 +0100 Will Deacon wrote: > It is common for architectures with hugepage support to require only a > single TLB invalidation operation per hugepage during unmap(), rather than > iterating through the mapping at a PAGE_SIZE increment. Currently, > however, the level in the page table where the unmap() operation occurs > is not stored in the mmu_gather structure, therefore forcing > architectures to issue additional TLB invalidation operations or to give > up and over-invalidate by e.g. invalidating the entire TLB. > > Ideally, we could add an interval rbtree to the mmu_gather structure, > which would allow us to associate the correct mapping granule with the > various sub-mappings within the range being invalidated. However, this > is costly in terms of book-keeping and memory management, so instead we > approximate by keeping track of the page table levels that are cleared > and provide a means to query the smallest granule required for invalidation. Actually the generic patches are pretty simple, and they look okay to me. powerpc *should* be able to switch to Peter's patch with a few lines of code with unchanged functionality as far as I can see. These flags we may use as well, but even if not if x86 and arm64 are using it, it seems reasonable to go in generic code for now. For the 3 generic patches, Acked-by: Nicholas Piggin Thanks, Nick
Re: [PATCH 09/12] asm-generic/tlb: Track which levels of the page tables have been cleared
On Thu, 30 Aug 2018 17:15:43 +0100 Will Deacon wrote: > It is common for architectures with hugepage support to require only a > single TLB invalidation operation per hugepage during unmap(), rather than > iterating through the mapping at a PAGE_SIZE increment. Currently, > however, the level in the page table where the unmap() operation occurs > is not stored in the mmu_gather structure, therefore forcing > architectures to issue additional TLB invalidation operations or to give > up and over-invalidate by e.g. invalidating the entire TLB. > > Ideally, we could add an interval rbtree to the mmu_gather structure, > which would allow us to associate the correct mapping granule with the > various sub-mappings within the range being invalidated. However, this > is costly in terms of book-keeping and memory management, so instead we > approximate by keeping track of the page table levels that are cleared > and provide a means to query the smallest granule required for invalidation. Actually the generic patches are pretty simple, and they look okay to me. powerpc *should* be able to switch to Peter's patch with a few lines of code with unchanged functionality as far as I can see. These flags we may use as well, but even if not if x86 and arm64 are using it, it seems reasonable to go in generic code for now. For the 3 generic patches, Acked-by: Nicholas Piggin Thanks, Nick
[PATCH 09/12] asm-generic/tlb: Track which levels of the page tables have been cleared
It is common for architectures with hugepage support to require only a
single TLB invalidation operation per hugepage during unmap(), rather than
iterating through the mapping at a PAGE_SIZE increment. Currently,
however, the level in the page table where the unmap() operation occurs
is not stored in the mmu_gather structure, therefore forcing
architectures to issue additional TLB invalidation operations or to give
up and over-invalidate by e.g. invalidating the entire TLB.
Ideally, we could add an interval rbtree to the mmu_gather structure,
which would allow us to associate the correct mapping granule with the
various sub-mappings within the range being invalidated. However, this
is costly in terms of book-keeping and memory management, so instead we
approximate by keeping track of the page table levels that are cleared
and provide a means to query the smallest granule required for invalidation.
Signed-off-by: Will Deacon
---
include/asm-generic/tlb.h | 58 ---
mm/memory.c | 4 +++-
2 files changed, 53 insertions(+), 9 deletions(-)
diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index 2b444ad94566..9791e98122a0 100644
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -116,6 +116,14 @@ struct mmu_gather {
*/
unsigned intfreed_tables : 1;
+ /*
+* at which levels have we cleared entries?
+*/
+ unsigned intcleared_ptes : 1;
+ unsigned intcleared_pmds : 1;
+ unsigned intcleared_puds : 1;
+ unsigned intcleared_p4ds : 1;
+
struct mmu_gather_batch *active;
struct mmu_gather_batch local;
struct page *__pages[MMU_GATHER_BUNDLE];
@@ -150,6 +158,10 @@ static inline void __tlb_reset_range(struct mmu_gather
*tlb)
tlb->end = 0;
}
tlb->freed_tables = 0;
+ tlb->cleared_ptes = 0;
+ tlb->cleared_pmds = 0;
+ tlb->cleared_puds = 0;
+ tlb->cleared_p4ds = 0;
}
static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
@@ -199,6 +211,25 @@ static inline void
tlb_remove_check_page_size_change(struct mmu_gather *tlb,
}
#endif
+static inline unsigned long tlb_get_unmap_shift(struct mmu_gather *tlb)
+{
+ if (tlb->cleared_ptes)
+ return PAGE_SHIFT;
+ if (tlb->cleared_pmds)
+ return PMD_SHIFT;
+ if (tlb->cleared_puds)
+ return PUD_SHIFT;
+ if (tlb->cleared_p4ds)
+ return P4D_SHIFT;
+
+ return PAGE_SHIFT;
+}
+
+static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb)
+{
+ return 1UL << tlb_get_unmap_shift(tlb);
+}
+
/*
* In the case of tlb vma handling, we can optimise these away in the
* case where we're doing a full MM flush. When we're doing a munmap,
@@ -232,13 +263,19 @@ static inline void
tlb_remove_check_page_size_change(struct mmu_gather *tlb,
#define tlb_remove_tlb_entry(tlb, ptep, address) \
do {\
__tlb_adjust_range(tlb, address, PAGE_SIZE);\
+ tlb->cleared_ptes = 1; \
__tlb_remove_tlb_entry(tlb, ptep, address); \
} while (0)
-#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address)\
- do { \
- __tlb_adjust_range(tlb, address, huge_page_size(h)); \
- __tlb_remove_tlb_entry(tlb, ptep, address); \
+#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \
+ do {\
+ unsigned long _sz = huge_page_size(h); \
+ __tlb_adjust_range(tlb, address, _sz); \
+ if (_sz == PMD_SIZE)\
+ tlb->cleared_pmds = 1; \
+ else if (_sz == PUD_SIZE) \
+ tlb->cleared_puds = 1; \
+ __tlb_remove_tlb_entry(tlb, ptep, address); \
} while (0)
/**
@@ -252,6 +289,7 @@ static inline void tlb_remove_check_page_size_change(struct
mmu_gather *tlb,
#define tlb_remove_pmd_tlb_entry(tlb, pmdp, address) \
do {\
__tlb_adjust_range(tlb, address, HPAGE_PMD_SIZE); \
+ tlb->cleared_pmds = 1; \
__tlb_remove_pmd_tlb_entry(tlb, pmdp, address); \
} while (0)
@@ -266,6 +304,7 @@ static inline void tlb_remove_check_page_size_change(struct
mmu_gather *tlb,
#define tlb_remove_pud_tlb_entry(tlb, pudp, address) \
do {
[PATCH 09/12] asm-generic/tlb: Track which levels of the page tables have been cleared
It is common for architectures with hugepage support to require only a
single TLB invalidation operation per hugepage during unmap(), rather than
iterating through the mapping at a PAGE_SIZE increment. Currently,
however, the level in the page table where the unmap() operation occurs
is not stored in the mmu_gather structure, therefore forcing
architectures to issue additional TLB invalidation operations or to give
up and over-invalidate by e.g. invalidating the entire TLB.
Ideally, we could add an interval rbtree to the mmu_gather structure,
which would allow us to associate the correct mapping granule with the
various sub-mappings within the range being invalidated. However, this
is costly in terms of book-keeping and memory management, so instead we
approximate by keeping track of the page table levels that are cleared
and provide a means to query the smallest granule required for invalidation.
Signed-off-by: Will Deacon
---
include/asm-generic/tlb.h | 58 ---
mm/memory.c | 4 +++-
2 files changed, 53 insertions(+), 9 deletions(-)
diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index 2b444ad94566..9791e98122a0 100644
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -116,6 +116,14 @@ struct mmu_gather {
*/
unsigned intfreed_tables : 1;
+ /*
+* at which levels have we cleared entries?
+*/
+ unsigned intcleared_ptes : 1;
+ unsigned intcleared_pmds : 1;
+ unsigned intcleared_puds : 1;
+ unsigned intcleared_p4ds : 1;
+
struct mmu_gather_batch *active;
struct mmu_gather_batch local;
struct page *__pages[MMU_GATHER_BUNDLE];
@@ -150,6 +158,10 @@ static inline void __tlb_reset_range(struct mmu_gather
*tlb)
tlb->end = 0;
}
tlb->freed_tables = 0;
+ tlb->cleared_ptes = 0;
+ tlb->cleared_pmds = 0;
+ tlb->cleared_puds = 0;
+ tlb->cleared_p4ds = 0;
}
static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
@@ -199,6 +211,25 @@ static inline void
tlb_remove_check_page_size_change(struct mmu_gather *tlb,
}
#endif
+static inline unsigned long tlb_get_unmap_shift(struct mmu_gather *tlb)
+{
+ if (tlb->cleared_ptes)
+ return PAGE_SHIFT;
+ if (tlb->cleared_pmds)
+ return PMD_SHIFT;
+ if (tlb->cleared_puds)
+ return PUD_SHIFT;
+ if (tlb->cleared_p4ds)
+ return P4D_SHIFT;
+
+ return PAGE_SHIFT;
+}
+
+static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb)
+{
+ return 1UL << tlb_get_unmap_shift(tlb);
+}
+
/*
* In the case of tlb vma handling, we can optimise these away in the
* case where we're doing a full MM flush. When we're doing a munmap,
@@ -232,13 +263,19 @@ static inline void
tlb_remove_check_page_size_change(struct mmu_gather *tlb,
#define tlb_remove_tlb_entry(tlb, ptep, address) \
do {\
__tlb_adjust_range(tlb, address, PAGE_SIZE);\
+ tlb->cleared_ptes = 1; \
__tlb_remove_tlb_entry(tlb, ptep, address); \
} while (0)
-#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address)\
- do { \
- __tlb_adjust_range(tlb, address, huge_page_size(h)); \
- __tlb_remove_tlb_entry(tlb, ptep, address); \
+#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \
+ do {\
+ unsigned long _sz = huge_page_size(h); \
+ __tlb_adjust_range(tlb, address, _sz); \
+ if (_sz == PMD_SIZE)\
+ tlb->cleared_pmds = 1; \
+ else if (_sz == PUD_SIZE) \
+ tlb->cleared_puds = 1; \
+ __tlb_remove_tlb_entry(tlb, ptep, address); \
} while (0)
/**
@@ -252,6 +289,7 @@ static inline void tlb_remove_check_page_size_change(struct
mmu_gather *tlb,
#define tlb_remove_pmd_tlb_entry(tlb, pmdp, address) \
do {\
__tlb_adjust_range(tlb, address, HPAGE_PMD_SIZE); \
+ tlb->cleared_pmds = 1; \
__tlb_remove_pmd_tlb_entry(tlb, pmdp, address); \
} while (0)
@@ -266,6 +304,7 @@ static inline void tlb_remove_check_page_size_change(struct
mmu_gather *tlb,
#define tlb_remove_pud_tlb_entry(tlb, pudp, address) \
do {
