As a step towards removing pagetable ctor/dtor calls for kernel
mm's, move the page type updates from the ctor/dtor to
pagetable_{alloc,free}.
__pagetable_ctor() is now empty so remove it;
pagetable_{pud,p4d,pgd}_ctor() will be removed at a later stage.
The ctor is normally called after pagetable_alloc() (and dtor before
pagetable_free()) so this patch should have no functional effect in
most cases. Where calls to ctor/dtor are currently missing (e.g.
x86's split_large_page()), this patch will ensure the page type is
set correctly.
This patch is based on Matthew's earlier implementation linked
below, with the difference that when freeing the page type is
cleared in __pagetable_free(), meaning that the update is delayed
until the pages are actually freed in the
CONFIG_ASYNC_KERNEL_PGTABLE_FREE case. This shouldn't matter in
practice.
Suggested-by: "Matthew Wilcox (Oracle)" <[email protected]>
Link: https://lore.kernel.org/r/[email protected]
Signed-off-by: Kevin Brodsky <[email protected]>
---
include/linux/mm.h | 17 ++---------------
1 file changed, 2 insertions(+), 15 deletions(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 361a0d7b74c5..94f0fb1c662a 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -3684,6 +3684,7 @@ static inline struct ptdesc *pagetable_alloc_noprof(gfp_t
gfp, unsigned int orde
if (!page)
return NULL;
+ __folio_set_pgtable(page_folio(page));
lruvec_stat_add_folio(page_folio(page), NR_PAGETABLE);
return page_ptdesc(page);
@@ -3694,6 +3695,7 @@ static inline void __pagetable_free(struct ptdesc *pt)
{
struct page *page = ptdesc_page(pt);
+ __folio_clear_pgtable(page_folio(page));
lruvec_stat_sub_folio(page_folio(page), NR_PAGETABLE);
__free_pages(page, compound_order(page));
@@ -3799,19 +3801,9 @@ static inline bool ptlock_init(struct ptdesc *ptdesc) {
return true; }
static inline void ptlock_free(struct ptdesc *ptdesc) {}
#endif /* defined(CONFIG_SPLIT_PTE_PTLOCKS) */
-static inline void __pagetable_ctor(struct ptdesc *ptdesc)
-{
- struct folio *folio = ptdesc_folio(ptdesc);
-
- __folio_set_pgtable(folio);
-}
-
static inline void pagetable_dtor(struct ptdesc *ptdesc)
{
- struct folio *folio = ptdesc_folio(ptdesc);
-
ptlock_free(ptdesc);
- __folio_clear_pgtable(folio);
}
static inline void pagetable_dtor_free(struct ptdesc *ptdesc)
@@ -3825,7 +3817,6 @@ static inline bool pagetable_pte_ctor(struct mm_struct
*mm,
{
if (!mm_is_kernel(mm) && !ptlock_init(ptdesc))
return false;
- __pagetable_ctor(ptdesc);
return true;
}
@@ -3917,7 +3908,6 @@ static inline bool pagetable_pmd_ctor(struct mm_struct
*mm,
if (!mm_is_kernel(mm) && !pmd_ptlock_init(ptdesc))
return false;
ptdesc_pmd_pts_init(ptdesc);
- __pagetable_ctor(ptdesc);
return true;
}
@@ -3942,17 +3932,14 @@ static inline spinlock_t *pud_lock(struct mm_struct
*mm, pud_t *pud)
static inline void pagetable_pud_ctor(struct ptdesc *ptdesc)
{
- __pagetable_ctor(ptdesc);
}
static inline void pagetable_p4d_ctor(struct ptdesc *ptdesc)
{
- __pagetable_ctor(ptdesc);
}
static inline void pagetable_pgd_ctor(struct ptdesc *ptdesc)
{
- __pagetable_ctor(ptdesc);
}
extern void __init pagecache_init(void);
--
2.51.2
