Shared vmemmap tail pages are now mapped read-only when their PTEs are installed, so HugeTLB bootmem optimization no longer needs a separate write-protect pass afterwards.
Remove vmemmap_wrprotect_hvo() and the bootmem-specific HugeTLB wrapper, and let bootmem folios use the normal hugetlb_vmemmap_optimize_folios() path. Signed-off-by: Muchun Song <[email protected]> --- include/linux/mm.h | 2 -- mm/hugetlb.c | 2 +- mm/hugetlb_vmemmap.c | 45 +++++++++----------------------------------- mm/hugetlb_vmemmap.h | 6 ------ mm/sparse-vmemmap.c | 23 ---------------------- 5 files changed, 10 insertions(+), 68 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 86d7cecb834e..5e38c9a16a0a 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4863,8 +4863,6 @@ int vmemmap_populate_hugepages(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap); int vmemmap_populate(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap); -void vmemmap_wrprotect_hvo(unsigned long start, unsigned long end, int node, - unsigned long headsize); void vmemmap_populate_print_last(void); struct page *vmemmap_shared_tail_page(unsigned int order, struct zone *zone); #ifdef CONFIG_MEMORY_HOTPLUG diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 74770c1648fc..54ef7d12c585 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3202,7 +3202,7 @@ static void __init prep_and_add_bootmem_folios(struct hstate *h, struct folio *folio, *tmp_f; /* Send list for bulk vmemmap optimization processing */ - hugetlb_vmemmap_optimize_bootmem_folios(h, folio_list); + hugetlb_vmemmap_optimize_folios(h, folio_list); list_for_each_entry_safe(folio, tmp_f, folio_list, lru) { if (!folio_test_hugetlb_vmemmap_optimized(folio)) { diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index d24143dd6051..fce772e95adc 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -595,31 +595,22 @@ static int hugetlb_vmemmap_split_folio(const struct hstate *h, struct folio *fol return vmemmap_remap_split(vmemmap_start, vmemmap_end); } -static void __hugetlb_vmemmap_optimize_folios(struct hstate *h, - struct list_head *folio_list, - bool boot) +void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list) { struct folio *folio; - int nr_to_optimize; + unsigned long nr_to_optimize = 0; LIST_HEAD(vmemmap_pages); unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH; - nr_to_optimize = 0; list_for_each_entry(folio, folio_list, lru) { int ret; - unsigned long spfn, epfn; - - if (boot && folio_test_hugetlb_vmemmap_optimized(folio)) { - /* - * Already optimized by pre-HVO, just map the - * mirrored tail page structs RO. - */ - spfn = (unsigned long)&folio->page; - epfn = spfn + hugetlb_vmemmap_size(h); - vmemmap_wrprotect_hvo(spfn, epfn, folio_nid(folio), - OPTIMIZED_FOLIO_VMEMMAP_SIZE); + + /* + * Bootmem gigantic folios may already be marked optimized when + * their vmemmap layout was prepared earlier, so skip them here. + */ + if (folio_test_hugetlb_vmemmap_optimized(folio)) continue; - } nr_to_optimize++; @@ -636,14 +627,7 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h, } if (!nr_to_optimize) - /* - * All pre-HVO folios, nothing left to do. It's ok if - * there is a mix of pre-HVO and not yet HVO-ed folios - * here, as __hugetlb_vmemmap_optimize_folio() will - * skip any folios that already have the optimized flag - * set, see vmemmap_should_optimize_folio(). - */ - goto out; + return; flush_tlb_all(); @@ -668,21 +652,10 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h, } } -out: flush_tlb_all(); free_vmemmap_page_list(&vmemmap_pages); } -void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list) -{ - __hugetlb_vmemmap_optimize_folios(h, folio_list, false); -} - -void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head *folio_list) -{ - __hugetlb_vmemmap_optimize_folios(h, folio_list, true); -} - void __init hugetlb_vmemmap_optimize_bootmem_page(struct huge_bootmem_page *m) { struct hstate *h = m->hstate; diff --git a/mm/hugetlb_vmemmap.h b/mm/hugetlb_vmemmap.h index 0d8c88997066..2b0a85e09602 100644 --- a/mm/hugetlb_vmemmap.h +++ b/mm/hugetlb_vmemmap.h @@ -17,7 +17,6 @@ long hugetlb_vmemmap_restore_folios(const struct hstate *h, struct list_head *non_hvo_folios); void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio); void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list); -void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head *folio_list); void hugetlb_vmemmap_optimize_bootmem_page(struct huge_bootmem_page *m); static inline unsigned int hugetlb_vmemmap_size(const struct hstate *h) @@ -59,11 +58,6 @@ static inline void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list { } -static inline void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, - struct list_head *folio_list) -{ -} - static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate *h) { return 0; diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 5d5cd5f73365..ce1cf5cdf613 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -265,29 +265,6 @@ int __meminit vmemmap_populate_basepages(unsigned long start, unsigned long end, return 0; } -/* - * Write protect the mirrored tail page structs for HVO. This will be - * called from the hugetlb code when gathering and initializing the - * memblock allocated gigantic pages. The write protect can't be - * done earlier, since it can't be guaranteed that the reserved - * page structures will not be written to during initialization, - * even if CONFIG_DEFERRED_STRUCT_PAGE_INIT is enabled. - * - * The PTEs are known to exist, and nothing else should be touching - * these pages. The caller is responsible for any TLB flushing. - */ -void vmemmap_wrprotect_hvo(unsigned long addr, unsigned long end, - int node, unsigned long headsize) -{ - unsigned long maddr; - pte_t *pte; - - for (maddr = addr + headsize; maddr < end; maddr += PAGE_SIZE) { - pte = virt_to_kpte(maddr); - ptep_set_wrprotect(&init_mm, maddr, pte); - } -} - struct page __ref *vmemmap_shared_tail_page(unsigned int order, struct zone *zone) { void *addr; -- 2.54.0
