Now that DAX and HugeTLB use the same optimized vmemmap layout, they no longer need separate population flows.
Move the shared-tail-page handling into vmemmap_pte_populate() so both users can go through the normal basepage population path. This removes the compound-page-specific population helper and leaves the optimized mapping decisions in one place. At runtime, the optimized users are limited to ZONE_DEVICE memory, so use device_zone() for shared-tail-page allocation instead of relying on pfn_to_zone() before zone spans are available. Signed-off-by: Muchun Song <[email protected]> --- arch/powerpc/mm/book3s64/radix_pgtable.c | 3 + mm/mm_init.c | 2 +- mm/sparse-vmemmap.c | 183 ++++++----------------- 3 files changed, 50 insertions(+), 138 deletions(-) diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c index f0043c57694e..c7f2327681cc 100644 --- a/arch/powerpc/mm/book3s64/radix_pgtable.c +++ b/arch/powerpc/mm/book3s64/radix_pgtable.c @@ -1121,7 +1121,10 @@ int __meminit radix__vmemmap_populate(unsigned long start, unsigned long end, in pud_t *pud; pmd_t *pmd; pte_t *pte; + unsigned long pfn = page_to_pfn((struct page *)start); + if (section_vmemmap_optimizable(__pfn_to_section(pfn))) + return vmemmap_populate_compound_pages(pfn, start, end, node, NULL); /* * If altmap is present, Make sure we align the start vmemmap addr * to PAGE_SIZE so that we calculate the correct start_pfn in diff --git a/mm/mm_init.c b/mm/mm_init.c index 2b94115e6dd5..9ff118e35641 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1068,7 +1068,7 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, * initialize is a lot smaller that the total amount of struct pages being * mapped. This is a paired / mild layering violation with explicit knowledge * of how the sparse_vmemmap internals handle compound pages in the lack - * of an altmap. See vmemmap_populate_compound_pages(). + * of an altmap. */ static inline unsigned long compound_nr_pages(unsigned long pfn, struct dev_pagemap *pgmap) diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index ad3e5b54abf7..4833a2295abb 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -127,49 +127,48 @@ static pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, in struct vmem_altmap *altmap, unsigned long ptpfn) { - pte_t *pte = pte_offset_kernel(pmd, addr); - - if (pte_none(ptep_get(pte))) { - pte_t entry; - - if (vmemmap_page_optimizable((struct page *)addr) && - ptpfn == (unsigned long)-1) { - struct page *page; - unsigned long pfn = page_to_pfn((struct page *)addr); - const struct mem_section *ms = __pfn_to_section(pfn); - struct zone *zone = pfn_to_zone(pfn, node); - - if (WARN_ON_ONCE(!zone)) - return NULL; - page = vmemmap_shared_tail_page(section_order(ms), zone); - if (!page) - return NULL; - ptpfn = page_to_pfn(page); - } + pte_t entry, *pte = pte_offset_kernel(pmd, addr); + struct page *page = (struct page *)addr; + + if (!pte_none(ptep_get(pte))) + return WARN_ON_ONCE(vmemmap_page_optimizable(page)) ? NULL : pte; + + /* See layout diagram in Documentation/mm/vmemmap_dedup.rst. */ + if (vmemmap_page_optimizable(page)) { + struct zone *zone; + unsigned long pfn = page_to_pfn(page); + + /* + * At runtime (slab available), only ZONE_DEVICE pages (DAX) + * trigger vmemmap optimization, so device_zone() suffices. + * Note: pfn_to_zone() cannot be used at runtime because the + * zone span is not set up now. + */ + zone = slab_is_available() ? device_zone(node) : pfn_to_zone(pfn, node); + if (WARN_ON_ONCE(!zone)) + return NULL; + page = vmemmap_shared_tail_page(pfn_to_section_order(pfn), zone); + if (!page) + return NULL; + + /* + * When a PTE entry is freed, a free_pages() call occurs. This + * get_page() pairs with put_page_testzero() on the freeing + * path. This can only occur when slab is available. + */ + if (slab_is_available()) + get_page(page); + ptpfn = page_to_pfn(page); + } else { + void *vaddr = vmemmap_alloc_block_buf(PAGE_SIZE, node, altmap); + + if (!vaddr) + return NULL; + ptpfn = PHYS_PFN(__pa(vaddr)); + } + entry = pfn_pte(ptpfn, PAGE_KERNEL); + set_pte_at(&init_mm, addr, pte, entry); - if (ptpfn == (unsigned long)-1) { - void *p = vmemmap_alloc_block_buf(PAGE_SIZE, node, altmap); - - if (!p) - return NULL; - ptpfn = PHYS_PFN(__pa(p)); - } else { - /* - * When a PTE/PMD entry is freed from the init_mm - * there's a free_pages() call to this page allocated - * above. Thus this get_page() is paired with the - * put_page_testzero() on the freeing path. - * This can only called by certain ZONE_DEVICE path, - * and through vmemmap_populate_compound_pages() when - * slab is available. - */ - if (slab_is_available()) - get_page(pfn_to_page(ptpfn)); - } - entry = pfn_pte(ptpfn, PAGE_KERNEL); - set_pte_at(&init_mm, addr, pte, entry); - } else if (WARN_ON_ONCE(vmemmap_page_optimizable((struct page *)addr))) - return NULL; return pte; } @@ -265,30 +264,16 @@ static pte_t * __meminit vmemmap_populate_address(unsigned long addr, int node, return pte; } -static int __meminit vmemmap_populate_range(unsigned long start, - unsigned long end, int node, - struct vmem_altmap *altmap, - unsigned long ptpfn) +int __meminit vmemmap_populate_basepages(unsigned long start, unsigned long end, + int node, struct vmem_altmap *altmap) { - unsigned long addr = start; - pte_t *pte; - - for (; addr < end; addr += PAGE_SIZE) { - pte = vmemmap_populate_address(addr, node, altmap, - ptpfn); - if (!pte) + for (; start < end; start += PAGE_SIZE) + if (!vmemmap_populate_address(start, node, altmap, -1)) return -ENOMEM; - } return 0; } -int __meminit vmemmap_populate_basepages(unsigned long start, unsigned long end, - int node, struct vmem_altmap *altmap) -{ - return vmemmap_populate_range(start, end, node, altmap, -1); -} - /* * Write protect the mirrored tail page structs for HVO. This will be * called from the hugetlb code when gathering and initializing the @@ -425,94 +410,18 @@ int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end, return 0; } -#ifndef vmemmap_populate_compound_pages -/* - * For compound pages bigger than section size (e.g. x86 1G compound - * pages with 2M subsection size) fill the rest of sections as tail - * pages. - * - * Note that memremap_pages() resets @nr_range value and will increment - * it after each range successful onlining. Thus the value or @nr_range - * at section memmap populate corresponds to the in-progress range - * being onlined here. - */ -static bool __meminit reuse_compound_section(unsigned long start_pfn, - struct dev_pagemap *pgmap) -{ - unsigned long nr_pages = pgmap_vmemmap_nr(pgmap); - unsigned long offset = start_pfn - - PHYS_PFN(pgmap->ranges[pgmap->nr_range].start); - - return !IS_ALIGNED(offset, nr_pages) && nr_pages > PAGES_PER_SUBSECTION; -} - -static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, - unsigned long start, - unsigned long end, int node, - struct dev_pagemap *pgmap) -{ - unsigned long size, addr; - pte_t *pte; - int rc; - struct page *page; - const struct mem_section *ms = __pfn_to_section(start_pfn); - - page = vmemmap_shared_tail_page(section_order(ms), device_zone(node)); - if (!page) - return -ENOMEM; - - if (reuse_compound_section(start_pfn, pgmap)) - return vmemmap_populate_range(start, end, node, NULL, - page_to_pfn(page)); - - size = min(end - start, (1UL << section_order(ms)) * sizeof(struct page)); - for (addr = start; addr < end; addr += size) { - unsigned long next, last = addr + size; - - /* Populate the head page vmemmap page */ - pte = vmemmap_populate_address(addr, node, NULL, -1); - if (!pte) - return -ENOMEM; - - /* - * Reuse the shared page for the rest of tail pages - * See layout diagram in Documentation/mm/vmemmap_dedup.rst - */ - next = addr + PAGE_SIZE; - rc = vmemmap_populate_range(next, last, node, NULL, - page_to_pfn(page)); - if (rc) - return -ENOMEM; - } - - return 0; -} - -#endif - struct page * __meminit __populate_section_memmap(unsigned long pfn, unsigned long nr_pages, int nid, struct vmem_altmap *altmap, struct dev_pagemap *pgmap) { unsigned long start = (unsigned long) pfn_to_page(pfn); unsigned long end = start + nr_pages * sizeof(struct page); - int r; if (WARN_ON_ONCE(!IS_ALIGNED(pfn, PAGES_PER_SUBSECTION) || !IS_ALIGNED(nr_pages, PAGES_PER_SUBSECTION))) return NULL; - /* This may occur in sub-section scenarios. */ - if (vmemmap_can_optimize(altmap, pgmap) && - section_vmemmap_optimizable(__pfn_to_section(pfn))) - r = vmemmap_populate_compound_pages(pfn, start, end, nid, pgmap); - else - r = vmemmap_populate(start, end, nid, altmap); - - if (r < 0) - return NULL; - - return pfn_to_page(pfn); + return vmemmap_populate(start, end, nid, altmap) ? NULL : (void *)start; } static void subsection_mask_set(unsigned long *map, unsigned long pfn, -- 2.54.0
