HVO depends on build-time conditions that are not fully expressible in Kconfig, including whether sizeof(struct page) is a power of two and whether the supported folio order range can use the optimized layout.
Those checks are currently duplicated in several places. Define SPARSEMEM_VMEMMAP_OPTIMIZATION in bounds.c when the build-time requirements are met, and use that generated constant to guard the generic HVO code. This centralizes the build-time checks instead of repeating them throughout the HVO paths. Signed-off-by: Muchun Song <[email protected]> --- arch/x86/entry/vdso/vdso32/fake_32bit_build.h | 1 - drivers/dax/Kconfig | 2 +- fs/Kconfig | 2 +- include/linux/mm_types.h | 3 +- include/linux/mmzone.h | 38 ++++++++----------- include/linux/page-flags-layout.h | 2 + include/linux/page-flags.h | 28 ++------------ kernel/bounds.c | 5 +++ mm/Kconfig | 2 +- mm/hugetlb_vmemmap.c | 2 + mm/hugetlb_vmemmap.h | 4 +- mm/internal.h | 3 -- mm/sparse.c | 6 +-- mm/util.c | 2 +- 14 files changed, 38 insertions(+), 62 deletions(-) diff --git a/arch/x86/entry/vdso/vdso32/fake_32bit_build.h b/arch/x86/entry/vdso/vdso32/fake_32bit_build.h index 5f8424eade2b..db1b15f686e3 100644 --- a/arch/x86/entry/vdso/vdso32/fake_32bit_build.h +++ b/arch/x86/entry/vdso/vdso32/fake_32bit_build.h @@ -11,7 +11,6 @@ #undef CONFIG_PGTABLE_LEVELS #undef CONFIG_ILLEGAL_POINTER_VALUE #undef CONFIG_SPARSEMEM_VMEMMAP -#undef CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION #undef CONFIG_NR_CPUS #undef CONFIG_PARAVIRT_XXL diff --git a/drivers/dax/Kconfig b/drivers/dax/Kconfig index 60cb05dce53d..cb7710c29885 100644 --- a/drivers/dax/Kconfig +++ b/drivers/dax/Kconfig @@ -8,7 +8,7 @@ if DAX config DEV_DAX tristate "Device DAX: direct access mapping device" depends on TRANSPARENT_HUGEPAGE - select SPARSEMEM_VMEMMAP_OPTIMIZATION if ARCH_WANT_OPTIMIZE_DAX_VMEMMAP && SPARSEMEM_VMEMMAP + select SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLE if ARCH_WANT_OPTIMIZE_DAX_VMEMMAP && SPARSEMEM_VMEMMAP help Support raw access to differentiated (persistence, bandwidth, latency...) memory via an mmap(2) capable character diff --git a/fs/Kconfig b/fs/Kconfig index 496cfa2379e5..ab3937abe07f 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -278,7 +278,7 @@ config HUGETLB_PAGE_OPTIMIZE_VMEMMAP def_bool HUGETLB_PAGE depends on ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP depends on SPARSEMEM_VMEMMAP - select SPARSEMEM_VMEMMAP_OPTIMIZATION + select SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLE config HUGETLB_PMD_PAGE_TABLE_SHARING def_bool HUGETLB_PAGE diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index a308e2c23b82..9a7cd7575f3a 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -546,6 +546,7 @@ FOLIO_MATCH(flags, _flags_3); FOLIO_MATCH(compound_info, _head_3); #undef FOLIO_MATCH +#ifndef __GENERATING_BOUNDS_H /** * struct ptdesc - Memory descriptor for page tables. * @pt_flags: enum pt_flags plus zone/node/section. @@ -1990,5 +1991,5 @@ static inline unsigned long mmf_init_legacy_flags(unsigned long flags) (1UL << MMF_HAS_MDWE_NO_INHERIT)); return flags & MMF_INIT_LEGACY_MASK; } - +#endif /* __GENERATING_BOUNDS_H */ #endif /* _LINUX_MM_TYPES_H */ diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index efb37f2ffec4..0d49d6e163ff 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -3,8 +3,6 @@ #define _LINUX_MMZONE_H #ifndef __ASSEMBLY__ -#ifndef __GENERATING_BOUNDS_H - #include <linux/spinlock.h> #include <linux/list.h> #include <linux/list_nulls.h> @@ -96,33 +94,32 @@ #define MAX_FOLIO_NR_PAGES (1UL << MAX_FOLIO_ORDER) -/* - * Hugepage Vmemmap Optimization (HVO) requires struct pages of the head page to - * be naturally aligned with regard to the folio size. - * - * HVO which is only active if the size of struct page is a power of 2. - */ -#define MAX_FOLIO_VMEMMAP_ALIGN \ - (IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION) && \ - is_power_of_2(sizeof(struct page)) ? \ - MAX_FOLIO_NR_PAGES * sizeof(struct page) : 0) - /* The number of vmemmap pages required by a vmemmap-optimized folio. */ #define OPTIMIZED_FOLIO_VMEMMAP_PAGES 1 #define OPTIMIZED_FOLIO_VMEMMAP_SIZE (OPTIMIZED_FOLIO_VMEMMAP_PAGES * PAGE_SIZE) #define OPTIMIZED_FOLIO_VMEMMAP_NR_STRUCT_PAGES (OPTIMIZED_FOLIO_VMEMMAP_SIZE / sizeof(struct page)) #define OPTIMIZABLE_FOLIO_MIN_ORDER (ilog2(OPTIMIZED_FOLIO_VMEMMAP_NR_STRUCT_PAGES) + 1) -#define __NR_OPTIMIZABLE_FOLIO_ORDERS (MAX_FOLIO_ORDER - OPTIMIZABLE_FOLIO_MIN_ORDER + 1) -#define NR_OPTIMIZABLE_FOLIO_ORDERS \ - ((__NR_OPTIMIZABLE_FOLIO_ORDERS > 0 && \ - IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION)) ? __NR_OPTIMIZABLE_FOLIO_ORDERS : 0) +#ifdef SPARSEMEM_VMEMMAP_OPTIMIZATION +/* + * Hugepage Vmemmap Optimization (HVO) requires the struct page of the head page + * to be naturally aligned with regard to the vmemmap size of the maximal folio. + */ +#define MAX_FOLIO_VMEMMAP_ALIGN (MAX_FOLIO_NR_PAGES * sizeof(struct page)) +#define NR_OPTIMIZABLE_FOLIO_ORDERS (MAX_FOLIO_ORDER - OPTIMIZABLE_FOLIO_MIN_ORDER + 1) +#else +#define MAX_FOLIO_VMEMMAP_ALIGN 0 +#define NR_OPTIMIZABLE_FOLIO_ORDERS 0 +#endif static inline bool order_vmemmap_optimizable(unsigned int order) { + if (!IS_ENABLED(SPARSEMEM_VMEMMAP_OPTIMIZATION)) + return false; return order >= OPTIMIZABLE_FOLIO_MIN_ORDER; } +#ifndef __GENERATING_BOUNDS_H enum migratetype { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, @@ -2044,7 +2041,7 @@ struct mem_section { */ struct page_ext *page_ext; #endif -#ifdef CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION +#ifdef SPARSEMEM_VMEMMAP_OPTIMIZATION /* * The order of compound pages in this section. Typically, the section * holds compound pages of this order; a larger compound page will span @@ -2236,7 +2233,7 @@ static inline bool pfn_section_first_valid(struct mem_section *ms, unsigned long } #endif -#ifdef CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION +#ifdef SPARSEMEM_VMEMMAP_OPTIMIZATION static inline void section_set_order(struct mem_section *section, unsigned int order) { VM_WARN_ON(section->order && order && section->order != order); @@ -2277,9 +2274,6 @@ static inline unsigned int pfn_to_section_order(unsigned long pfn) static inline bool section_vmemmap_optimizable(const struct mem_section *section) { - if (!is_power_of_2(sizeof(struct page))) - return false; - return order_vmemmap_optimizable(section_order(section)); } diff --git a/include/linux/page-flags-layout.h b/include/linux/page-flags-layout.h index 760006b1c480..6a7e7f3dbb93 100644 --- a/include/linux/page-flags-layout.h +++ b/include/linux/page-flags-layout.h @@ -2,6 +2,7 @@ #ifndef PAGE_FLAGS_LAYOUT_H #define PAGE_FLAGS_LAYOUT_H +#ifndef __GENERATING_BOUNDS_H #include <linux/numa.h> #include <generated/bounds.h> @@ -121,4 +122,5 @@ (NR_NON_PAGEFLAG_BITS + NR_PAGEFLAGS)) #endif +#endif /* __GENERATING_BOUNDS_H */ #endif /* _LINUX_PAGE_FLAGS_LAYOUT */ diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 12665b34586c..df7f6dea2e5b 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -198,32 +198,12 @@ enum pageflags { #ifndef __GENERATING_BOUNDS_H -/* - * For tail pages, if the size of struct page is power-of-2 ->compound_info - * encodes the mask that converts the address of the tail page address to - * the head page address. - * - * Otherwise, ->compound_info has direct pointer to head pages. - */ -static __always_inline bool compound_info_has_mask(void) -{ - /* - * The approach with mask would work in the wider set of conditions, - * but it requires validating that struct pages are naturally aligned - * for all orders up to the MAX_FOLIO_ORDER, which can be tricky. - */ - if (!IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION)) - return false; - - return is_power_of_2(sizeof(struct page)); -} - static __always_inline unsigned long _compound_head(const struct page *page) { unsigned long info = READ_ONCE(page->compound_info); unsigned long mask; - if (!compound_info_has_mask()) { + if (!IS_ENABLED(SPARSEMEM_VMEMMAP_OPTIMIZATION)) { /* Bit 0 encodes PageTail() */ if (info & 1) return info - 1; @@ -232,8 +212,8 @@ static __always_inline unsigned long _compound_head(const struct page *page) } /* - * If compound_info_has_mask() is true the rest of the info encodes - * the mask that converts the address of the tail page to the head page. + * If HVO is enabled the rest of the info encodes the mask that converts + * the address of the tail page to the head page. * * No need to clear bit 0 in the mask as 'page' always has it clear. * @@ -257,7 +237,7 @@ static __always_inline void set_compound_head(struct page *tail, unsigned int shift; unsigned long mask; - if (!compound_info_has_mask()) { + if (!IS_ENABLED(SPARSEMEM_VMEMMAP_OPTIMIZATION)) { WRITE_ONCE(tail->compound_info, (unsigned long)head | 1); return; } diff --git a/kernel/bounds.c b/kernel/bounds.c index 02b619eb6106..9638260d67f8 100644 --- a/kernel/bounds.c +++ b/kernel/bounds.c @@ -8,6 +8,7 @@ #define __GENERATING_BOUNDS_H #define COMPILE_OFFSETS /* Include headers that define the enum constants of interest */ +#include <linux/mm_types.h> #include <linux/page-flags.h> #include <linux/mmzone.h> #include <linux/kbuild.h> @@ -30,6 +31,10 @@ int main(void) DEFINE(LRU_GEN_WIDTH, 0); DEFINE(__LRU_REFS_WIDTH, 0); #endif + if (IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLE) && + is_power_of_2(sizeof(struct page)) && + MAX_FOLIO_ORDER >= OPTIMIZABLE_FOLIO_MIN_ORDER) + DEFINE(SPARSEMEM_VMEMMAP_OPTIMIZATION, 1); /* End of constants */ return 0; diff --git a/mm/Kconfig b/mm/Kconfig index c85ed7d7f37d..52d9d69a95ff 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -410,7 +410,7 @@ config SPARSEMEM_VMEMMAP pfn_to_page and page_to_pfn operations. This is the most efficient option when sufficient kernel resources are available. -config SPARSEMEM_VMEMMAP_OPTIMIZATION +config SPARSEMEM_VMEMMAP_OPTIMIZATION_ENABLE bool depends on SPARSEMEM_VMEMMAP diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index 6f6f1740f540..1305bee1195a 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -22,6 +22,7 @@ #include "hugetlb_vmemmap.h" #include "internal.h" +#ifdef SPARSEMEM_VMEMMAP_OPTIMIZATION /** * struct vmemmap_remap_walk - walk vmemmap page table * @@ -693,3 +694,4 @@ static int __init hugetlb_vmemmap_init(void) return 0; } late_initcall(hugetlb_vmemmap_init); +#endif diff --git a/mm/hugetlb_vmemmap.h b/mm/hugetlb_vmemmap.h index b4d0ba27b42c..dfd48be6b231 100644 --- a/mm/hugetlb_vmemmap.h +++ b/mm/hugetlb_vmemmap.h @@ -10,7 +10,7 @@ #define _LINUX_HUGETLB_VMEMMAP_H #include <linux/hugetlb.h> -#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP +#if defined(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP) && defined(SPARSEMEM_VMEMMAP_OPTIMIZATION) int hugetlb_vmemmap_restore_folio(const struct hstate *h, struct folio *folio); long hugetlb_vmemmap_restore_folios(const struct hstate *h, struct list_head *folio_list, @@ -32,8 +32,6 @@ static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate { int size = hugetlb_vmemmap_size(h) - OPTIMIZED_FOLIO_VMEMMAP_SIZE; - if (!is_power_of_2(sizeof(struct page))) - return 0; return size > 0 ? size : 0; } #else diff --git a/mm/internal.h b/mm/internal.h index 9597a703bc73..afdae79640b5 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1023,9 +1023,6 @@ static inline bool vmemmap_page_optimizable(const struct page *page) unsigned long pfn = page_to_pfn(page); unsigned long nr_pages = 1UL << pfn_to_section_order(pfn); - if (!is_power_of_2(sizeof(struct page))) - return false; - return (pfn & (nr_pages - 1)) >= OPTIMIZED_FOLIO_VMEMMAP_NR_STRUCT_PAGES; } #else diff --git a/mm/sparse.c b/mm/sparse.c index bdf23709a1c7..598da1651e49 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -301,10 +301,8 @@ void __init sparse_init(void) unsigned long pnum_end, pnum_begin, map_count = 1; int nid_begin; - if (compound_info_has_mask()) { - VM_WARN_ON_ONCE(!IS_ALIGNED((unsigned long) pfn_to_page(0), - MAX_FOLIO_VMEMMAP_ALIGN)); - } + VM_WARN_ON(IS_ENABLED(SPARSEMEM_VMEMMAP_OPTIMIZATION) && + !IS_ALIGNED((unsigned long)pfn_to_page(0), MAX_FOLIO_VMEMMAP_ALIGN)); pnum_begin = first_present_section_nr(); nid_begin = sparse_early_nid(__nr_to_section(pnum_begin)); diff --git a/mm/util.c b/mm/util.c index 3cc949a0b7ed..4543f2b6ffa1 100644 --- a/mm/util.c +++ b/mm/util.c @@ -1338,7 +1338,7 @@ void snapshot_page(struct page_snapshot *ps, const struct page *page) foliop = (struct folio *)page; } else { /* See compound_head() */ - if (compound_info_has_mask()) { + if (IS_ENABLED(SPARSEMEM_VMEMMAP_OPTIMIZATION)) { unsigned long p = (unsigned long)page; foliop = (struct folio *)(p & info); -- 2.54.0
