Since glibc commit 321e1fc73f (“malloc: Enable 2MB THP by default on AArch64”), glibc may call madvise(MADV_HUGEPAGE) for sufficiently large allocations made by memalign().
The underlying VMA may start at a different address from the aligned address returned by memalign(). Furthermore, a subsequent madvise(MADV_HUGEPAGE) call does not split the VMA because the flag is already set. This causes split_huge_page_test to fail because the check_huge_xxx() helpers incorrectly require the address returned by memalign() to match the VMA start address reported in /proc/self/smaps. Fix this by using /proc/self/pagemap and /proc/kpageflags instead of /proc/self/smaps to detect huge pages. Reported-by: David Hildenbrand (Arm) <[email protected]> Suggested-by: Zi Yan <[email protected]> Signed-off-by: Yeoreum Yun <[email protected]> --- tools/testing/selftests/mm/vm_util.c | 145 ++++++++++++++++++++++------------- tools/testing/selftests/mm/vm_util.h | 1 + 2 files changed, 92 insertions(+), 54 deletions(-) diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c index 4821a3563036..083b72d92b14 100644 --- a/tools/testing/selftests/mm/vm_util.c +++ b/tools/testing/selftests/mm/vm_util.c @@ -351,31 +351,13 @@ char *__get_smap_entry(void *addr, const char *pattern, char *buf, size_t len) return entry; } -static bool __check_pmd_huge(void *addr, char *pattern, int nr_hpages, - uint64_t hpage_size) -{ - char buffer[MAX_LINE_LENGTH]; - uint64_t thp = -1; - char *entry; - - entry = __get_smap_entry(addr, pattern, buffer, sizeof(buffer)); - if (!entry) - goto err_out; - - if (sscanf(entry, "%9" SCNu64 " kB", &thp) != 1) - ksft_exit_fail_msg("Reading smap error\n"); - -err_out: - return thp == (nr_hpages * (hpage_size >> 10)); -} - -static bool check_large_folios(void *addr, size_t len, int nr_hpages, - uint64_t hpage_size) +static bool check_large_folios(int pagemap_fd, int kpageflags_fd, + void *addr, size_t len, int nr_hpages, + uint64_t hpage_size) { int order = 0, pagesize = getpagesize(); unsigned int nr_pages = hpage_size / pagesize; int orders[MAX_NR_ORDERS], status; - int pagemap_fd, kpageflags_fd; bool ret = false; if (!nr_pages) @@ -386,15 +368,6 @@ static bool check_large_folios(void *addr, size_t len, int nr_hpages, ksft_exit_fail_msg("invalid order\n"); memset(orders, 0, sizeof(int) * MAX_NR_ORDERS); - pagemap_fd = open(PAGEMAP_PATH, O_RDONLY); - if (pagemap_fd == -1) - ksft_exit_fail_msg("read pagemap fail\n"); - - kpageflags_fd = open(KPAGEFLAGS_PATH, O_RDONLY); - if (kpageflags_fd == -1) { - close(pagemap_fd); - ksft_exit_fail_msg("read kpageflags fail\n"); - } status = gather_folio_orders(addr, len, pagemap_fd, kpageflags_fd, orders, MAX_NR_ORDERS); @@ -405,48 +378,112 @@ static bool check_large_folios(void *addr, size_t len, int nr_hpages, ret = true; out: - close(pagemap_fd); - close(kpageflags_fd); return ret; } -bool check_huge_anon(void *addr, size_t len, int nr_hpages, uint64_t hpage_size) -{ - uint64_t pmd_pagesize = read_pmd_pagesize(); - - if (!pmd_pagesize) - ksft_exit_fail_msg("reading PMD pagesize failed\n"); +enum check_huge_type { + CHECK_HUGE_ANON, + CHECK_HUGE_FILE, + CHECK_HUGE_SHMEM, +}; - if (hpage_size == pmd_pagesize) - return __check_pmd_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size); +static bool check_huge_type(uint64_t categories, uint64_t kpageflags, + enum check_huge_type type) +{ + bool file = categories & PAGE_IS_FILE; + bool swapbacked = kpageflags & KPF_SWAPBACKED; + + switch (type) { + case CHECK_HUGE_ANON: + return !file; + case CHECK_HUGE_FILE: + return file && !swapbacked; + case CHECK_HUGE_SHMEM: + return file & swapbacked; + } - return check_large_folios(addr, len, nr_hpages, hpage_size); + return false; } -bool check_huge_file(void *addr, size_t len, int nr_hpages, uint64_t hpage_size) +static bool __check_huge(void *addr, size_t len, int nr_hpages, + uint64_t hpage_size, enum check_huge_type type) { - uint64_t pmd_pagesize = read_pmd_pagesize(); + bool ret = false; + int pagemap_fd, kpageflags_fd; + int nr_pmd_mappings = 0; + uint64_t pmd_pagesize, scan_mapping_size; + uint64_t categories, kpf; + unsigned long pfn; + bool check_pmd_mapping, allow_nonpresent; + char *start = addr; + char *end = start + len; + pmd_pagesize = read_pmd_pagesize(); if (!pmd_pagesize) ksft_exit_fail_msg("reading PMD pagesize failed\n"); - if (hpage_size == pmd_pagesize) - return __check_pmd_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size); + check_pmd_mapping = (hpage_size == pmd_pagesize); + scan_mapping_size = (nr_hpages > 0) ? hpage_size : psize(); + /* Some mTHP tests check a partially populated PMD-sized range. */ + allow_nonpresent = (uint64_t)nr_hpages * hpage_size < len; - return check_large_folios(addr, len, nr_hpages, hpage_size); + pagemap_fd = open(PAGEMAP_PATH, O_RDONLY); + if (pagemap_fd < 0) + ksft_exit_fail_msg("open pagemap fail\n"); + + kpageflags_fd = open(KPAGEFLAGS_PATH, O_RDONLY); + if (kpageflags_fd < 0) { + close(pagemap_fd); + ksft_exit_fail_msg("open kpageflags fail\n"); + } + + if (!check_pmd_mapping && + !check_large_folios(pagemap_fd, kpageflags_fd, + addr, len, nr_hpages, hpage_size)) + goto out; + + for (; start < end; start += scan_mapping_size) { + categories = pagemap_scan_get_categories(pagemap_fd, start); + pfn = pagemap_get_pfn(pagemap_fd, start); + if (pfn == -1UL) { + if (!allow_nonpresent) + goto out; + else + continue; + } + if (pageflags_get(pfn, kpageflags_fd, &kpf)) + ksft_exit_fail_msg("read kpageflags: %s\n", strerror(errno)); + if (check_pmd_mapping && (categories & PAGE_IS_HUGE)) + nr_pmd_mappings++; + if (kpf & KPF_COMPOUND_TAIL) + continue; + if (!check_huge_type(categories, kpf, type)) + goto out; + } + + if (check_pmd_mapping && (nr_pmd_mappings != nr_hpages)) + goto out; + ret = true; + +out: + close(pagemap_fd); + close(kpageflags_fd); + return ret; } -bool check_huge_shmem(void *addr, size_t len, int nr_hpages, uint64_t hpage_size) +bool check_huge_anon(void *addr, size_t len, int nr_hpages, uint64_t hpage_size) { - uint64_t pmd_pagesize = read_pmd_pagesize(); - - if (!pmd_pagesize) - ksft_exit_fail_msg("reading PMD pagesize failed\n"); + return __check_huge(addr, len, nr_hpages, hpage_size, CHECK_HUGE_ANON); +} - if (hpage_size == pmd_pagesize) - return __check_pmd_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size); +bool check_huge_file(void *addr, size_t len, int nr_hpages, uint64_t hpage_size) +{ + return __check_huge(addr, len, nr_hpages, hpage_size, CHECK_HUGE_FILE); +} - return check_large_folios(addr, len, nr_hpages, hpage_size); +bool check_huge_shmem(void *addr, size_t len, int nr_hpages, uint64_t hpage_size) +{ + return __check_huge(addr, len, nr_hpages, hpage_size, CHECK_HUGE_SHMEM); } int64_t allocate_transhuge(void *ptr, int pagemap_fd) diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h index 9a49af88702e..f19bd17817e5 100644 --- a/tools/testing/selftests/mm/vm_util.h +++ b/tools/testing/selftests/mm/vm_util.h @@ -18,6 +18,7 @@ #define PM_SWAP BIT_ULL(62) #define PM_PRESENT BIT_ULL(63) +#define KPF_SWAPBACKED BIT_ULL(14) #define KPF_COMPOUND_HEAD BIT_ULL(15) #define KPF_COMPOUND_TAIL BIT_ULL(16) #define KPF_HWPOISON BIT_ULL(19) -- 2.43.0
