Since glibc commit 321e1fc73f (“malloc: Enable 2MB THP by default on AArch64”), glibc may call madvise(MADV_HUGEPAGE) for sufficiently large allocations made by memalign().
The underlying VMA may start at a different address from the aligned address returned by memalign(). Furthermore, a subsequent madvise(MADV_HUGEPAGE) call does not split the VMA because the flag is already set. This causes split_huge_page_test to fail because the check_huge_xxx() helpers incorrectly require the address returned by memalign() to match the VMA start address reported in /proc/self/smaps. Fix this by using /proc/self/pagemap and /proc/kpageflags instead of /proc/self/smaps to detect huge pages. Reported-by: David Hildenbrand (Arm) <[email protected]> Signed-off-by: Yeoreum Yun <[email protected]> --- tools/testing/selftests/mm/vm_util.c | 130 ++++++++++++++++++++--------------- tools/testing/selftests/mm/vm_util.h | 1 + 2 files changed, 77 insertions(+), 54 deletions(-) diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c index 4821a3563036..1d0959b3b9e8 100644 --- a/tools/testing/selftests/mm/vm_util.c +++ b/tools/testing/selftests/mm/vm_util.c @@ -351,31 +351,13 @@ char *__get_smap_entry(void *addr, const char *pattern, char *buf, size_t len) return entry; } -static bool __check_pmd_huge(void *addr, char *pattern, int nr_hpages, - uint64_t hpage_size) -{ - char buffer[MAX_LINE_LENGTH]; - uint64_t thp = -1; - char *entry; - - entry = __get_smap_entry(addr, pattern, buffer, sizeof(buffer)); - if (!entry) - goto err_out; - - if (sscanf(entry, "%9" SCNu64 " kB", &thp) != 1) - ksft_exit_fail_msg("Reading smap error\n"); - -err_out: - return thp == (nr_hpages * (hpage_size >> 10)); -} - -static bool check_large_folios(void *addr, size_t len, int nr_hpages, - uint64_t hpage_size) +static bool check_large_folios(int pagemap_fd, int kpageflags_fd, + void *addr, size_t len, int nr_hpages, + uint64_t hpage_size) { int order = 0, pagesize = getpagesize(); unsigned int nr_pages = hpage_size / pagesize; int orders[MAX_NR_ORDERS], status; - int pagemap_fd, kpageflags_fd; bool ret = false; if (!nr_pages) @@ -386,15 +368,6 @@ static bool check_large_folios(void *addr, size_t len, int nr_hpages, ksft_exit_fail_msg("invalid order\n"); memset(orders, 0, sizeof(int) * MAX_NR_ORDERS); - pagemap_fd = open(PAGEMAP_PATH, O_RDONLY); - if (pagemap_fd == -1) - ksft_exit_fail_msg("read pagemap fail\n"); - - kpageflags_fd = open(KPAGEFLAGS_PATH, O_RDONLY); - if (kpageflags_fd == -1) { - close(pagemap_fd); - ksft_exit_fail_msg("read kpageflags fail\n"); - } status = gather_folio_orders(addr, len, pagemap_fd, kpageflags_fd, orders, MAX_NR_ORDERS); @@ -405,48 +378,97 @@ static bool check_large_folios(void *addr, size_t len, int nr_hpages, ret = true; out: - close(pagemap_fd); - close(kpageflags_fd); return ret; } -bool check_huge_anon(void *addr, size_t len, int nr_hpages, uint64_t hpage_size) +enum check_huge_type { + CHECK_HUGE_ANON, + CHECK_HUGE_FILE, + CHECK_HUGE_SHMEM, +}; + +static bool __check_pmd_huge(void *addr, size_t len, int nr_hpages, + uint64_t hpage_size, enum check_huge_type type) { - uint64_t pmd_pagesize = read_pmd_pagesize(); + int pagemap_fd, kpageflags_fd; + uint64_t pmd_pagesize, granule; + uint64_t categories, kpf; + unsigned long pfn; + bool check_large, huge_mapped; + char *start = addr; + char *end = start + len; + pmd_pagesize = read_pmd_pagesize(); if (!pmd_pagesize) ksft_exit_fail_msg("reading PMD pagesize failed\n"); - if (hpage_size == pmd_pagesize) - return __check_pmd_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size); + if (nr_hpages > 0) { + check_large = true; + granule = hpage_size; + } else { + check_large = false; + granule = psize(); + } - return check_large_folios(addr, len, nr_hpages, hpage_size); -} + pagemap_fd = open(PAGEMAP_PATH, O_RDONLY); + if (pagemap_fd < 0) + ksft_exit_fail_msg("open pagemap fail\n"); -bool check_huge_file(void *addr, size_t len, int nr_hpages, uint64_t hpage_size) -{ - uint64_t pmd_pagesize = read_pmd_pagesize(); + kpageflags_fd = open(KPAGEFLAGS_PATH, O_RDONLY); + if (kpageflags_fd < 0) { + close(pagemap_fd); + ksft_exit_fail_msg("open kpageflags fail\n"); + } - if (!pmd_pagesize) - ksft_exit_fail_msg("reading PMD pagesize failed\n"); + if (check_large && !check_large_folios(pagemap_fd, kpageflags_fd, + addr, len, nr_hpages, hpage_size)) + goto out; - if (hpage_size == pmd_pagesize) - return __check_pmd_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size); + for (; start < end; start += granule) { + categories = pagemap_scan_get_categories(pagemap_fd, start); + pfn = pagemap_get_pfn(pagemap_fd, start); + if (pfn == -1UL) { + if (check_large) + goto out; + else + continue; + } + if (pageflags_get(pfn, kpageflags_fd, &kpf)) + ksft_exit_fail_msg("read kpageflags: %s\n", strerror(errno)); + huge_mapped = categories & PAGE_IS_HUGE; + if (check_large != huge_mapped) { + if (!check_large || granule == pmd_pagesize) + goto out; + } + if (kpf & KPF_COMPOUND_TAIL) + continue; + if (!!(categories & PAGE_IS_FILE) != (type != CHECK_HUGE_ANON)) + goto out; + if (type == CHECK_HUGE_ANON) + continue; + if (!!(kpf & KPF_SWAPBACKED) != (type != CHECK_HUGE_FILE)) + goto out; + } - return check_large_folios(addr, len, nr_hpages, hpage_size); +out: + close(pagemap_fd); + close(kpageflags_fd); + return start >= end; } -bool check_huge_shmem(void *addr, size_t len, int nr_hpages, uint64_t hpage_size) +bool check_huge_anon(void *addr, size_t len, int nr_hpages, uint64_t hpage_size) { - uint64_t pmd_pagesize = read_pmd_pagesize(); - - if (!pmd_pagesize) - ksft_exit_fail_msg("reading PMD pagesize failed\n"); + return __check_pmd_huge(addr, len, nr_hpages, hpage_size, CHECK_HUGE_ANON); +} - if (hpage_size == pmd_pagesize) - return __check_pmd_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size); +bool check_huge_file(void *addr, size_t len, int nr_hpages, uint64_t hpage_size) +{ + return __check_pmd_huge(addr, len, nr_hpages, hpage_size, CHECK_HUGE_FILE); +} - return check_large_folios(addr, len, nr_hpages, hpage_size); +bool check_huge_shmem(void *addr, size_t len, int nr_hpages, uint64_t hpage_size) +{ + return __check_pmd_huge(addr, len, nr_hpages, hpage_size, CHECK_HUGE_SHMEM); } int64_t allocate_transhuge(void *ptr, int pagemap_fd) diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h index 9a49af88702e..f19bd17817e5 100644 --- a/tools/testing/selftests/mm/vm_util.h +++ b/tools/testing/selftests/mm/vm_util.h @@ -18,6 +18,7 @@ #define PM_SWAP BIT_ULL(62) #define PM_PRESENT BIT_ULL(63) +#define KPF_SWAPBACKED BIT_ULL(14) #define KPF_COMPOUND_HEAD BIT_ULL(15) #define KPF_COMPOUND_TAIL BIT_ULL(16) #define KPF_HWPOISON BIT_ULL(19) -- 2.43.0
