Since glibc commit 321e1fc73f (“malloc: Enable 2MB THP by default on AArch64”), glibc may call madvise(MADV_HUGEPAGE) for sufficiently large allocations made by memalign().
The underlying VMA may start at a different address from the aligned address returned by memalign(). Furthermore, a subsequent madvise(MADV_HUGEPAGE) call does not split the VMA because the flag is already set. This causes split_huge_page_test to fail because the check_huge_xxx() helpers incorrectly require the address returned by memalign() to match the VMA start address reported in /proc/self/smaps. Fix this by using /proc/self/pagemap and /proc/kpageflags instead of /proc/self/smaps to detect huge pages and change the meaning of check_huge_xxx()'s nr_hpages argument: - nr_hpages > 0: check all of pages in the range are huge page. - nr_hpages < 0: check all of pages in the range are not huge page. - nr_hpages == 0: invalid. Reported-by: David Hildenbrand (Arm) <[email protected]> Signed-off-by: Yeoreum Yun <[email protected]> --- tools/testing/selftests/mm/khugepaged.c | 26 ++--- tools/testing/selftests/mm/split_huge_page_test.c | 6 +- tools/testing/selftests/mm/vm_util.c | 135 ++++++++++++++++++---- tools/testing/selftests/mm/vm_util.h | 1 + 4 files changed, 131 insertions(+), 37 deletions(-) diff --git a/tools/testing/selftests/mm/khugepaged.c b/tools/testing/selftests/mm/khugepaged.c index 10e8dedcb087d..2e4665e6d5852 100644 --- a/tools/testing/selftests/mm/khugepaged.c +++ b/tools/testing/selftests/mm/khugepaged.c @@ -533,7 +533,7 @@ static void __madvise_collapse(const char *msg, char *p, int nr_hpages, ret = madvise_collapse_retry(p, nr_hpages * hpage_pmd_size); if (((bool)ret) == expect) fail("Fail: Bad return value"); - else if (!ops->check_huge(p, expect ? nr_hpages : 0)) + else if (!ops->check_huge(p, expect ? nr_hpages : -nr_hpages)) fail("Fail: check_huge()"); else success("OK"); @@ -545,7 +545,7 @@ static void madvise_collapse(const char *msg, char *p, int nr_hpages, struct mem_ops *ops, bool expect) { /* Sanity check */ - if (!ops->check_huge(p, 0)) + if (!ops->check_huge(p, -nr_hpages)) ksft_exit_fail_msg("Unexpected huge page\n"); __madvise_collapse(msg, p, nr_hpages, ops, expect); } @@ -558,7 +558,7 @@ static bool wait_for_scan(const char *msg, char *p, int nr_hpages, int timeout = 6; /* 3 seconds */ /* Sanity check */ - if (!ops->check_huge(p, 0)) + if (!ops->check_huge(p, -nr_hpages)) ksft_exit_fail_msg("Unexpected huge page\n"); madvise(p, nr_hpages * hpage_pmd_size, MADV_HUGEPAGE); @@ -605,7 +605,7 @@ static void khugepaged_collapse(const char *msg, char *p, int nr_hpages, if (ops != &__anon_ops) ops->fault(p, 0, nr_hpages * hpage_pmd_size); - if (ops->check_huge(p, expect ? nr_hpages : 0)) + if (ops->check_huge(p, expect ? nr_hpages : -nr_hpages)) success("OK"); else fail("Fail"); @@ -643,7 +643,7 @@ static void alloc_at_fault(void) madvise(p, page_size, MADV_DONTNEED); ksft_print_msg("Split huge PMD on MADV_DONTNEED..."); - if (check_huge_anon(p, 0, hpage_pmd_size)) + if (check_huge_anon(p, -1, hpage_pmd_size)) success("OK"); else fail("Fail"); @@ -815,7 +815,7 @@ static void collapse_single_pte_entry_compound(struct collapse_context *c, struc madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); ksft_print_msg("Split huge page leaving single PTE mapping compound page..."); madvise(p + page_size, hpage_pmd_size - page_size, MADV_DONTNEED); - if (ops->check_huge(p, 0)) + if (ops->check_huge(p, -1)) success("OK"); else fail("Fail"); @@ -836,7 +836,7 @@ static void collapse_full_of_compound(struct collapse_context *c, struct mem_ops ksft_print_msg("Split huge page leaving single PTE page table full of compound pages..."); madvise(p, page_size, MADV_NOHUGEPAGE); madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); - if (ops->check_huge(p, 0)) + if (ops->check_huge(p, -1)) success("OK"); else fail("Fail"); @@ -903,7 +903,7 @@ static void collapse_fork(struct collapse_context *c, struct mem_ops *ops) ksft_print_msg("Allocate small page..."); ops->fault(p, 0, page_size); - if (ops->check_huge(p, 0)) + if (ops->check_huge(p, -1)) success("OK"); else fail("Fail"); @@ -911,7 +911,7 @@ static void collapse_fork(struct collapse_context *c, struct mem_ops *ops) ksft_print_msg("Share small page over fork()..."); if (!fork()) { /* Do not touch settings on child exit */ - if (ops->check_huge(p, 0)) + if (ops->check_huge(p, -1)) success("OK"); else fail("Fail"); @@ -929,7 +929,7 @@ static void collapse_fork(struct collapse_context *c, struct mem_ops *ops) exit_status = WEXITSTATUS(wstatus); ksft_print_msg("Check if parent still has small page..."); - if (ops->check_huge(p, 0)) + if (ops->check_huge(p, -1)) success("OK"); else fail("Fail"); @@ -955,7 +955,7 @@ static void collapse_fork_compound(struct collapse_context *c, struct mem_ops *o ksft_print_msg("Split huge page PMD in child process..."); madvise(p, page_size, MADV_NOHUGEPAGE); madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); - if (ops->check_huge(p, 0)) + if (ops->check_huge(p, -1)) success("OK"); else fail("Fail"); @@ -1003,7 +1003,7 @@ static void collapse_max_ptes_shared(struct collapse_context *c, struct mem_ops ksft_print_msg("Trigger CoW on page %d of %d...", hpage_pmd_nr - max_ptes_shared - 1, hpage_pmd_nr); ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared - 1) * page_size); - if (ops->check_huge(p, 0)) + if (ops->check_huge(p, -1)) success("OK"); else fail("Fail"); @@ -1016,7 +1016,7 @@ static void collapse_max_ptes_shared(struct collapse_context *c, struct mem_ops hpage_pmd_nr - max_ptes_shared, hpage_pmd_nr); ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared) * page_size); - if (ops->check_huge(p, 0)) + if (ops->check_huge(p, -1)) success("OK"); else fail("Fail"); diff --git a/tools/testing/selftests/mm/split_huge_page_test.c b/tools/testing/selftests/mm/split_huge_page_test.c index 99b45e5518199..4fcd330cf0726 100644 --- a/tools/testing/selftests/mm/split_huge_page_test.c +++ b/tools/testing/selftests/mm/split_huge_page_test.c @@ -314,7 +314,7 @@ static void verify_rss_anon_split_huge_page_all_zeroes(char *one_page, int nr_hp if (one_page[i] != (char)0) ksft_exit_fail_msg("%ld byte corrupted\n", i); - if (!check_huge_anon(one_page, 0, pmd_pagesize)) + if (!check_huge_anon(one_page, -nr_hpages, pmd_pagesize)) ksft_exit_fail_msg("Still AnonHugePages not split\n"); rss_anon_after = rss_anon(); @@ -372,7 +372,7 @@ static void split_pmd_thp_to_order(int order) (pmd_order + 1))) ksft_exit_fail_msg("Unexpected THP split\n"); - if (!check_huge_anon(one_page, 0, pmd_pagesize)) + if (!check_huge_anon(one_page, -4, pmd_pagesize)) ksft_exit_fail_msg("Still AnonHugePages not split\n"); ksft_test_result_pass("Split huge pages to order %d successful\n", order); @@ -746,7 +746,7 @@ static void split_thp_in_pagecache_to_order_at(size_t fd_size, goto out; } - if (!check_huge_file(addr, 0, pmd_pagesize)) { + if (!check_huge_file(addr, -(fd_size / pmd_pagesize), pmd_pagesize)) { ksft_print_msg("Still FilePmdMapped not split\n"); err = EXIT_FAILURE; goto out; diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c index 311fc5b4513eb..166d5d79289f3 100644 --- a/tools/testing/selftests/mm/vm_util.c +++ b/tools/testing/selftests/mm/vm_util.c @@ -14,6 +14,8 @@ #define PMD_SIZE_FILE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size" #define SMAP_FILE_PATH "/proc/self/smaps" #define STATUS_FILE_PATH "/proc/self/status" +#define PAGEMAP_FILE_PATH "/proc/self/pagemap" +#define KPAGEFLAGS_FILE_PATH "/proc/kpageflags" #define MAX_LINE_LENGTH 500 unsigned int __page_size; @@ -229,37 +231,128 @@ char *__get_smap_entry(void *addr, const char *pattern, char *buf, size_t len) return entry; } -bool __check_huge(void *addr, char *pattern, int nr_hpages, - uint64_t hpage_size) -{ - char buffer[MAX_LINE_LENGTH]; - uint64_t thp = -1; - char *entry; - - entry = __get_smap_entry(addr, pattern, buffer, sizeof(buffer)); - if (!entry) - goto err_out; - - if (sscanf(entry, "%9" SCNu64 " kB", &thp) != 1) - ksft_exit_fail_msg("Reading smap error\n"); - -err_out: - return thp == (nr_hpages * (hpage_size >> 10)); -} - bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size) { - return __check_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size); + int i, pagemap_fd, nr = 0; + uint64_t categories; + bool expect_huge; + + if (nr_hpages == 0 || hpage_size == 0) + return false; + + if (nr_hpages < 0) { + nr_hpages = -nr_hpages; + expect_huge = false; + } else + expect_huge = true; + + pagemap_fd = open(PAGEMAP_FILE_PATH, O_RDONLY); + if (pagemap_fd < 0) + ksft_exit_fail_msg("open pagemap: %s\n", strerror(errno)); + + for (i = 0; i < nr_hpages; i++, addr += hpage_size) { + categories = pagemap_scan_get_categories(pagemap_fd, addr); + if (!(categories & PAGE_IS_HUGE)) + continue; + if (categories & PAGE_IS_FILE) + continue; + nr++; + } + + close(pagemap_fd); + return expect_huge ? nr == nr_hpages : nr == 0; } bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size) { - return __check_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size); + int i, pagemap_fd, kpf_fd, nr = 0; + unsigned long pfn; + uint64_t categories, kpf; + bool expect_huge; + + if (nr_hpages == 0 || hpage_size == 0) + return false; + + if (nr_hpages < 0) { + nr_hpages = -nr_hpages; + expect_huge = false; + } else + expect_huge = true; + + pagemap_fd = open(PAGEMAP_FILE_PATH, O_RDONLY); + if (pagemap_fd < 0) + ksft_exit_fail_msg("open pagemap: %s\n", strerror(errno)); + + kpf_fd = open(KPAGEFLAGS_FILE_PATH, O_RDONLY); + if (kpf_fd < 0) + ksft_exit_fail_msg("open kpageflags: %s\n", strerror(errno)); + + for (i = 0; i < nr_hpages; i++, addr += hpage_size) { + categories = pagemap_scan_get_categories(pagemap_fd, addr); + pfn = pagemap_get_pfn(pagemap_fd, addr); + if (pfn == -1UL) + continue; + if (pageflags_get(pfn, kpf_fd, &kpf)) + ksft_exit_fail_msg("read kpageflags: %s\n", strerror(errno)); + if (!(categories & PAGE_IS_HUGE)) + continue; + if (!(categories & PAGE_IS_FILE)) + continue; + if (kpf & KPF_SWAPBACKED) + continue; + nr++; + } + + close(pagemap_fd); + close(kpf_fd); + + return expect_huge ? nr == nr_hpages : nr == 0; } bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size) { - return __check_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size); + int i, pagemap_fd, kpf_fd, nr = 0; + unsigned long pfn; + uint64_t categories, kpf; + bool expect_huge; + + if (nr_hpages == 0 || hpage_size == 0) + return false; + + if (nr_hpages < 0) { + nr_hpages = -nr_hpages; + expect_huge = true; + } else + expect_huge = false; + + pagemap_fd = open(PAGEMAP_FILE_PATH, O_RDONLY); + if (pagemap_fd < 0) + ksft_exit_fail_msg("open pagemap: %s\n", strerror(errno)); + + kpf_fd = open(KPAGEFLAGS_FILE_PATH, O_RDONLY); + if (kpf_fd < 0) + ksft_exit_fail_msg("open kpageflags: %s\n", strerror(errno)); + + for (i = 0; i < nr_hpages; i++, addr += hpage_size) { + categories = pagemap_scan_get_categories(pagemap_fd, addr); + pfn = pagemap_get_pfn(pagemap_fd, addr); + if (pfn == -1UL) + continue; + if (pageflags_get(pfn, kpf_fd, &kpf)) + ksft_exit_fail_msg("read kpageflags: %s\n", strerror(errno)); + if (!(categories & PAGE_IS_HUGE)) + continue; + if (!(categories & PAGE_IS_FILE)) + continue; + if (!(kpf & KPF_SWAPBACKED)) + continue; + nr++; + } + + close(pagemap_fd); + close(kpf_fd); + + return expect_huge ? nr == nr_hpages : nr == 0; } int64_t allocate_transhuge(void *ptr, int pagemap_fd) diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h index ea8fc8fdf0eb0..0a1627f36efc3 100644 --- a/tools/testing/selftests/mm/vm_util.h +++ b/tools/testing/selftests/mm/vm_util.h @@ -18,6 +18,7 @@ #define PM_SWAP BIT_ULL(62) #define PM_PRESENT BIT_ULL(63) +#define KPF_SWAPBACKED BIT_ULL(14) #define KPF_COMPOUND_HEAD BIT_ULL(15) #define KPF_COMPOUND_TAIL BIT_ULL(16) #define KPF_HWPOISON BIT_ULL(19) -- 2.43.0
