Both hints are non-destructive, but the tests so far only check that for private anonymous memory. Shared memory reaches reclaim differently:
Test the following: - a dirty shared file page must be written back or left resident, - shared anonymous memory is paged out through shmem - a SHM_LOCK'd segment is unevictable without its VMA being VM_LOCKED. MADV_PAGEOUT unmaps a shmem page before it tries to swap it out so none of these needs swap. The SHM_LOCK test confirms the page is unevictable and skips otherwise. Signed-off-by: Gregory Price (Meta) <[email protected]> --- v3: Dropped Sarthak's Reviewed-by from v2 5/8: added the fixture check for hidden PFNs and create_temp_file(), and the shared anonymous test now fails rather than skips when the page is not unmapped. --- .../testing/selftests/mm/madv_cold_pageout.c | 177 ++++++++++++++++++ tools/testing/selftests/mm/vm_util.h | 1 + 2 files changed, 178 insertions(+) diff --git a/tools/testing/selftests/mm/madv_cold_pageout.c b/tools/testing/selftests/mm/madv_cold_pageout.c index 0874726ec141..7d92f0ecbd27 100644 --- a/tools/testing/selftests/mm/madv_cold_pageout.c +++ b/tools/testing/selftests/mm/madv_cold_pageout.c @@ -7,7 +7,11 @@ #include <fcntl.h> #include <sched.h> #include <stdbool.h> +#include <stdint.h> +#include <sys/ipc.h> #include <sys/mman.h> +#include <sys/shm.h> +#include <sys/stat.h> #include <unistd.h> #include "kselftest_harness.h" @@ -44,19 +48,47 @@ static void pin_to_current_cpu(void) CPU_FREE(mask); } +/* Whether pagemap shows PFNs, which it only does with CAP_SYS_ADMIN. */ +static bool pagemap_shows_pfns(struct __test_metadata *_metadata, + int pagemap_fd) +{ + unsigned long pfn; + char *page; + + page = mmap(NULL, page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + ASSERT_NE(page, MAP_FAILED); + page[0] = 1; + pfn = pagemap_get_pfn(pagemap_fd, page); + ASSERT_EQ(munmap(page, page_size), 0); + + return pfn && pfn != -1UL; +} + FIXTURE(madv_cold_pageout) { int pagemap_fd; + /* Needs root, and is -1 unless pagemap also shows PFNs. */ + int kpageflags_fd; }; FIXTURE_SETUP(madv_cold_pageout) { self->pagemap_fd = open("/proc/self/pagemap", O_RDONLY); ASSERT_GE(self->pagemap_fd, 0); + self->kpageflags_fd = open("/proc/kpageflags", O_RDONLY); + /* Page flags are no use without the PFNs to look them up by. */ + if (self->kpageflags_fd >= 0 && + !pagemap_shows_pfns(_metadata, self->pagemap_fd)) { + close(self->kpageflags_fd); + self->kpageflags_fd = -1; + } } FIXTURE_TEARDOWN(madv_cold_pageout) { + if (self->kpageflags_fd >= 0) + close(self->kpageflags_fd); close(self->pagemap_fd); } @@ -76,6 +108,58 @@ static bool pageout_until_swapped(char *mapping, size_t size, int pagemap_fd) return false; } +/* Page out the range until its first page is unmapped, or madvise() fails. */ +static bool pageout_until_unmapped(char *mapping, size_t size, int pagemap_fd) +{ + int retry; + + for (retry = 0; retry < PAGEOUT_RETRIES; retry++) { + if (madvise(mapping, size, MADV_PAGEOUT)) + return false; + if (!pagemap_is_populated(pagemap_fd, mapping)) + return true; + usleep(10000); + } + + return false; +} + +/* Create an unlinked, zero-filled temporary file, or return -1. */ +static int create_temp_file(size_t size) +{ + char template[] = "/tmp/madv_cold_pageout-XXXXXX"; + int fd; + + fd = mkstemp(template); + if (fd < 0) + return -1; + unlink(template); + if (ftruncate(fd, size)) { + close(fd); + return -1; + } + + return fd; +} + +/* Map an unlinked, zero-filled temporary file with the given permissions. */ +static char *map_temp_file(struct __test_metadata *_metadata, size_t size, + int flags, mode_t mode) +{ + char *mapping; + int fd; + + fd = create_temp_file(size); + ASSERT_GE(fd, 0); + ASSERT_EQ(fchmod(fd, mode), 0); + + mapping = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, fd, 0); + ASSERT_NE(mapping, MAP_FAILED); + close(fd); + + return mapping; +} + /* * Both hints are best effort, so these tests check what userspace can rely * on: which mappings are accepted, which pages must never be reclaimed, and @@ -275,6 +359,99 @@ TEST_F(madv_cold_pageout, pageout_skips_swapped_pte) check_skips_swapped_pte(_metadata, self->pagemap_fd, MADV_PAGEOUT); } +/* MADV_COLD must not discard a dirty page of a shared file mapping. */ +TEST_F(madv_cold_pageout, cold_preserves_dirty_shared_file_page) +{ + char *mapping; + + mapping = map_temp_file(_metadata, page_size, MAP_SHARED, 0600); + mapping[0] = 1; + + ASSERT_EQ(madvise(mapping, page_size, MADV_COLD), 0); + ASSERT_EQ(mapping[0], 1); + + ASSERT_EQ(munmap(mapping, page_size), 0); +} + +/* + * MADV_PAGEOUT may write a dirty shared file page back or leave it resident, + * but must not discard it. + */ +TEST_F(madv_cold_pageout, pageout_preserves_dirty_shared_file_page) +{ + char *mapping; + + mapping = map_temp_file(_metadata, page_size, MAP_SHARED, 0600); + mapping[0] = 1; + + if (!pageout_until_unmapped(mapping, page_size, self->pagemap_fd)) + SKIP(return, "MADV_PAGEOUT did not unmap the page"); + ASSERT_EQ(mapping[0], 1); + + ASSERT_EQ(munmap(mapping, page_size), 0); +} + +/* MADV_PAGEOUT must not lose the contents of shared anonymous memory. */ +TEST_F(madv_cold_pageout, pageout_preserves_shared_anon_page) +{ + char *mapping; + + mapping = mmap(NULL, page_size, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1, 0); + ASSERT_NE(mapping, MAP_FAILED); + mapping[0] = 1; + + /* MADV_PAGEOUT unmaps a shmem page even when it cannot swap it out. */ + ASSERT_TRUE(pageout_until_unmapped(mapping, page_size, + self->pagemap_fd)); + ASSERT_EQ(mapping[0], 1); + + ASSERT_EQ(munmap(mapping, page_size), 0); +} + +/* + * SHM_LOCK makes shmem pages unevictable without setting VM_LOCKED on the + * VMA, so MADV_PAGEOUT accepts the range but must not reclaim the page. + */ +TEST_F(madv_cold_pageout, pageout_skips_unevictable_page) +{ + unsigned long pfn; + uint64_t flags; + char *mapping; + int shmid; + + if (self->kpageflags_fd < 0) + SKIP(return, "cannot read page flags"); + shmid = shmget(IPC_PRIVATE, page_size, IPC_CREAT | 0600); + ASSERT_GE(shmid, 0); + if (shmctl(shmid, SHM_LOCK, NULL)) { + shmctl(shmid, IPC_RMID, NULL); + SKIP(return, "could not lock a shmem segment"); + } + mapping = shmat(shmid, NULL, 0); + /* The segment is destroyed once it is detached. */ + shmctl(shmid, IPC_RMID, NULL); + ASSERT_NE(mapping, (void *)-1); + mapping[0] = 1; + + /* + * MADV_COLD drains this CPU's LRU add batch, which makes the page + * unevictable. + */ + ASSERT_EQ(madvise(mapping, page_size, MADV_COLD), 0); + pfn = pagemap_get_pfn(self->pagemap_fd, mapping); + ASSERT_NE(pfn, -1UL); + ASSERT_EQ(pageflags_get(pfn, self->kpageflags_fd, &flags), 0); + if (!(flags & KPF_UNEVICTABLE)) + SKIP(return, "could not make the page unevictable"); + + ASSERT_EQ(madvise(mapping, page_size, MADV_PAGEOUT), 0); + ASSERT_TRUE(pagemap_is_populated(self->pagemap_fd, mapping)); + ASSERT_EQ(mapping[0], 1); + + ASSERT_EQ(shmdt(mapping), 0); +} + /* Swap out an anonymous page, to see whether swap works at all. */ static bool probe_swap(int pagemap_fd) { diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h index b5d59729d432..95898e045bac 100644 --- a/tools/testing/selftests/mm/vm_util.h +++ b/tools/testing/selftests/mm/vm_util.h @@ -24,6 +24,7 @@ #define KPF_COMPOUND_HEAD BIT_ULL(15) #define KPF_COMPOUND_TAIL BIT_ULL(16) +#define KPF_UNEVICTABLE BIT_ULL(18) #define KPF_HWPOISON BIT_ULL(19) #define KPF_THP BIT_ULL(22) /* -- 2.56.0
