We must correctly update VMA anonymous page offset state on all VMA operations that would result in it changing, with special attention given to remapping.
We cover most cases by simply updating vma_set_range() to do so (with a new anonymous page offset parameter), but also notably must update the merging and mapping logic to propagate this parameter correctly. The remap logic remains the same - we may update the anonymous page offset if the VMA is unfaulted, but now this applies to MAP_PRIVATE file-backed mappings too, so we update the code to reflect this. Note that we use __linear_anon_page_index() upon remap as the VMA may be shared, in order that we update the field consistently regardless of VMA type. Similarly, pass through anon page offset to the merge logic, updating the vma_merge_struct struct to propagate it, and also use __linear_anon_page_index() to obtain the anonymous page index so it can be safely used for both shared and MAP_PRIVATE file-backed mappings. Finally, we update insert_vm_struct() to correctly set the anonymous page offset on insertion of a VMA. We simply ensure state is correctly propagated here, so no functional changes are intended. Also while we're here, replace a VM_BUG_ON_VMA() with a VM_WARN_ON_ONCE_VMA(). Also update VMA userland tests to reflect this change. Signed-off-by: Lorenzo Stoakes (ARM) <[email protected]> --- mm/mremap.c | 6 ++-- mm/vma.c | 47 +++++++++++++++++-------- mm/vma.h | 75 ++++++++++++++++++++++++---------------- mm/vma_exec.c | 2 +- tools/testing/vma/shared.c | 3 +- tools/testing/vma/tests/merge.c | 4 ++- tools/testing/vma/tests/vma.c | 4 +-- tools/testing/vma/vma_internal.h | 1 + 8 files changed, 90 insertions(+), 52 deletions(-) diff --git a/mm/mremap.c b/mm/mremap.c index b64aa1f6e07e..9ea1707eafa5 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -1265,7 +1265,9 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) static int copy_vma_and_data(struct vma_remap_struct *vrm, struct vm_area_struct **new_vma_ptr) { - const unsigned long new_pgoff = linear_page_index(vrm->vma, vrm->addr); + const pgoff_t new_pgoff = linear_page_index(vrm->vma, vrm->addr); + const pgoff_t new_anon_pgoff = + __linear_anon_page_index(vrm->vma, vrm->addr); struct vm_area_struct *vma = vrm->vma; struct vm_area_struct *new_vma; unsigned long moved_len; @@ -1273,7 +1275,7 @@ static int copy_vma_and_data(struct vma_remap_struct *vrm, PAGETABLE_MOVE(pmc, NULL, NULL, vrm->addr, vrm->new_addr, vrm->old_len); new_vma = copy_vma(&vma, vrm->new_addr, vrm->new_len, new_pgoff, - &pmc.need_rmap_locks); + new_anon_pgoff, &pmc.need_rmap_locks); if (!new_vma) { vrm_uncharge(vrm); *new_vma_ptr = NULL; diff --git a/mm/vma.c b/mm/vma.c index b5bc3eec961c..91bb1c6160a0 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -18,6 +18,7 @@ struct mmap_state { unsigned long addr; unsigned long end; pgoff_t pgoff; + pgoff_t anon_pgoff; unsigned long pglen; union { vm_flags_t vm_flags; @@ -46,13 +47,14 @@ struct mmap_state { bool file_doesnt_need_get :1; }; -#define MMAP_STATE(name, mm_, vmi_, addr_, len_, pgoff_, vma_flags_, file_) \ +#define MMAP_STATE(name, mm_, vmi_, addr_, len_, pgoff_, anon_pgoff_, vma_flags_, file_) \ struct mmap_state name = { \ .mm = mm_, \ .vmi = vmi_, \ .addr = addr_, \ .end = (addr_) + (len_), \ .pgoff = pgoff_, \ + .anon_pgoff = anon_pgoff_, \ .pglen = PHYS_PFN(len_), \ .vma_flags = vma_flags_, \ .file = file_, \ @@ -67,6 +69,7 @@ struct mmap_state { .end = (map_)->end, \ .vma_flags = (map_)->vma_flags, \ .pgoff = (map_)->pgoff, \ + .anon_pgoff = (map_)->anon_pgoff, \ .file = (map_)->file, \ .prev = (map_)->prev, \ .middle = vma_, \ @@ -82,10 +85,11 @@ static void __vma_set_range(struct vm_area_struct *vma, unsigned long start, } static void vma_set_range(struct vm_area_struct *vma, unsigned long start, - unsigned long end, pgoff_t pgoff) + unsigned long end, pgoff_t pgoff, pgoff_t anon_pgoff) { __vma_set_range(vma, start, end); vma_set_pgoff(vma, pgoff); + vma_set_anon_pgoff(vma, anon_pgoff); } /* Was this VMA ever forked from a parent, i.e. maybe contains CoW mappings? */ @@ -812,7 +816,8 @@ static int commit_merge(struct vma_merge_struct *vmg) */ vma_adjust_trans_huge(vma, vmg->start, vmg->end, vmg->__adjust_middle_start ? vmg->middle : NULL); - vma_set_range(vma, vmg->start, vmg->end, vmg_start_pgoff(vmg)); + vma_set_range(vma, vmg->start, vmg->end, vmg_start_pgoff(vmg), + vmg_start_anon_pgoff(vmg)); vmg_adjust_set_range(vmg); vma_iter_store_overwrite(vmg->vmi, vmg->target); @@ -982,6 +987,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( vmg->start = prev->vm_start; vmg->end = next->vm_end; vmg->pgoff = vma_start_pgoff(prev); + vmg->anon_pgoff = vma_start_anon_pgoff(prev); /* * We already ensured anon_vma compatibility above, so now it's @@ -1000,6 +1006,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( */ vmg->start = prev->vm_start; vmg->pgoff = vma_start_pgoff(prev); + vmg->anon_pgoff = vma_start_anon_pgoff(prev); if (!vmg->__remove_middle) vmg->__adjust_middle_start = true; @@ -1022,12 +1029,14 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( if (vmg->__remove_middle) { vmg->end = next->vm_end; vmg->pgoff = vma_start_pgoff(next) - pglen; + vmg->anon_pgoff = vma_start_anon_pgoff(next) - pglen; } else { /* We shrink middle and expand next. */ vmg->__adjust_next_start = true; vmg->start = middle->vm_start; vmg->end = start; vmg->pgoff = vma_start_pgoff(middle); + vmg->anon_pgoff = vma_start_anon_pgoff(middle); } err = dup_anon_vma(next, middle, &anon_dup); @@ -1137,6 +1146,7 @@ struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg) vmg->start = prev->vm_start; vmg->target = prev; vmg->pgoff = vma_start_pgoff(prev); + vmg->anon_pgoff = vma_start_anon_pgoff(prev); /* * If this merge would result in removal of the next VMA but we @@ -1908,7 +1918,7 @@ static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma) */ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, unsigned long addr, unsigned long len, pgoff_t pgoff, - bool *need_rmap_locks) + pgoff_t anon_pgoff, bool *need_rmap_locks) { struct vm_area_struct *vma = *vmap; unsigned long vma_start = vma->vm_start; @@ -1919,11 +1929,14 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, VMG_VMA_STATE(vmg, &vmi, NULL, vma, addr, addr + len); /* - * If anonymous vma has not yet been faulted, update new pgoff - * to match new location, to increase its chance of merging. + * If a vma has not yet been faulted, update its anonymous pgoff to + * match the new location to increase its chance of merging. */ - if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) { - pgoff = addr >> PAGE_SHIFT; + if (!vma->anon_vma && !vma_test(vma, VMA_SHARED_BIT)) { + anon_pgoff = addr >> PAGE_SHIFT; + + if (vma_is_anonymous(vma)) + pgoff = anon_pgoff; faulted_in_anon_vma = false; } @@ -1940,6 +1953,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, return NULL; /* should never get here */ vmg.pgoff = pgoff; + vmg.anon_pgoff = anon_pgoff; vmg.next = vma_iter_next_rewind(&vmi, NULL); new_vma = vma_merge_copied_range(&vmg); @@ -1961,7 +1975,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, * safe. It is only safe to keep the vm_pgoff * linear if there are no pages mapped yet. */ - VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma); + VM_WARN_ON_ONCE_VMA(faulted_in_anon_vma, new_vma); *vmap = vma = new_vma; } *need_rmap_locks = @@ -1970,7 +1984,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, new_vma = vm_area_dup(vma); if (!new_vma) goto out; - vma_set_range(new_vma, addr, addr + len, pgoff); + vma_set_range(new_vma, addr, addr + len, pgoff, anon_pgoff); if (vma_dup_policy(vma, new_vma)) goto out_free_vma; if (anon_vma_clone(new_vma, vma, VMA_OP_REMAP)) @@ -2612,7 +2626,7 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, if (is_anon) vma_set_anonymous(vma); - vma_set_range(vma, map->addr, map->end, map->pgoff); + vma_set_range(vma, map->addr, map->end, map->pgoff, map->anon_pgoff); vma->flags = map->vma_flags; vma->vm_page_prot = map->page_prot; @@ -2801,7 +2815,8 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, struct vm_area_struct *vma = NULL; bool have_mmap_prepare = file && file->f_op->mmap_prepare; VMA_ITERATOR(vmi, mm, addr); - MMAP_STATE(map, mm, &vmi, addr, len, pgoff, vma_flags, file); + const pgoff_t anon_pgoff = addr >> PAGE_SHIFT; + MMAP_STATE(map, mm, &vmi, addr, len, pgoff, anon_pgoff, vma_flags, file); struct vm_area_desc desc = { .mm = mm, .file = file, @@ -2946,6 +2961,7 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, unsigned long addr, unsigned long len, vma_flags_t vma_flags) { struct mm_struct *mm = current->mm; + const pgoff_t pgoff = addr >> PAGE_SHIFT; /* * Check against address space limits by the changed size @@ -2970,7 +2986,7 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, * occur after forking, so the expand will only happen on new VMAs. */ if (vma && vma->vm_end == addr) { - VMG_STATE(vmg, mm, vmi, addr, addr + len, vma_flags, PHYS_PFN(addr)); + VMG_STATE(vmg, mm, vmi, addr, addr + len, vma_flags, pgoff, pgoff); vmg.prev = vma; /* vmi is positioned at prev, which this mode expects. */ @@ -2990,7 +3006,7 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, goto unacct_fail; vma_set_anonymous(vma); - vma_set_range(vma, addr, addr + len, addr >> PAGE_SHIFT); + vma_set_range(vma, addr, addr + len, pgoff, pgoff); vma->flags = vma_flags; vma->vm_page_prot = vm_get_page_prot(vma_flags_to_legacy(vma_flags)); vma_start_write(vma); @@ -3382,6 +3398,7 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) WARN_ON_ONCE(vma->anon_vma); vma_set_pgoff(vma, vma->vm_start >> PAGE_SHIFT); } + vma_set_anon_pgoff(vma, vma->vm_start >> PAGE_SHIFT); if (vma_link(mm, vma)) { if (vma_test(vma, VMA_ACCOUNT_BIT)) @@ -3437,7 +3454,7 @@ struct vm_area_struct *__install_special_mapping( vma->vm_ops = ops; vma->vm_private_data = priv; - vma_set_range(vma, addr, addr + len, 0); + vma_set_range(vma, addr, addr + len, 0, addr >> PAGE_SHIFT); ret = insert_vm_struct(mm, vma); if (ret) diff --git a/mm/vma.h b/mm/vma.h index 54ed7c744e3b..024fabe63560 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -104,6 +104,7 @@ struct vma_merge_struct { unsigned long start; unsigned long end; pgoff_t pgoff; + pgoff_t anon_pgoff; union { /* Temporary while VMA flags are being converted. */ @@ -237,11 +238,6 @@ static inline bool vmg_nomem(struct vma_merge_struct *vmg) return vmg->state == VMA_MERGE_ERROR_NOMEM; } -static inline pgoff_t vmg_start_pgoff(const struct vma_merge_struct *vmg) -{ - return vmg->pgoff; -} - static inline pgoff_t vmg_pages(const struct vma_merge_struct *vmg) { const unsigned long size = vmg->end - vmg->start; @@ -249,6 +245,11 @@ static inline pgoff_t vmg_pages(const struct vma_merge_struct *vmg) return size >> PAGE_SHIFT; } +static inline pgoff_t vmg_start_pgoff(const struct vma_merge_struct *vmg) +{ + return vmg->pgoff; +} + static inline pgoff_t vmg_end_pgoff(const struct vma_merge_struct *vmg) { return vmg_start_pgoff(vmg) + vmg_pages(vmg); @@ -283,6 +284,16 @@ static inline void vma_set_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) vma->vm_pgoff = pgoff; } +static inline pgoff_t vmg_start_anon_pgoff(const struct vma_merge_struct *vmg) +{ + return vmg->anon_pgoff; +} + +static inline pgoff_t vmg_end_anon_pgoff(const struct vma_merge_struct *vmg) +{ + return vmg_start_anon_pgoff(vmg) + vmg_pages(vmg); +} + static inline void __vma_set_anon_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) { #ifdef CONFIG_64BIT @@ -301,44 +312,48 @@ static inline void vma_add_pgoff(struct vm_area_struct *vma, pgoff_t delta) { vma_assert_can_modify(vma); vma_set_pgoff(vma, vma_start_pgoff(vma) + delta); + vma_set_anon_pgoff(vma, vma_start_anon_pgoff(vma) + delta); } static inline void vma_sub_pgoff(struct vm_area_struct *vma, pgoff_t delta) { vma_assert_can_modify(vma); vma_set_pgoff(vma, vma_start_pgoff(vma) - delta); -} + vma_set_anon_pgoff(vma, vma_start_anon_pgoff(vma) - delta); +} + +#define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_, anon_pgoff_) \ + struct vma_merge_struct name = { \ + .mm = mm_, \ + .vmi = vmi_, \ + .start = start_, \ + .end = end_, \ + .vma_flags = vma_flags_, \ + .pgoff = pgoff_, \ + .anon_pgoff = anon_pgoff_, \ + .state = VMA_MERGE_START, \ + } -#define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_) \ +#define VMG_VMA_STATE(name, vmi_, prev_, vma_, start_, end_) \ struct vma_merge_struct name = { \ - .mm = mm_, \ + .mm = vma_->vm_mm, \ .vmi = vmi_, \ + .prev = prev_, \ + .middle = vma_, \ + .next = NULL, \ .start = start_, \ .end = end_, \ - .vma_flags = vma_flags_, \ - .pgoff = pgoff_, \ + .vm_flags = vma_->vm_flags, \ + .pgoff = linear_page_index(vma_, start_), \ + .anon_pgoff = __linear_anon_page_index(vma_, start_), \ + .file = vma_->vm_file, \ + .anon_vma = vma_->anon_vma, \ + .policy = vma_policy(vma_), \ + .uffd_ctx = vma_->vm_userfaultfd_ctx, \ + .anon_name = anon_vma_name(vma_), \ .state = VMA_MERGE_START, \ } -#define VMG_VMA_STATE(name, vmi_, prev_, vma_, start_, end_) \ - struct vma_merge_struct name = { \ - .mm = vma_->vm_mm, \ - .vmi = vmi_, \ - .prev = prev_, \ - .middle = vma_, \ - .next = NULL, \ - .start = start_, \ - .end = end_, \ - .vm_flags = vma_->vm_flags, \ - .pgoff = linear_page_index(vma_, start_), \ - .file = vma_->vm_file, \ - .anon_vma = vma_->anon_vma, \ - .policy = vma_policy(vma_), \ - .uffd_ctx = vma_->vm_userfaultfd_ctx, \ - .anon_name = anon_vma_name(vma_), \ - .state = VMA_MERGE_START, \ - } - #ifdef CONFIG_DEBUG_VM_MAPLE_TREE void validate_mm(struct mm_struct *mm); #else @@ -520,7 +535,7 @@ void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb, struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, unsigned long addr, unsigned long len, pgoff_t pgoff, - bool *need_rmap_locks); + pgoff_t anon_pgoff, bool *need_rmap_locks); struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma); diff --git a/mm/vma_exec.c b/mm/vma_exec.c index 7af1260689b9..586c52155942 100644 --- a/mm/vma_exec.c +++ b/mm/vma_exec.c @@ -41,7 +41,7 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) unsigned long new_end = old_end - shift; VMA_ITERATOR(vmi, mm, new_start); VMG_STATE(vmg, mm, &vmi, new_start, old_end, EMPTY_VMA_FLAGS, - vma_start_pgoff(vma)); + vma_start_pgoff(vma), vma_start_anon_pgoff(vma)); struct vm_area_struct *next; struct mmu_gather tlb; PAGETABLE_MOVE(pmc, vma, vma, old_start, new_start, length); diff --git a/tools/testing/vma/shared.c b/tools/testing/vma/shared.c index bea9ea6db02a..4a39c9d50489 100644 --- a/tools/testing/vma/shared.c +++ b/tools/testing/vma/shared.c @@ -23,7 +23,8 @@ struct vm_area_struct *alloc_vma(struct mm_struct *mm, vma->vm_start = start; vma->vm_end = end; - vma->vm_pgoff = pgoff; + vma_set_pgoff(vma, pgoff); + vma_set_anon_pgoff(vma, start >> PAGE_SHIFT); vma->flags = vma_flags; vma_assert_detached(vma); diff --git a/tools/testing/vma/tests/merge.c b/tools/testing/vma/tests/merge.c index e357accc8499..48418b82b01d 100644 --- a/tools/testing/vma/tests/merge.c +++ b/tools/testing/vma/tests/merge.c @@ -45,6 +45,7 @@ void vmg_set_range(struct vma_merge_struct *vmg, unsigned long start, vmg->start = start; vmg->end = end; vmg->pgoff = pgoff; + vmg->anon_pgoff = start >> PAGE_SHIFT; vmg->vma_flags = vma_flags; vmg->just_expand = false; @@ -108,6 +109,7 @@ static bool test_simple_merge(void) .end = 0x2000, .vma_flags = vma_flags, .pgoff = 1, + .anon_pgoff = 1, }; ASSERT_FALSE(attach_vma(&mm, vma_left)); @@ -1431,7 +1433,7 @@ static bool test_expand_only_mode(void) struct mm_struct mm = {}; VMA_ITERATOR(vmi, &mm, 0); struct vm_area_struct *vma_prev, *vma; - VMG_STATE(vmg, &mm, &vmi, 0x5000, 0x9000, vma_flags, 5); + VMG_STATE(vmg, &mm, &vmi, 0x5000, 0x9000, vma_flags, 5, 5); /* * Place a VMA prior to the one we're expanding so we assert that we do diff --git a/tools/testing/vma/tests/vma.c b/tools/testing/vma/tests/vma.c index 754a2da06321..7ca5289e0f95 100644 --- a/tools/testing/vma/tests/vma.c +++ b/tools/testing/vma/tests/vma.c @@ -38,7 +38,7 @@ static bool test_copy_vma(void) /* Move backwards and do not merge. */ vma = alloc_and_link_vma(&mm, 0x3000, 0x5000, 3, vma_flags); - vma_new = copy_vma(&vma, 0, 0x2000, 0, &need_locks); + vma_new = copy_vma(&vma, 0, 0x2000, 0, 3, &need_locks); ASSERT_NE(vma_new, vma); ASSERT_EQ(vma_new->vm_start, 0); ASSERT_EQ(vma_new->vm_end, 0x2000); @@ -51,7 +51,7 @@ static bool test_copy_vma(void) vma = alloc_and_link_vma(&mm, 0, 0x2000, 0, vma_flags); vma_next = alloc_and_link_vma(&mm, 0x6000, 0x8000, 6, vma_flags); - vma_new = copy_vma(&vma, 0x4000, 0x2000, 4, &need_locks); + vma_new = copy_vma(&vma, 0x4000, 0x2000, 4, 4, &need_locks); vma_assert_attached(vma_new); ASSERT_EQ(vma_new, vma_next); diff --git a/tools/testing/vma/vma_internal.h b/tools/testing/vma/vma_internal.h index 4f6c5666ac07..8a48b231aa7a 100644 --- a/tools/testing/vma/vma_internal.h +++ b/tools/testing/vma/vma_internal.h @@ -53,6 +53,7 @@ typedef __bitwise unsigned int vm_fault_t; #define VM_WARN_ON(_expr) (WARN_ON(_expr)) #define VM_WARN_ON_ONCE(_expr) (WARN_ON_ONCE(_expr)) +#define VM_WARN_ON_ONCE_VMA(_expr, _vma) (WARN_ON_ONCE(_expr)) #define VM_WARN_ON_VMG(_expr, _vmg) (WARN_ON(_expr)) #define VM_BUG_ON(_expr) (BUG_ON(_expr)) #define VM_BUG_ON_VMA(_expr, _vma) (BUG_ON(_expr)) -- 2.55.0
