When automatic NUMA balancing marks a PTE or PMD as protnone (PAGE_NONE), hmm_pte_need_fault() requests a fault to restore access permissions, including _PAGE_RW when write access is needed.
Calling handle_mm_fault() from hmm_do_fault() currently triggers NUMA balancing in do_numa_page() and do_huge_pmd_numa_page(). When hmm_range_fault() runs from a device driver or kworker context (such as a GPU SVM or userptr rebind worker), folios get migrated toward the CPU node executing the remote fault. Also, each folio migration invokes try_to_migrate_one() -> mmu_notifier_invalidate_range_start(), which invalidates the caller's mmu_interval_notifier while holding mmap_lock. That causes hmm_range_fault() to abort with -EBUSY after every 2 MiB PMD and exhaust its retry timeout on large buffers, which has been observed with large imported Xe userptr buffers. Add FAULT_FLAG_NO_NUMA and set it in hmm_do_fault() so protnone entries restore PTE/PMD permissions in-place without performing NUMA balancing, matching GUP's default !FOLL_HONOR_NUMA_FAULT behaviour. Signed-off-by: Matthew Auld <[email protected]> Cc: Thomas Hellström <[email protected]> Cc: Matthew Brost <[email protected]> Cc: David Hildenbrand <[email protected]> Cc: Mika Penttilä <[email protected]> Cc: Leon Romanovsky <[email protected]> Cc: Jason Gunthorpe <[email protected]> --- include/linux/mm.h | 3 ++- include/linux/mm_types.h | 3 +++ mm/hmm.c | 2 +- mm/huge_memory.c | 3 +++ mm/memory.c | 3 +++ 5 files changed, 12 insertions(+), 2 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 77b5d450fe45..4b6ad3c0b591 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -717,7 +717,8 @@ static inline bool fault_flag_allow_retry_first(enum fault_flag flags) { FAULT_FLAG_REMOTE, "REMOTE" }, \ { FAULT_FLAG_INSTRUCTION, "INSTRUCTION" }, \ { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" }, \ - { FAULT_FLAG_VMA_LOCK, "VMA_LOCK" } + { FAULT_FLAG_VMA_LOCK, "VMA_LOCK" }, \ + { FAULT_FLAG_NO_NUMA, "NO_NUMA" } /* * vm_fault is filled by the pagefault handler and passed to the vma's diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 1ed35d258d2e..2ea71c1bb170 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -1819,6 +1819,8 @@ enum tlb_flush_reason { * @FAULT_FLAG_ORIG_PTE_VALID: whether the fault has vmf->orig_pte cached. * We should only access orig_pte if this flag set. * @FAULT_FLAG_VMA_LOCK: The fault is handled under VMA lock. + * @FAULT_FLAG_NO_NUMA: Do not perform NUMA balancing migration or NUMA + * task access accounting on protnone faults. * * About @FAULT_FLAG_ALLOW_RETRY and @FAULT_FLAG_TRIED: we can specify * whether we would allow page faults to retry by specifying these two @@ -1857,6 +1859,7 @@ enum fault_flag { FAULT_FLAG_UNSHARE = 1 << 10, FAULT_FLAG_ORIG_PTE_VALID = 1 << 11, FAULT_FLAG_VMA_LOCK = 1 << 12, + FAULT_FLAG_NO_NUMA = 1 << 13, }; typedef unsigned int __bitwise zap_flags_t; diff --git a/mm/hmm.c b/mm/hmm.c index 2f1e98c6b644..62cdeb071781 100644 --- a/mm/hmm.c +++ b/mm/hmm.c @@ -648,7 +648,7 @@ static int hmm_do_fault(struct mm_struct *mm, unsigned long addr = hmm_vma_walk->last; unsigned long end = hmm_vma_walk->end; unsigned int required_fault = hmm_vma_walk->required_fault; - unsigned int fault_flags = FAULT_FLAG_REMOTE; + unsigned int fault_flags = FAULT_FLAG_REMOTE | FAULT_FLAG_NO_NUMA; struct vm_area_struct *vma; if (hmm_vma_walk->locked) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 0d589be136d2..ea380bf45df0 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2361,6 +2361,9 @@ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) if (!folio) goto out_map; + if (vmf->flags & FAULT_FLAG_NO_NUMA) + goto out_map; + nid = folio_nid(folio); target_nid = numa_migrate_check(folio, vmf, haddr, &flags, writable, diff --git a/mm/memory.c b/mm/memory.c index fe72e66c9c2e..53c819a6e49a 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -6404,6 +6404,9 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf) if (!folio || folio_is_zone_device(folio)) goto out_map; + if (vmf->flags & FAULT_FLAG_NO_NUMA) + goto out_map; + nid = folio_nid(folio); nr_pages = folio_nr_pages(folio); --- base-commit: 4318887a3efa4c53d6f083ccfd7105b809a3ac65 change-id: 20261112-numa-3649f4bf57c9
