[PATCH 34/41] x86/mm: try VMA lock-based page fault handling first

[Suren Baghdasaryan]

Attempt VMA lock-based page fault handling first, and fall back to the
existing mmap_lock-based handling if that fails.

Signed-off-by: Suren Baghdasaryan <sur...@google.com>
---
 arch/x86/Kconfig    |  1 +
 arch/x86/mm/fault.c | 36 ++++++++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 3604074a878b..3647f7bdb110 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -27,6 +27,7 @@ config X86_64
        # Options that are inherently 64-bit kernel only:
        select ARCH_HAS_GIGANTIC_PAGE
        select ARCH_SUPPORTS_INT128 if CC_HAS_INT128
+       select ARCH_SUPPORTS_PER_VMA_LOCK
        select ARCH_USE_CMPXCHG_LOCKREF
        select HAVE_ARCH_SOFT_DIRTY
        select MODULES_USE_ELF_RELA
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index 7b0d4ab894c8..983266e7c49b 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -19,6 +19,7 @@
 #include <linux/uaccess.h>             /* faulthandler_disabled()      */
 #include <linux/efi.h>                 /* 
efi_crash_gracefully_on_page_fault()*/
 #include <linux/mm_types.h>
+#include <linux/mm.h>                  /* find_and_lock_vma() */
 
 #include <asm/cpufeature.h>            /* boot_cpu_has, ...            */
 #include <asm/traps.h>                 /* dotraplinkage, ...           */
@@ -1354,6 +1355,38 @@ void do_user_addr_fault(struct pt_regs *regs,
        }
 #endif
 
+#ifdef CONFIG_PER_VMA_LOCK
+       if (!(flags & FAULT_FLAG_USER) || atomic_read(&mm->mm_users) == 1)
+               goto lock_mmap;
+
+       vma = lock_vma_under_rcu(mm, address);
+       if (!vma)
+               goto lock_mmap;
+
+       if (unlikely(access_error(error_code, vma))) {
+               vma_read_unlock(vma);
+               goto lock_mmap;
+       }
+       fault = handle_mm_fault(vma, address, flags | FAULT_FLAG_VMA_LOCK, 
regs);
+       vma_read_unlock(vma);
+
+       if (!(fault & VM_FAULT_RETRY)) {
+               count_vm_vma_lock_event(VMA_LOCK_SUCCESS);
+               goto done;
+       }
+       count_vm_vma_lock_event(VMA_LOCK_RETRY);
+
+       /* Quick path to respond to signals */
+       if (fault_signal_pending(fault, regs)) {
+               if (!user_mode(regs))
+                       kernelmode_fixup_or_oops(regs, error_code, address,
+                                                SIGBUS, BUS_ADRERR,
+                                                ARCH_DEFAULT_PKEY);
+               return;
+       }
+lock_mmap:
+#endif /* CONFIG_PER_VMA_LOCK */
+
        /*
         * Kernel-mode access to the user address space should only occur
         * on well-defined single instructions listed in the exception
@@ -1454,6 +1487,9 @@ void do_user_addr_fault(struct pt_regs *regs,
        }
 
        mmap_read_unlock(mm);
+#ifdef CONFIG_PER_VMA_LOCK
+done:
+#endif
        if (likely(!(fault & VM_FAULT_ERROR)))
                return;
 
-- 
2.39.0