[PATCH v3 8/8] mm/maps: execute PROCMAP_QUERY ioctl under RCU

[Suren Baghdasaryan]

Utilize speculative vma lookup to find and snapshot a vma without
taking mmap_lock during PROCMAP_QUERY ioctl execution. Concurrent
address space modifications are detected and the lookup is retried.
While we take the mmap_lock for reading during such contention, we
do that momentarily only to record new mm_wr_seq counter.
This change is designed to reduce mmap_lock contention and prevent
PROCMAP_QUERY ioctl calls (often a low priority task, such as
monitoring/data collection services) from blocking address space
updates.

Signed-off-by: Suren Baghdasaryan <sur...@google.com>
---
 fs/proc/task_mmu.c | 63 ++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 55 insertions(+), 8 deletions(-)

diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index f9d50a61167c..28b975ddff26 100644
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -525,9 +525,53 @@ static int pid_maps_open(struct inode *inode, struct file 
*file)
                PROCMAP_QUERY_VMA_FLAGS                         \
 )
 
-static int query_vma_setup(struct mm_struct *mm)
+#ifdef CONFIG_PER_VMA_LOCK
+
+static int query_vma_setup(struct proc_maps_private *priv)
 {
-       return mmap_read_lock_killable(mm);
+       if (!mmap_lock_speculate_try_begin(priv->mm, &priv->mm_wr_seq)) {
+               int ret = mmap_read_lock_killable(priv->mm);
+
+               if (ret)
+                       return ret;
+
+               /* mmap_lock_speculate_try_begin() succeeds when holding 
mmap_read_lock */
+               mmap_lock_speculate_try_begin(priv->mm, &priv->mm_wr_seq);
+               mmap_read_unlock(priv->mm);
+       }
+
+       memset(&priv->vma_copy, 0, sizeof(priv->vma_copy));
+       rcu_read_lock();
+
+       return 0;
+}
+
+static void query_vma_teardown(struct mm_struct *mm, struct vm_area_struct 
*vma)
+{
+       rcu_read_unlock();
+}
+
+static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_private 
*priv,
+                                                    unsigned long addr)
+{
+       struct vm_area_struct *vma;
+
+       vma_iter_init(&priv->iter, priv->mm, addr);
+       vma = vma_next(&priv->iter);
+       if (!vma)
+               return NULL;
+
+       vma = get_stable_vma(vma, priv, addr);
+
+       /* The only possible error is EINTR, just pretend we found nothing */
+       return IS_ERR(vma) ? NULL : vma;
+}
+
+#else /* CONFIG_PER_VMA_LOCK */
+
+static int query_vma_setup(struct proc_maps_private *priv)
+{
+       return mmap_read_lock_killable(priv->mm);
 }
 
 static void query_vma_teardown(struct mm_struct *mm, struct vm_area_struct 
*vma)
@@ -535,18 +579,21 @@ static void query_vma_teardown(struct mm_struct *mm, 
struct vm_area_struct *vma)
        mmap_read_unlock(mm);
 }
 
-static struct vm_area_struct *query_vma_find_by_addr(struct mm_struct *mm, 
unsigned long addr)
+static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_private 
*priv,
+                                                    unsigned long addr)
 {
-       return find_vma(mm, addr);
+       return find_vma(priv->mm, addr);
 }
 
-static struct vm_area_struct *query_matching_vma(struct mm_struct *mm,
+#endif /* CONFIG_PER_VMA_LOCK */
+
+static struct vm_area_struct *query_matching_vma(struct proc_maps_private 
*priv,
                                                 unsigned long addr, u32 flags)
 {
        struct vm_area_struct *vma;
 
 next_vma:
-       vma = query_vma_find_by_addr(mm, addr);
+       vma = query_vma_find_by_addr(priv, addr);
        if (!vma)
                goto no_vma;
 
@@ -622,13 +669,13 @@ static int do_procmap_query(struct proc_maps_private 
*priv, void __user *uarg)
        if (!mm || !mmget_not_zero(mm))
                return -ESRCH;
 
-       err = query_vma_setup(mm);
+       err = query_vma_setup(priv);
        if (err) {
                mmput(mm);
                return err;
        }
 
-       vma = query_matching_vma(mm, karg.query_addr, karg.query_flags);
+       vma = query_matching_vma(priv, karg.query_addr, karg.query_flags);
        if (IS_ERR(vma)) {
                err = PTR_ERR(vma);
                vma = NULL;
-- 
2.49.0.805.g082f7c87e0-goog