[PATCH v3 05/28] mm: gup: allow VM_FAULT_RETRY for multiple times

[Peter Xu]

This is the gup counterpart of the change that allows the VM_FAULT_RETRY
to happen for more than once.

Reviewed-by: Jerome Glisse <jgli...@redhat.com>
Signed-off-by: Peter Xu <pet...@redhat.com>
---
 mm/gup.c     | 17 +++++++++++++----
 mm/hugetlb.c |  6 ++++--
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/mm/gup.c b/mm/gup.c
index 9bb3bed68ee3..f56dee055f26 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -528,7 +528,10 @@ static int faultin_page(struct task_struct *tsk, struct 
vm_area_struct *vma,
        if (*flags & FOLL_NOWAIT)
                fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT;
        if (*flags & FOLL_TRIED) {
-               VM_WARN_ON_ONCE(fault_flags & FAULT_FLAG_ALLOW_RETRY);
+               /*
+                * Note: FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_TRIED
+                * can co-exist
+                */
                fault_flags |= FAULT_FLAG_TRIED;
        }
 
@@ -943,17 +946,23 @@ static __always_inline long 
__get_user_pages_locked(struct task_struct *tsk,
                /* VM_FAULT_RETRY triggered, so seek to the faulting offset */
                pages += ret;
                start += ret << PAGE_SHIFT;
+               lock_dropped = true;
 
+retry:
                /*
                 * Repeat on the address that fired VM_FAULT_RETRY
-                * without FAULT_FLAG_ALLOW_RETRY but with
+                * with both FAULT_FLAG_ALLOW_RETRY and
                 * FAULT_FLAG_TRIED.
                 */
                *locked = 1;
-               lock_dropped = true;
                down_read(&mm->mmap_sem);
                ret = __get_user_pages(tsk, mm, start, 1, flags | FOLL_TRIED,
-                                      pages, NULL, NULL);
+                                      pages, NULL, locked);
+               if (!*locked) {
+                       /* Continue to retry until we succeeded */
+                       BUG_ON(ret != 0);
+                       goto retry;
+               }
                if (ret != 1) {
                        BUG_ON(ret > 1);
                        if (!pages_done)
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 52296ce4025a..040779a7b906 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -4267,8 +4267,10 @@ long follow_hugetlb_page(struct mm_struct *mm, struct 
vm_area_struct *vma,
                                fault_flags |= FAULT_FLAG_ALLOW_RETRY |
                                        FAULT_FLAG_RETRY_NOWAIT;
                        if (flags & FOLL_TRIED) {
-                               VM_WARN_ON_ONCE(fault_flags &
-                                               FAULT_FLAG_ALLOW_RETRY);
+                               /*
+                                * Note: FAULT_FLAG_ALLOW_RETRY and
+                                * FAULT_FLAG_TRIED can co-exist
+                                */
                                fault_flags |= FAULT_FLAG_TRIED;
                        }
                        ret = hugetlb_fault(mm, vma, vaddr, fault_flags);
-- 
2.17.1