[PATCH] mm/THP: use hugepage_vma_check() in khugepaged_enter_vma_merge()

[Song Liu]

khugepaged_enter_vma_merge() is using a different approach to check
whether a vma is valid for khugepaged_enter():

    if (!vma->anon_vma)
            /*
             * Not yet faulted in so we will register later in the
             * page fault if needed.
             */
            return 0;
    if (vma->vm_ops || (vm_flags & VM_NO_KHUGEPAGED))
            /* khugepaged not yet working on file or special mappings */
            return 0;

This check has some problems. One of the obvious problems is that
it doesn't check shmem_file(), so that vma backed with shmem files
will not call khugepaged_enter().

This patch fixes these problems by reusing hugepage_vma_check() in
khugepaged_enter_vma_merge().

Signed-off-by: Song Liu <songliubrav...@fb.com>
---
 mm/khugepaged.c | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index d7b2a4b..e50c2bd 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -430,18 +430,14 @@ int __khugepaged_enter(struct mm_struct *mm)
        return 0;
 }
 
+static bool hugepage_vma_check(struct vm_area_struct *vma);
+
 int khugepaged_enter_vma_merge(struct vm_area_struct *vma,
                               unsigned long vm_flags)
 {
        unsigned long hstart, hend;
-       if (!vma->anon_vma)
-               /*
-                * Not yet faulted in so we will register later in the
-                * page fault if needed.
-                */
-               return 0;
-       if (vma->vm_ops || (vm_flags & VM_NO_KHUGEPAGED))
-               /* khugepaged not yet working on file or special mappings */
+
+       if (!hugepage_vma_check(vma))
                return 0;
        hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
        hend = vma->vm_end & HPAGE_PMD_MASK;
-- 
2.9.5