From: Wandun Chen <[email protected]>
In RT kernels, sysctl_compact_unevictable_allowed is false by default,
when the mlock/mlockall system call try to lock all the present page,
the mlock_pte_range function skips non-present entries. If these
non-present entries are migration entries, and the migration is not
guaranteed to have completed before the mlock/mlockall, it may result
in a page fault on subsequent access, which then waits for the
migration to finish, causing spike latency in RT kernels.
Fix it by waiting for the migration to complete during the mlock/mlockall
syscall when sysctl_compact_unevictable_allowed is false.
Fixes: 90d07210ab55 ("mm: mlock: use folios and a folio batch internally")
Suggested-by: Vlastimil Babka <[email protected]>
Signed-off-by: Wandun Chen <[email protected]>
Link:
https://lore.kernel.org/lkml/[email protected]/#t
---
mm/mlock.c | 23 +++++++++++++++++++++--
1 file changed, 21 insertions(+), 2 deletions(-)
diff --git a/mm/mlock.c b/mm/mlock.c
index 97e49038d8d3..ac65de40b22b 100644
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -25,6 +25,7 @@
#include <linux/memcontrol.h>
#include <linux/mm_inline.h>
#include <linux/secretmem.h>
+#include <linux/compaction.h>
#include "internal.h"
@@ -361,8 +362,17 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr,
ptl = pmd_trans_huge_lock(pmd, vma);
if (ptl) {
- if (!pmd_present(*pmd))
+ if (!pmd_present(*pmd)) {
+ if (unlikely((vma->vm_flags & VM_LOCKED) &&
+ !compaction_allow_unevictable() &&
+ softleaf_is_migration(softleaf_from_pmd(*pmd)))) {
+ spin_unlock(ptl);
+ pmd_migration_entry_wait(vma->vm_mm, pmd);
+ walk->action = ACTION_AGAIN;
+ return 0;
+ }
goto out;
+ }
if (is_huge_zero_pmd(*pmd))
goto out;
folio = pmd_folio(*pmd);
@@ -383,8 +393,17 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr,
for (pte = start_pte; addr != end; pte++, addr += PAGE_SIZE) {
ptent = ptep_get(pte);
- if (!pte_present(ptent))
+ if (!pte_present(ptent)) {
+ if (unlikely((vma->vm_flags & VM_LOCKED) &&
+ !compaction_allow_unevictable() &&
+ softleaf_is_migration(softleaf_from_pte(ptent)))) {
+ pte_unmap_unlock(start_pte, ptl);
+ migration_entry_wait(vma->vm_mm, pmd, addr);
+ walk->action = ACTION_AGAIN;
+ return 0;
+ }
continue;
+ }
folio = vm_normal_folio(vma, addr, ptent);
if (!folio || folio_is_zone_device(folio))
continue;
--
2.43.0
