From: Wandun Chen <[email protected]>
When compact_unevictable_allowed=0, unevictable pages should not be
migrated. However, mlock_folio_batch in the mlock[all] syscall introduces
a race, mlock_folio() sets PG_mlocked immediately but defers PG_unevictable
to mlock_folio_batch(), causing pages that are about to become unevictable
to be migrated, which violates the intent of compact_unevictable_allowed,
and causes spike latency in RT kernels [1].
In order to fix this, migration is forbidden for pages mapped into VMAs
marked with VM_LOCKED. In addition, two early-return paths are introduced,
filter out mlocked pages, return early to avoid unnecessary operations.
Fixes: 90d07210ab55 ("mm: mlock: use folios and a folio batch internally")
Reported-by: Alexander Krabler <[email protected]>
Closes:
https://lore.kernel.org/all/du0pr01mb10385345f7153f3341009818882...@du0pr01mb10385.eurprd01.prod.exchangelabs.com/
[1]
Suggested-by: Vlastimil Babka <[email protected]>
Signed-off-by: Wandun Chen <[email protected]>
Link:
https://lore.kernel.org/linux-rt-users/[email protected]/#t
---
include/linux/compaction.h | 6 ++++++
include/linux/rmap.h | 3 +++
mm/compaction.c | 8 +++++++-
mm/migrate.c | 23 +++++++++++++++++++----
mm/rmap.c | 12 +++++++++---
5 files changed, 44 insertions(+), 8 deletions(-)
diff --git a/include/linux/compaction.h b/include/linux/compaction.h
index f29ef0653546..04e60f65b976 100644
--- a/include/linux/compaction.h
+++ b/include/linux/compaction.h
@@ -106,6 +106,7 @@ bool compaction_zonelist_suitable(struct alloc_context *ac,
int order,
extern void __meminit kcompactd_run(int nid);
extern void __meminit kcompactd_stop(int nid);
extern void wakeup_kcompactd(pg_data_t *pgdat, int order, int highest_zoneidx);
+extern bool compaction_allow_unevictable(void);
#else
static inline void reset_isolation_suitable(pg_data_t *pgdat)
@@ -131,6 +132,11 @@ static inline void wakeup_kcompactd(pg_data_t *pgdat,
{
}
+static inline bool compaction_allow_unevictable(void)
+{
+ return true;
+}
+
#endif /* CONFIG_COMPACTION */
struct node;
diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index 8dc0871e5f00..359c7426b6b9 100644
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -102,6 +102,9 @@ enum ttu_flags {
* do a final flush if necessary */
TTU_RMAP_LOCKED = 0x80, /* do not grab rmap lock:
* caller holds it */
+ TTU_RESPECT_MLOCK = 0x100,/* leave VM_LOCKED vmas mapped instead
+ * of installing a migration entry
+ */
};
#ifdef CONFIG_MMU
diff --git a/mm/compaction.c b/mm/compaction.c
index f08765ade014..5d256930e389 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -1116,7 +1116,8 @@ isolate_migratepages_block(struct compact_control *cc,
unsigned long low_pfn,
is_unevictable = folio_test_unevictable(folio);
/* Compaction might skip unevictable pages but CMA takes them */
- if (!(mode & ISOLATE_UNEVICTABLE) && is_unevictable)
+ if (!(mode & ISOLATE_UNEVICTABLE) &&
+ (is_unevictable || folio_test_mlocked(folio)))
goto isolate_fail_put;
/*
@@ -1898,6 +1899,11 @@ typedef enum {
* compactable pages.
*/
static int sysctl_compact_unevictable_allowed __read_mostly =
CONFIG_COMPACT_UNEVICTABLE_DEFAULT;
+
+bool compaction_allow_unevictable(void)
+{
+ return sysctl_compact_unevictable_allowed;
+}
/*
* Tunable for proactive compaction. It determines how
* aggressively the kernel should compact memory in the
diff --git a/mm/migrate.c b/mm/migrate.c
index a786549551e3..3a15eb13e82b 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -1202,7 +1202,7 @@ static void migrate_folio_done(struct folio *src,
static int migrate_folio_unmap(new_folio_t get_new_folio,
free_folio_t put_new_folio, unsigned long private,
struct folio *src, struct folio **dstp, enum migrate_mode mode,
- struct list_head *ret)
+ struct list_head *ret, enum migrate_reason reason)
{
struct folio *dst;
int rc = -EAGAIN;
@@ -1210,6 +1210,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
struct anon_vma *anon_vma = NULL;
bool locked = false;
bool dst_locked = false;
+ enum ttu_flags ttu = 0;
dst = get_new_folio(src, private);
if (!dst)
@@ -1249,9 +1250,15 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
folio_lock(src);
}
locked = true;
- if (folio_test_mlocked(src))
+ if (folio_test_mlocked(src)) {
old_folio_state |= FOLIO_WAS_MLOCKED;
+ if (reason == MR_COMPACTION && !compaction_allow_unevictable())
{
+ rc = -EBUSY;
+ goto out;
+ }
+ }
+
if (folio_test_writeback(src)) {
/*
* Only in the case of a full synchronous migration is it
@@ -1324,7 +1331,14 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
/* Establish migration ptes */
VM_BUG_ON_FOLIO(folio_test_anon(src) &&
!folio_test_ksm(src) && !anon_vma, src);
- try_to_migrate(src, mode == MIGRATE_ASYNC ? TTU_BATCH_FLUSH :
0);
+
+ if (mode == MIGRATE_ASYNC)
+ ttu |= TTU_BATCH_FLUSH;
+
+ if (reason == MR_COMPACTION && !compaction_allow_unevictable())
+ ttu |= TTU_RESPECT_MLOCK;
+
+ try_to_migrate(src, ttu);
old_folio_state |= FOLIO_WAS_MAPPED;
}
@@ -1905,7 +1919,8 @@ static int migrate_pages_batch(struct list_head *from,
}
rc = migrate_folio_unmap(get_new_folio, put_new_folio,
- private, folio, &dst, mode, ret_folios);
+ private, folio, &dst, mode, ret_folios,
+ reason);
/*
* The rules are:
* 0: folio will be put on unmap_folios list,
diff --git a/mm/rmap.c b/mm/rmap.c
index 0fb7a1b82cf3..3cb7f6337d38 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -2420,6 +2420,9 @@ static bool try_to_migrate_one(struct folio *folio,
struct vm_area_struct *vma,
unsigned long pfn;
unsigned long hsz = 0;
+ if ((flags & TTU_RESPECT_MLOCK) && (vma->vm_flags & VM_LOCKED))
+ return false;
+
/*
* When racing against e.g. zap_pte_range() on another cpu,
* in between its ptep_get_and_clear_full() and folio_remove_rmap_*(),
@@ -2741,11 +2744,14 @@ void try_to_migrate(struct folio *folio, enum ttu_flags
flags)
};
/*
- * Migration always ignores mlock and only supports TTU_RMAP_LOCKED and
- * TTU_SPLIT_HUGE_PMD, TTU_SYNC, and TTU_BATCH_FLUSH flags.
+ * Migration normally ignores mlock, but TTU_RESPECT_MLOCK asks it to
+ * leave folios mapped into VM_LOCKED vmas alone. Only TTU_RMAP_LOCKED,
+ * TTU_SPLIT_HUGE_PMD, TTU_SYNC, TTU_BATCH_FLUSH and TTU_RESPECT_MLOCK
+ * are supported.
*/
if (WARN_ON_ONCE(flags & ~(TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD |
- TTU_SYNC | TTU_BATCH_FLUSH)))
+ TTU_SYNC | TTU_BATCH_FLUSH |
+ TTU_RESPECT_MLOCK)))
return;
if (folio_is_zone_device(folio) &&
--
2.43.0
