Add observability for the deferred teardown path:
- vmstat (CONFIG_ASYNC_MM_TEARDOWN, in /proc/vmstat):
async_mm_teardown_queued - deferred to the mm_reaper kthread
async_mm_teardown_sync - torn down inline on the exit path:
feature off, mm below the RSS
threshold, MMF_OOM_SKIP or
MMF_OOM_TARGETED set, or rejected by
backpressure. Counts only teardowns
reaching mmput_exit(); final drops via
mmput()/mmput_async() elsewhere in the
kernel are inline but not counted here.
async_mm_teardown_rejected - subset of the above that was eligible
but lost to async_mm_teardown_reserve()
(counted for both this and sync, so
sync stays the total for the exit path)
- tracepoints in a new include/trace/events/mm_reaper.h:
mm_async_teardown_queue at enqueue, and
mm_async_teardown_reap when the reaper picks an mm up, just before
__mmput(). queue carries the mm, its RSS,
and the exiting task's pid/comm, captured
while current is still that task, plus the
node of the exiting task. reap carries the
mm, the charged RSS (mm->async_reap_rss, the
amount mmput_exit() reserved and what the
pending-pages accounting releases), a fresh
live RSS read taken just before __mmput(),
and the node of the reaper. The gap between
charged and live measures how much reclaim
ate out of the queue before the reaper got
to it.
reap is emitted before __mmput() because the final mmdrop() inside
__mmput() may free the mm.
Signed-off-by: Aditya Sharma <[email protected]>
---
include/linux/vm_event_item.h | 5 +++
include/trace/events/mm_reaper.h | 73 ++++++++++++++++++++++++++++++++
kernel/fork.c | 14 +++++-
mm/vmstat.c | 5 +++
4 files changed, 95 insertions(+), 2 deletions(-)
create mode 100644 include/trace/events/mm_reaper.h
diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index 2628ccda0..de4dc20b7 100644
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -179,6 +179,11 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
NRSWPIN,
NRSWPOUT,
#endif /* CONFIG_SWAP */
+#ifdef CONFIG_ASYNC_MM_TEARDOWN
+ ASYNC_MM_TEARDOWN_QUEUED,
+ ASYNC_MM_TEARDOWN_SYNC,
+ ASYNC_MM_TEARDOWN_REJECTED,
+#endif
NR_VM_EVENT_ITEMS
};
diff --git a/include/trace/events/mm_reaper.h b/include/trace/events/mm_reaper.h
new file mode 100644
index 000000000..315bced7c
--- /dev/null
+++ b/include/trace/events/mm_reaper.h
@@ -0,0 +1,73 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM mm_reaper
+
+#if !defined(_TRACE_MM_REAPER_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_MM_REAPER_H
+
+#include <linux/tracepoint.h>
+#include <linux/sched.h>
+#include <linux/topology.h>
+
+TRACE_EVENT(mm_async_teardown_queue,
+
+ TP_PROTO(struct mm_struct *mm, unsigned long rss),
+
+ TP_ARGS(mm, rss),
+
+ TP_STRUCT__entry(
+ __field(struct mm_struct *, mm)
+ __field(int, pid)
+ __array(char, comm, TASK_COMM_LEN)
+ __field(unsigned long, rss)
+ __field(int, node)
+ ),
+
+ TP_fast_assign(
+ __entry->mm = mm;
+ __entry->pid = current->pid;
+ memcpy(__entry->comm, current->comm, TASK_COMM_LEN);
+ __entry->rss = rss;
+ __entry->node = numa_node_id();
+ ),
+
+ TP_printk("mm=%p pid=%d comm=%s rss=%lukB node=%d",
+ __entry->mm,
+ __entry->pid,
+ __entry->comm,
+ __entry->rss << (PAGE_SHIFT - 10),
+ __entry->node
+ )
+);
+
+TRACE_EVENT(mm_async_teardown_reap,
+
+ TP_PROTO(struct mm_struct *mm, unsigned long charged_rss, unsigned long
live_rss),
+
+ TP_ARGS(mm, charged_rss, live_rss),
+
+ TP_STRUCT__entry(
+ __field(struct mm_struct *, mm)
+ __field(unsigned long, charged_rss)
+ __field(unsigned long, live_rss)
+ __field(int, node)
+ ),
+
+ TP_fast_assign(
+ __entry->mm = mm;
+ __entry->charged_rss = charged_rss;
+ __entry->live_rss = live_rss;
+ __entry->node = numa_node_id();
+ ),
+
+ TP_printk("mm=%p charged_rss=%lukB live_rss=%lukB node=%d",
+ __entry->mm,
+ __entry->charged_rss << (PAGE_SHIFT - 10),
+ __entry->live_rss << (PAGE_SHIFT - 10),
+ __entry->node
+ )
+);
+
+#endif /* _TRACE_MM_REAPER_H */
+
+#include <trace/define_trace.h>
diff --git a/kernel/fork.c b/kernel/fork.c
index 0976770db..4bdffa188 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -126,6 +126,9 @@
#define CREATE_TRACE_POINTS
#include <trace/events/task.h>
+#ifdef CONFIG_ASYNC_MM_TEARDOWN
+#include <trace/events/mm_reaper.h>
+#endif
#include <kunit/visibility.h>
@@ -3461,8 +3464,10 @@ static bool async_mm_teardown_reserve(unsigned long rss)
return true;
}
-static void async_mm_teardown_queue(struct mm_struct *mm)
+static void async_mm_teardown_queue(struct mm_struct *mm, unsigned long rss)
{
+ count_vm_event(ASYNC_MM_TEARDOWN_QUEUED);
+ trace_mm_async_teardown_queue(mm, rss);
if (llist_add(&mm->async_reap_node, &mm_reaper_list))
wake_up(&mm_reaper_wait);
}
@@ -3478,6 +3483,9 @@ static int mm_reaper(void *unused)
batch = llist_del_all(&mm_reaper_list);
llist_for_each_entry_safe(mm, n, batch, async_reap_node) {
unsigned long pages = mm->async_reap_rss;
+ unsigned long live = get_mm_rss(mm);
+
+ trace_mm_async_teardown_reap(mm, pages, live);
__mmput(mm); /* may free mm via mmdrop */
atomic_long_sub(pages, &mm_reaper_pending_pages);
@@ -3515,12 +3523,14 @@ void mmput_exit(struct mm_struct *mm)
if (async_mm_teardown_reserve(rss)) {
mm->async_reap_rss = rss;
- async_mm_teardown_queue(mm);
+ async_mm_teardown_queue(mm, rss);
return;
}
+ count_vm_event(ASYNC_MM_TEARDOWN_REJECTED);
}
}
+ count_vm_event(ASYNC_MM_TEARDOWN_SYNC);
__mmput(mm);
}
diff --git a/mm/vmstat.c b/mm/vmstat.c
index 4e26e5fd6..e3ff30027 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1494,6 +1494,11 @@ const char * const vmstat_text[] = {
[I(NRSWPIN)] = "nrswpin",
[I(NRSWPOUT)] = "nrswpout",
#endif /* CONFIG_SWAP */
+#ifdef CONFIG_ASYNC_MM_TEARDOWN
+ [I(ASYNC_MM_TEARDOWN_QUEUED)] = "async_mm_teardown_queued",
+ [I(ASYNC_MM_TEARDOWN_SYNC)] = "async_mm_teardown_sync",
+ [I(ASYNC_MM_TEARDOWN_REJECTED)] = "async_mm_teardown_rejected",
+#endif
#undef I
#endif /* CONFIG_VM_EVENT_COUNTERS */
};
--
2.34.1
