From: "Kiryl Shutsemau (Meta)" <[email protected]> The engine decides per candidate, and every one of those decisions is currently invisible: the mechanism it is about to replace reports through mm_collapse_huge_page_isolate, which the engine never calls. Switching the anonymous path over without something in its place would take existing tracing with it.
Add one tracepoint, mm_collapse_candidate: a window's address and order, the pass that reached a verdict on it, and what that verdict was. Every candidate a pass judged produces exactly one -- the pass that refused it, or the install for one that made it. A candidate the round gave up on before any pass judged it produces none. That is enough to follow a round: which windows were attempted, and which ones the batch dropped and where. It is also what a scan of the trace buffer can attribute to an address. It goes in the huge_memory trace system, next to the events it stands in for, so a consumer enabling that system keeps seeing collapses. Assisted-by: Claude-Code:claude-opus-5 Signed-off-by: Kiryl Shutsemau (Meta) <[email protected]> --- include/trace/events/huge_memory.h | 40 ++++++++++++++++++++++++++++++ mm/collapse.c | 32 +++++++++++++++++++++++- mm/collapse.h | 13 ++++++++++ 3 files changed, 84 insertions(+), 1 deletion(-) diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h index ff938ac9c43c..86131845b761 100644 --- a/include/trace/events/huge_memory.h +++ b/include/trace/events/huge_memory.h @@ -44,12 +44,21 @@ EM( SCAN_PAGE_NOT_EXCLUSIVE, "page_not_exclusive") \ EMe(SCAN_ALLOC_LIGHT_MISS, "alloc_light_miss") +#define COLLAPSE_PASS_STATUS \ + EM( COLLAPSE_PASS_ALLOC, "alloc") \ + EM( COLLAPSE_PASS_REVALIDATE, "revalidate") \ + EM( COLLAPSE_PASS_FAULTIN, "faultin") \ + EM( COLLAPSE_PASS_FREEZE, "freeze") \ + EM( COLLAPSE_PASS_COPY, "copy") \ + EMe(COLLAPSE_PASS_INSTALL, "install") + #undef EM #undef EMe #define EM(a, b) TRACE_DEFINE_ENUM(a); #define EMe(a, b) TRACE_DEFINE_ENUM(a); SCAN_STATUS +COLLAPSE_PASS_STATUS #undef EM #undef EMe @@ -117,6 +126,37 @@ TRACE_EVENT(mm_collapse_huge_page, __entry->order) ); +TRACE_EVENT(mm_collapse_candidate, + + TP_PROTO(struct mm_struct *mm, unsigned long addr, unsigned int order, + int pass, int result), + + TP_ARGS(mm, addr, order, pass, result), + + TP_STRUCT__entry( + __field(struct mm_struct *, mm) + __field(unsigned long, addr) + __field(unsigned int, order) + __field(int, pass) + __field(int, result) + ), + + TP_fast_assign( + __entry->mm = mm; + __entry->addr = addr; + __entry->order = order; + __entry->pass = pass; + __entry->result = result; + ), + + TP_printk("mm=%p, addr=0x%lx, order=%u, pass=%s, result=%s", + __entry->mm, + __entry->addr, + __entry->order, + __print_symbolic(__entry->pass, COLLAPSE_PASS_STATUS), + __print_symbolic(__entry->result, SCAN_STATUS)) +); + TRACE_EVENT(mm_collapse_huge_page_isolate, TP_PROTO(struct folio *folio, int none_or_zero, diff --git a/mm/collapse.c b/mm/collapse.c index 9b73ebff1103..91ff20138a8e 100644 --- a/mm/collapse.c +++ b/mm/collapse.c @@ -20,6 +20,7 @@ #include <linux/vmstat.h> #include <asm/tlb.h> +#include <trace/events/huge_memory.h> #include "collapse.h" #include "internal.h" @@ -195,6 +196,14 @@ static unsigned int candidate_nr_pages(const struct collapse_candidate *cand) return 1U << cand->order; } +static void collapse_trace_candidate(struct mm_struct *mm, + const struct collapse_candidate *cand, + enum collapse_pass pass) +{ + trace_mm_collapse_candidate(mm, cand->addr, cand->order, pass, + cand->result); +} + /* Where a candidate sits in the table, in the PTE offsets selection counts in */ static unsigned int candidate_offset(const struct collapse_candidate *cand, unsigned long pmd_addr) @@ -278,6 +287,8 @@ static enum scan_result collapse_revalidate(struct vm_area_struct *vma, BIT(cand->order))) { cand->state = CAND_SKIPPED; cand->result = SCAN_VMA_CHECK; + collapse_trace_candidate(mm, cand, + COLLAPSE_PASS_REVALIDATE); continue; } @@ -420,6 +431,8 @@ static enum scan_result collapse_faultin(struct vm_area_struct *vma, if (r == SCAN_EXCEED_SWAP_PTE) { cand->state = CAND_SKIPPED; cand->result = r; + collapse_trace_candidate(vma->vm_mm, cand, + COLLAPSE_PASS_FAULTIN); break; } if (r != SCAN_SUCCEED) { @@ -878,6 +891,7 @@ static void collapse_freeze(struct vm_area_struct *vma, continue; cand->state = CAND_SKIPPED; cand->result = SCAN_NO_PTE_TABLE; + collapse_trace_candidate(mm, cand, COLLAPSE_PASS_FREEZE); } return; } @@ -904,6 +918,7 @@ static void collapse_freeze(struct vm_area_struct *vma, cand->result = result; if (result != SCAN_SUCCEED) { cand->state = CAND_SKIPPED; + collapse_trace_candidate(mm, cand, COLLAPSE_PASS_FREEZE); continue; } @@ -986,6 +1001,7 @@ static void collapse_reserve(struct mm_struct *mm, struct collapse_control *cc) cand->state = CAND_SKIPPED; cand->result = result; + collapse_trace_candidate(mm, cand, COLLAPSE_PASS_ALLOC); } } @@ -1016,6 +1032,7 @@ static void collapse_deposit(struct mm_struct *mm, struct collapse_control *cc) if (!cand->deposit) { cand->state = CAND_SKIPPED; cand->result = SCAN_ALLOC_HUGE_PAGE_FAIL; + collapse_trace_candidate(mm, cand, COLLAPSE_PASS_ALLOC); } } @@ -1060,6 +1077,8 @@ static void collapse_provision(struct mm_struct *mm, } cand->result = result; } + + collapse_trace_candidate(mm, cand, COLLAPSE_PASS_ALLOC); } } @@ -1106,6 +1125,8 @@ static void collapse_copy(struct vm_area_struct *vma, */ if (copy_mc_user_highpage(dst, src, addr, vma)) { cand->result = SCAN_COPY_MC; + collapse_trace_candidate(vma->vm_mm, cand, + COLLAPSE_PASS_COPY); break; } } @@ -1294,6 +1315,7 @@ static void collapse_install_pmd(struct vm_area_struct *vma, /* Table gone under us; see collapse_abort_candidate() on @pte */ spin_unlock(pmd_ptl); cand->result = SCAN_NO_PTE_TABLE; + collapse_trace_candidate(mm, cand, COLLAPSE_PASS_INSTALL); collapse_abort_candidate(vma, cand, NULL); return; } @@ -1315,6 +1337,7 @@ static void collapse_install_pmd(struct vm_area_struct *vma, if (!collapse_verify_candidate(cand, pte, &nr_populated)) { cand->result = SCAN_PTE_NON_PRESENT; + collapse_trace_candidate(mm, cand, COLLAPSE_PASS_INSTALL); collapse_abort_candidate(vma, cand, pte); goto out_unlock; } @@ -1411,6 +1434,8 @@ static void collapse_install(struct vm_area_struct *vma, continue; cand->result = SCAN_NO_PTE_TABLE; + collapse_trace_candidate(mm, cand, + COLLAPSE_PASS_INSTALL); collapse_abort_candidate(vma, cand, NULL); } return; @@ -1439,6 +1464,8 @@ static void collapse_install(struct vm_area_struct *vma, if (!collapse_verify_candidate(cand, cand_pte, &nr_populated)) { cand->result = SCAN_PTE_NON_PRESENT; + collapse_trace_candidate(mm, cand, + COLLAPSE_PASS_INSTALL); collapse_abort_candidate(vma, cand, cand_pte); continue; } @@ -1548,8 +1575,11 @@ static unsigned int collapse_finish(struct mm_struct *mm, pte_free(mm, cand->deposit); cand->deposit = NULL; } - if (cand->state == CAND_INSTALLED) + if (cand->state == CAND_INSTALLED) { nr_installed++; + collapse_trace_candidate(mm, cand, + COLLAPSE_PASS_INSTALL); + } } return nr_installed; diff --git a/mm/collapse.h b/mm/collapse.h index 3803f5a89087..34de3ebb05e3 100644 --- a/mm/collapse.h +++ b/mm/collapse.h @@ -13,6 +13,19 @@ struct collapse_candidate; struct collapse_retry; +/* + * Which pass of a round reached a verdict on a candidate. Only collapse.c + * produces these; the trace header khugepaged.c builds names them. + */ +enum collapse_pass { + COLLAPSE_PASS_ALLOC, + COLLAPSE_PASS_REVALIDATE, + COLLAPSE_PASS_FAULTIN, + COLLAPSE_PASS_FREEZE, + COLLAPSE_PASS_COPY, + COLLAPSE_PASS_INSTALL, +}; + enum scan_result { SCAN_FAIL, SCAN_SUCCEED, -- 2.54.0
