Rename page_fault_{user,kernel} to page_fault_{user,kernel}_enter, and
add the exit counterparts. This might be useful for measuring page fault
handling latencies.The exit tracepoints use a separate static key. Signed-off-by: Junxuan Liao <[email protected]> Link: https://lore.kernel.org/[email protected] --- arch/x86/include/asm/trace/common.h | 18 ++++++++++---- arch/x86/include/asm/trace/exceptions.h | 32 ++++++++++++++++++++----- arch/x86/kernel/tracepoint.c | 22 +++++++++++++---- arch/x86/mm/fault.c | 21 +++++++++++++--- 4 files changed, 75 insertions(+), 18 deletions(-) diff --git a/arch/x86/include/asm/trace/common.h b/arch/x86/include/asm/trace/common.h index f0f9bcdb74d9..4b289be75173 100644 --- a/arch/x86/include/asm/trace/common.h +++ b/arch/x86/include/asm/trace/common.h @@ -2,11 +2,21 @@ #define _ASM_TRACE_COMMON_H #ifdef CONFIG_TRACING -DECLARE_STATIC_KEY_FALSE(trace_pagefault_key); -#define trace_pagefault_enabled() \ - static_branch_unlikely(&trace_pagefault_key) +DECLARE_STATIC_KEY_FALSE(trace_pagefault_enter_key); +DECLARE_STATIC_KEY_FALSE(trace_pagefault_exit_key); +#define trace_pagefault_enter_enabled() \ + static_branch_unlikely(&trace_pagefault_enter_key) +#define trace_pagefault_exit_enabled() \ + static_branch_unlikely(&trace_pagefault_exit_key) #else -static inline bool trace_pagefault_enabled(void) { return false; } +static inline bool trace_pagefault_enter_enabled(void) +{ + return false; +} +static inline bool trace_pagefault_exit_enabled(void) +{ + return false; +} #endif #endif diff --git a/arch/x86/include/asm/trace/exceptions.h b/arch/x86/include/asm/trace/exceptions.h index 6b1e87194809..76729fcdb249 100644 --- a/arch/x86/include/asm/trace/exceptions.h +++ b/arch/x86/include/asm/trace/exceptions.h @@ -8,8 +8,10 @@ #include <linux/tracepoint.h> #include <asm/trace/common.h> -extern int trace_pagefault_reg(void); -extern void trace_pagefault_unreg(void); +extern int trace_pagefault_enter_reg(void); +extern void trace_pagefault_enter_unreg(void); +extern int trace_pagefault_exit_reg(void); +extern void trace_pagefault_exit_unreg(void); DECLARE_EVENT_CLASS(x86_exceptions, @@ -34,15 +36,33 @@ DECLARE_EVENT_CLASS(x86_exceptions, (void *)__entry->address, (void *)__entry->ip, __entry->error_code) ); -#define DEFINE_PAGE_FAULT_EVENT(name) \ +#define DEFINE_PAGE_FAULT_EVENT(name, reg, unreg) \ DEFINE_EVENT_FN(x86_exceptions, name, \ TP_PROTO(unsigned long address, struct pt_regs *regs, \ unsigned long error_code), \ TP_ARGS(address, regs, error_code), \ - trace_pagefault_reg, trace_pagefault_unreg); + reg, unreg) -DEFINE_PAGE_FAULT_EVENT(page_fault_user); -DEFINE_PAGE_FAULT_EVENT(page_fault_kernel); +DEFINE_PAGE_FAULT_EVENT( + page_fault_user_enter, + trace_pagefault_enter_reg, + trace_pagefault_enter_unreg +); +DEFINE_PAGE_FAULT_EVENT( + page_fault_user_exit, + trace_pagefault_exit_reg, + trace_pagefault_exit_unreg +); +DEFINE_PAGE_FAULT_EVENT( + page_fault_kernel_enter, + trace_pagefault_enter_reg, + trace_pagefault_enter_unreg +); +DEFINE_PAGE_FAULT_EVENT( + page_fault_kernel_exit, + trace_pagefault_exit_reg, + trace_pagefault_exit_unreg +); #undef TRACE_INCLUDE_PATH #undef TRACE_INCLUDE_FILE diff --git a/arch/x86/kernel/tracepoint.c b/arch/x86/kernel/tracepoint.c index 03ae1caaa878..108285046226 100644 --- a/arch/x86/kernel/tracepoint.c +++ b/arch/x86/kernel/tracepoint.c @@ -7,15 +7,27 @@ #include <asm/trace/exceptions.h> -DEFINE_STATIC_KEY_FALSE(trace_pagefault_key); +DEFINE_STATIC_KEY_FALSE(trace_pagefault_enter_key); +DEFINE_STATIC_KEY_FALSE(trace_pagefault_exit_key); -int trace_pagefault_reg(void) +int trace_pagefault_enter_reg(void) { - static_branch_inc(&trace_pagefault_key); + static_branch_inc(&trace_pagefault_enter_key); return 0; } -void trace_pagefault_unreg(void) +void trace_pagefault_enter_unreg(void) { - static_branch_dec(&trace_pagefault_key); + static_branch_dec(&trace_pagefault_enter_key); +} + +int trace_pagefault_exit_reg(void) +{ + static_branch_inc(&trace_pagefault_exit_key); + return 0; +} + +void trace_pagefault_exit_unreg(void) +{ + static_branch_dec(&trace_pagefault_exit_key); } diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 296d294142c8..cd933edf1e19 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -1455,13 +1455,26 @@ static __always_inline void trace_page_fault_entries(struct pt_regs *regs, unsigned long error_code, unsigned long address) { - if (!trace_pagefault_enabled()) + if (!trace_pagefault_enter_enabled()) return; if (user_mode(regs)) - trace_page_fault_user(address, regs, error_code); + trace_page_fault_user_enter(address, regs, error_code); else - trace_page_fault_kernel(address, regs, error_code); + trace_page_fault_kernel_enter(address, regs, error_code); +} + +static __always_inline void +trace_page_fault_exits(struct pt_regs *regs, unsigned long error_code, + unsigned long address) +{ + if (!trace_pagefault_exit_enabled()) + return; + + if (user_mode(regs)) + trace_page_fault_user_exit(address, regs, error_code); + else + trace_page_fault_kernel_exit(address, regs, error_code); } static __always_inline void @@ -1487,6 +1500,8 @@ handle_page_fault(struct pt_regs *regs, unsigned long error_code, */ local_irq_disable(); } + + trace_page_fault_exits(regs, error_code, address); } DEFINE_IDTENTRY_RAW_ERRORCODE(exc_page_fault) -- 2.48.1
