PMU overflow delivery through SSE enters Linux with a synthetic supervisor context on a dedicated event stack. The perf unwinder must use the context interrupted by the overflow rather than treating the SSE handler frame as the sampled frame.
Use the interrupted pt_regs published by the RISC-V SSE entry path. Walk a kernel callchain only when the interrupted PC, SP, and frame pointer are consistent with the current task stack. For sensitive entry windows and IRQ stacks whose bounds cannot be proven, retain the interrupted PC without following an unsafe frame chain. User callchains continue through the existing nofault RISC-V user unwinder. If hstatus.SPV says the interrupted context was a guest, do not interpret the guest stack through the host address space. DWARF callchains additionally copy a raw user stack. The generic arch_perf_out_copy_user() implementation can take an exception-table handled fault when a source page is not resident. Repeated nested faults from the SSE handler can corrupt the interrupted kernel context under load. Provide an SSE-specific RISC-V copy path. Verify that the active page table belongs to current, use fast-only GUP to acquire each resident source page without falling back to a faulting slow path, and copy through the kernel mapping while holding the page reference. Stop at the first unavailable page and preserve perf's existing truncated-user-stack semantics. Keep the generic in-atomic user copy unchanged outside an SSE handler. Susheng Yang reported this failure with perf callchain workloads. Reported-by: Susheng Yang <[email protected]> Signed-off-by: Zhanpeng Zhang <[email protected]> --- arch/riscv/include/asm/perf_event.h | 10 ++ arch/riscv/kernel/perf_callchain.c | 142 ++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) diff --git a/arch/riscv/include/asm/perf_event.h b/arch/riscv/include/asm/perf_event.h index bcc928fd3785..ddc404794751 100644 --- a/arch/riscv/include/asm/perf_event.h +++ b/arch/riscv/include/asm/perf_event.h @@ -18,6 +18,16 @@ (regs)->sp = current_stack_pointer; \ (regs)->status = SR_PP; \ } + +#ifdef CONFIG_RISCV_SBI_SSE +/* + * Raw user-stack sampling can run in the NMI-like SSE context. Route it + * through an implementation that does not fault on a non-resident page. + */ +unsigned long riscv_perf_out_copy_user(void *dst, const void *src, + unsigned long n); +#define arch_perf_out_copy_user riscv_perf_out_copy_user +#endif #endif #endif /* _ASM_RISCV_PERF_EVENT_H */ diff --git a/arch/riscv/kernel/perf_callchain.c b/arch/riscv/kernel/perf_callchain.c index b465bc9eb870..ec75689c7aec 100644 --- a/arch/riscv/kernel/perf_callchain.c +++ b/arch/riscv/kernel/perf_callchain.c @@ -1,9 +1,15 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd. */ +#include <linux/kallsyms.h> +#include <linux/mm.h> #include <linux/perf_event.h> +#include <linux/sched/task_stack.h> +#include <linux/thread_info.h> #include <linux/uaccess.h> +#include <asm/csr.h> +#include <asm/sse.h> #include <asm/stacktrace.h> static bool fill_callchain(void *entry, unsigned long pc) @@ -11,6 +17,128 @@ static bool fill_callchain(void *entry, unsigned long pc) return perf_callchain_store(entry, pc) == 0; } +#ifdef CONFIG_RISCV_SBI_SSE +static bool sse_addr_on_task_stack(unsigned long addr, unsigned long size) +{ + unsigned long end = addr + size; + unsigned long stack; + + if (end < addr) + return false; + + stack = (unsigned long)task_stack_page(current); + if (addr >= stack && end <= stack + THREAD_SIZE) + return true; + + return false; +} + +static bool sse_kernel_regs_safe(struct pt_regs *regs) +{ + unsigned long fp = frame_pointer(regs); + unsigned long pc = instruction_pointer(regs); + unsigned long sp = user_stack_pointer(regs); + + if (!__kernel_text_address(pc)) + return false; + if (!sse_addr_on_task_stack(sp, sizeof(unsigned long))) + return false; + if (fp < sizeof(struct stackframe)) + return false; + + return sse_addr_on_task_stack(fp - sizeof(struct stackframe), + sizeof(struct stackframe)); +} + +static bool sse_callchain_is_guest(const struct riscv_sse_interrupted_context *context) +{ + return context && (context->hstatus & HSTATUS_SPV); +} + +static bool sse_callchain_kernel(struct perf_callchain_entry_ctx *entry, + struct pt_regs *regs) +{ + const struct riscv_sse_interrupted_context *context; + unsigned long pc; + + context = riscv_sse_get_interrupted_context(); + if (!context || context->regs != regs) + return false; + + /* A guest stack cannot be walked using the host kernel address space. */ + if (sse_callchain_is_guest(context)) + return true; + + if (user_mode(regs)) + return true; + + if (sse_kernel_regs_safe(regs)) { + walk_stackframe(NULL, regs, fill_callchain, entry); + return true; + } + + /* + * Keep the sample useful for sensitive entry paths and IRQ stacks. The + * generic walker does not take explicit IRQ stack bounds, and its + * THREAD_SIZE alignment assumption fails for non-vmapped IRQ stacks. + * Conservatively avoid walking IRQ stacks in every configuration. + */ + pc = instruction_pointer(regs); + if (__kernel_text_address(pc)) + perf_callchain_store(entry, pc); + + return true; +} + +unsigned long riscv_perf_out_copy_user(void *dst, const void *src, + unsigned long n) +{ + unsigned long addr = (unsigned long)src; + unsigned long copied = 0; + + /* Keep the generic fast path unchanged outside an SSE handler. */ + if (!riscv_sse_get_interrupted_context()) { + unsigned long ret; + + pagefault_disable(); + ret = __copy_from_user_inatomic(dst, src, n); + pagefault_enable(); + return ret; + } + + if (!access_ok(src, n)) + return n; + + /* Do not sample user memory through an unrelated active page table. */ + if (!current->mm || + (csr_read(CSR_SATP) & SATP_PPN) != virt_to_pfn(current->mm->pgd)) + return n; + + while (copied < n) { + unsigned long offset = offset_in_page(addr); + unsigned long chunk = min(n - copied, PAGE_SIZE - offset); + struct page *page; + + /* + * Fast-only GUP cannot fault. This follows perf_virt_to_phys(): + * local interrupts remain disabled throughout SSE processing, so a + * concurrent unmap cannot complete its TLB teardown before this + * temporary reference is put. + */ + if (!get_user_page_fast_only(addr, 0, &page)) + break; + + memcpy((char *)dst + copied, + (char *)page_address(page) + offset, chunk); + put_page(page); + addr += chunk; + copied += chunk; + } + + return n - copied; +} +#endif + /* * This will be called when the target is in user mode * This function will only be called when we use @@ -28,6 +156,15 @@ static bool fill_callchain(void *entry, unsigned long pc) void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { +#ifdef CONFIG_RISCV_SBI_SSE + const struct riscv_sse_interrupted_context *context; + + context = riscv_sse_get_interrupted_context(); + /* A guest stack cannot be walked using the host address space. */ + if (sse_callchain_is_guest(context)) + return; +#endif + if (perf_guest_state()) { /* TODO: We don't support guest os callchain now */ return; @@ -39,6 +176,11 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry, void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { +#ifdef CONFIG_RISCV_SBI_SSE + if (sse_callchain_kernel(entry, regs)) + return; +#endif + if (perf_guest_state()) { /* TODO: We don't support guest os callchain now */ return; -- 2.50.1 (Apple Git-155)
