jump to FTRACE_ADDR if distance is out of reach
Co-developed-by: Björn Töpel <[email protected]>
Signed-off-by: Björn Töpel <[email protected]>
Signed-off-by: Andy Chiu <[email protected]>
---
Changelog v4:
- New patch since v4
- Include Björn's fix for kprobe (adjusting ftrace address with
MCOUNT_INSN_SIZE)
- Clean out an unused variable
---
arch/riscv/Kconfig | 2 +-
arch/riscv/include/asm/ftrace.h | 6 ++++
arch/riscv/kernel/asm-offsets.c | 3 ++
arch/riscv/kernel/ftrace.c | 13 ++++-----
arch/riscv/kernel/mcount-dyn.S | 51 +++++++++++++++++++++------------
5 files changed, 48 insertions(+), 27 deletions(-)
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index ec986c9120e3..8fdca6345fa3 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -152,7 +152,7 @@ config RISCV
select HAVE_DMA_CONTIGUOUS if MMU
select HAVE_DYNAMIC_FTRACE if !XIP_KERNEL && MMU &&
(CLANG_SUPPORTS_DYNAMIC_FTRACE || GCC_SUPPORTS_DYNAMIC_FTRACE)
select FUNCTION_ALIGNMENT_4B if HAVE_DYNAMIC_FTRACE && RISCV_ISA_C
- select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
+ select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS if
HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS
select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS if (DYNAMIC_FTRACE_WITH_ARGS
&& !CFI_CLANG)
select HAVE_DYNAMIC_FTRACE_WITH_ARGS if HAVE_DYNAMIC_FTRACE
select HAVE_FTRACE_GRAPH_FUNC
diff --git a/arch/riscv/include/asm/ftrace.h b/arch/riscv/include/asm/ftrace.h
index 6a5c0a7fb826..22ebea3c2b26 100644
--- a/arch/riscv/include/asm/ftrace.h
+++ b/arch/riscv/include/asm/ftrace.h
@@ -130,6 +130,9 @@ struct __arch_ftrace_regs {
unsigned long sp;
unsigned long s0;
unsigned long t1;
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
+ unsigned long direct_tramp;
+#endif
union {
unsigned long args[8];
struct {
@@ -223,10 +226,13 @@ void ftrace_graph_func(unsigned long ip, unsigned long
parent_ip,
struct ftrace_ops *op, struct ftrace_regs *fregs);
#define ftrace_graph_func ftrace_graph_func
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
static inline void arch_ftrace_set_direct_caller(struct ftrace_regs *fregs,
unsigned long addr)
{
arch_ftrace_regs(fregs)->t1 = addr;
}
+#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
+
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */
#endif /* __ASSEMBLY__ */
diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c
index 2d96197a8abf..b26334075697 100644
--- a/arch/riscv/kernel/asm-offsets.c
+++ b/arch/riscv/kernel/asm-offsets.c
@@ -495,6 +495,9 @@ void asm_offsets(void)
OFFSET(STACKFRAME_RA, stackframe, ra);
#ifdef CONFIG_FUNCTION_TRACER
DEFINE(FTRACE_OPS_FUNC, offsetof(struct ftrace_ops, func));
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
+ DEFINE(FTRACE_OPS_DIRECT_CALL, offsetof(struct ftrace_ops,
direct_call));
+#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
#endif
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c
index 30bcf60135d8..d65f06bfb457 100644
--- a/arch/riscv/kernel/ftrace.c
+++ b/arch/riscv/kernel/ftrace.c
@@ -16,7 +16,7 @@
unsigned long ftrace_call_adjust(unsigned long addr)
{
if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS))
- return addr + 8;
+ return addr + 8 + MCOUNT_AUIPC_SIZE;
return addr + MCOUNT_AUIPC_SIZE;
}
@@ -83,10 +83,9 @@ static const struct ftrace_ops *riscv64_rec_get_ops(struct
dyn_ftrace *rec)
return ops;
}
-static int ftrace_rec_set_ops(const struct dyn_ftrace *rec,
- const struct ftrace_ops *ops)
+static int ftrace_rec_set_ops(const struct dyn_ftrace *rec, const struct
ftrace_ops *ops)
{
- unsigned long literal = rec->ip - 8;
+ unsigned long literal = ALIGN_DOWN(rec->ip - 12, 8);
return patch_text_nosync((void *)literal, &ops, sizeof(ops));
}
@@ -117,7 +116,7 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long
addr)
orig_addr = (unsigned long)&ftrace_caller;
distance = addr > orig_addr ? addr - orig_addr : orig_addr - addr;
if (distance > JALR_RANGE)
- return -EINVAL;
+ addr = FTRACE_ADDR;
return __ftrace_modify_call(pc, addr, false);
}
@@ -199,15 +198,13 @@ int ftrace_modify_call(struct dyn_ftrace *rec, unsigned
long old_addr,
unsigned long addr)
{
unsigned long caller = rec->ip - MCOUNT_AUIPC_SIZE;
- unsigned int call[2];
int ret;
- make_call_t0(caller, old_addr, call);
ret = ftrace_rec_update_ops(rec);
if (ret)
return ret;
- return __ftrace_modify_call(caller, addr, true);
+ return __ftrace_modify_call(caller, FTRACE_ADDR, true);
}
#endif
diff --git a/arch/riscv/kernel/mcount-dyn.S b/arch/riscv/kernel/mcount-dyn.S
index 699684eea7f0..48f6c4f7dca0 100644
--- a/arch/riscv/kernel/mcount-dyn.S
+++ b/arch/riscv/kernel/mcount-dyn.S
@@ -82,12 +82,9 @@
* +++++++++
**/
.macro SAVE_ABI_REGS
- mv t4, sp // Save original SP in T4
addi sp, sp, -FREGS_SIZE_ON_STACK
-
REG_S t0, FREGS_EPC(sp)
REG_S x1, FREGS_RA(sp)
- REG_S t4, FREGS_SP(sp) // Put original SP on stack
#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
REG_S x8, FREGS_S0(sp)
#endif
@@ -108,9 +105,12 @@
REG_S x15, FREGS_A5(sp)
REG_S x16, FREGS_A6(sp)
REG_S x17, FREGS_A7(sp)
+ mv a0, sp
+ addi a0, a0, FREGS_SIZE_ON_STACK
+ REG_S a0, FREGS_SP(sp) // Put original SP on stack
.endm
- .macro RESTORE_ABI_REGS, all=0
+ .macro RESTORE_ABI_REGS
REG_L t0, FREGS_EPC(sp)
REG_L x1, FREGS_RA(sp)
#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
@@ -139,6 +139,19 @@
.macro PREPARE_ARGS
addi a0, t0, -MCOUNT_JALR_SIZE // ip (callsite's jalr insn)
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
+ mv a1, ra // parent_ip
+ REG_L a2, -16(t0) // op
+ REG_L ra, FTRACE_OPS_FUNC(a2) // op->func
+#else
+ la a1, function_trace_op
+ REG_L a2, 0(a1) // op
+ mv a1, ra // parent_ip
+#endif
+ mv a3, sp // regs
+ .endm
+
+SYM_FUNC_START(ftrace_caller)
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
/*
* When CALL_OPS is enabled (2 or 4) nops [8B] are placed before the
@@ -158,19 +171,17 @@
* t0 is set to ip+8 after the jalr is executed at the callsite,
* so we find the associated op at t0-16.
*/
- mv a1, ra // parent_ip
- REG_L a2, -16(t0) // op
- REG_L ra, FTRACE_OPS_FUNC(a2) // op->func
-#else
- la a1, function_trace_op
- REG_L a2, 0(a1) // op
- mv a1, ra // parent_ip
-#endif
- mv a3, sp // regs
- .endm
+ REG_L t1, -16(t0) // op Should be SZ_REG instead of 16
-SYM_FUNC_START(ftrace_caller)
- mv t1, zero
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
+ /*
+ * If the op has a direct call, handle it immediately without
+ * saving/restoring registers.
+ */
+ REG_L t1, FTRACE_OPS_DIRECT_CALL(t1)
+ bnez t1, ftrace_caller_direct
+#endif
+#endif
SAVE_ABI_REGS
PREPARE_ARGS
@@ -182,10 +193,14 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
jalr ra, 0(ra)
#endif
RESTORE_ABI_REGS
- bnez t1, .Ldirect
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
+ bnez t1, ftrace_caller_direct
+#endif
jr t0
-.Ldirect:
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
+SYM_INNER_LABEL(ftrace_caller_direct, SYM_L_LOCAL)
jr t1
+#endif
SYM_FUNC_END(ftrace_caller)
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
--
2.39.3 (Apple Git-145)
