The Ziccid extension provides hardware synchronization between Dcache and Icache. With this hardware support, there's no longer a need to trigger remote hart execution of fence.i via IPI.
Signed-off-by: Yunhui Cui <[email protected]> --- arch/riscv/include/asm/cacheflush.h | 4 ++-- arch/riscv/include/asm/hwcap.h | 1 + arch/riscv/include/asm/switch_to.h | 10 ++++++++++ arch/riscv/kernel/cpufeature.c | 1 + arch/riscv/kernel/ftrace.c | 2 +- arch/riscv/kernel/hibernate.c | 2 +- arch/riscv/kernel/jump_label.c | 2 +- arch/riscv/mm/cacheflush.c | 16 ++++++++++++++-- 8 files changed, 31 insertions(+), 7 deletions(-) diff --git a/arch/riscv/include/asm/cacheflush.h b/arch/riscv/include/asm/cacheflush.h index 0092513c3376c..3a8cdf30bb4b1 100644 --- a/arch/riscv/include/asm/cacheflush.h +++ b/arch/riscv/include/asm/cacheflush.h @@ -68,7 +68,7 @@ static inline void flush_cache_vmap(unsigned long start, unsigned long end) #else /* CONFIG_SMP */ -void flush_icache_all(void); +void flush_icache_all(bool force); void flush_icache_mm(struct mm_struct *mm, bool local); #endif /* CONFIG_SMP */ @@ -80,7 +80,7 @@ void flush_icache_mm(struct mm_struct *mm, bool local); #define flush_icache_range flush_icache_range static inline void flush_icache_range(unsigned long start, unsigned long end) { - flush_icache_all(); + flush_icache_all(false); } extern unsigned int riscv_cbom_block_size; diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index affd63e11b0a3..ad97d8955b501 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -106,6 +106,7 @@ #define RISCV_ISA_EXT_ZAAMO 97 #define RISCV_ISA_EXT_ZALRSC 98 #define RISCV_ISA_EXT_ZICBOP 99 +#define RISCV_ISA_EXT_ZICCID 100 #define RISCV_ISA_EXT_XLINUXENVCFG 127 diff --git a/arch/riscv/include/asm/switch_to.h b/arch/riscv/include/asm/switch_to.h index 0e71eb82f920c..b8a9e455efe9e 100644 --- a/arch/riscv/include/asm/switch_to.h +++ b/arch/riscv/include/asm/switch_to.h @@ -98,7 +98,17 @@ static inline bool switch_to_should_flush_icache(struct task_struct *task) bool stale_thread = task->thread.force_icache_flush; bool thread_migrated = smp_processor_id() != task->thread.prev_cpu; + asm goto(ALTERNATIVE("nop", "j %l[ziccid]", 0, RISCV_ISA_EXT_ZICCID, 1) + : : : : ziccid); + return thread_migrated && (stale_mm || stale_thread); + +ziccid: + /* + * Process switching writes to SATP, which flushes the pipeline, + * so only the thread scenario is considered. + */ + return thread_migrated && stale_thread; #else return false; #endif diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 67b59699357da..2da82aa2dbf0a 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -540,6 +540,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(svnapot, RISCV_ISA_EXT_SVNAPOT), __RISCV_ISA_EXT_DATA(svpbmt, RISCV_ISA_EXT_SVPBMT), __RISCV_ISA_EXT_DATA(svvptc, RISCV_ISA_EXT_SVVPTC), + __RISCV_ISA_EXT_DATA(ziccid, RISCV_ISA_EXT_ZICCID), }; const size_t riscv_isa_ext_count = ARRAY_SIZE(riscv_isa_ext); diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c index 8d18d6727f0fc..431448e818363 100644 --- a/arch/riscv/kernel/ftrace.c +++ b/arch/riscv/kernel/ftrace.c @@ -43,7 +43,7 @@ void arch_ftrace_update_code(int command) { command |= FTRACE_MAY_SLEEP; ftrace_modify_all_code(command); - flush_icache_all(); + flush_icache_all(false); } static int __ftrace_modify_call(unsigned long source, unsigned long target, bool validate) diff --git a/arch/riscv/kernel/hibernate.c b/arch/riscv/kernel/hibernate.c index 671b686c01587..388f10e187bae 100644 --- a/arch/riscv/kernel/hibernate.c +++ b/arch/riscv/kernel/hibernate.c @@ -153,7 +153,7 @@ int swsusp_arch_suspend(void) } else { suspend_restore_csrs(hibernate_cpu_context); flush_tlb_all(); - flush_icache_all(); + flush_icache_all(true); /* * Tell the hibernation core that we've just restored the memory. diff --git a/arch/riscv/kernel/jump_label.c b/arch/riscv/kernel/jump_label.c index b4c1a6a3fbd28..680b29f4c09c4 100644 --- a/arch/riscv/kernel/jump_label.c +++ b/arch/riscv/kernel/jump_label.c @@ -51,5 +51,5 @@ bool arch_jump_label_transform_queue(struct jump_entry *entry, void arch_jump_label_transform_apply(void) { - flush_icache_all(); + flush_icache_all(false); } diff --git a/arch/riscv/mm/cacheflush.c b/arch/riscv/mm/cacheflush.c index d83a612464f6c..01f9f7a45e8d2 100644 --- a/arch/riscv/mm/cacheflush.c +++ b/arch/riscv/mm/cacheflush.c @@ -12,19 +12,24 @@ #ifdef CONFIG_SMP #include <asm/sbi.h> +#include <asm/alternative-macros.h> static void ipi_remote_fence_i(void *info) { return local_flush_icache_all(); } -void flush_icache_all(void) +void flush_icache_all(bool force) { local_flush_icache_all(); if (num_online_cpus() < 2) return; + if (!force) + asm goto(ALTERNATIVE("nop", "j %l[ziccid]", 0, + RISCV_ISA_EXT_ZICCID, 1) + : : : : ziccid); /* * Make sure all previous writes to the D$ are ordered before making * the IPI. The RISC-V spec states that a hart must execute a data fence @@ -41,6 +46,7 @@ void flush_icache_all(void) sbi_remote_fence_i(NULL); else on_each_cpu(ipi_remote_fence_i, NULL, 1); +ziccid:; } EXPORT_SYMBOL(flush_icache_all); @@ -61,13 +67,17 @@ void flush_icache_mm(struct mm_struct *mm, bool local) preempt_disable(); + local_flush_icache_all(); + + asm goto(ALTERNATIVE("nop", "j %l[ziccid]", 0, RISCV_ISA_EXT_ZICCID, 1) + : : : : ziccid); + /* Mark every hart's icache as needing a flush for this MM. */ mask = &mm->context.icache_stale_mask; cpumask_setall(mask); /* Flush this hart's I$ now, and mark it as flushed. */ cpu = smp_processor_id(); cpumask_clear_cpu(cpu, mask); - local_flush_icache_all(); /* * Flush the I$ of other harts concurrently executing, and mark them as @@ -91,6 +101,8 @@ void flush_icache_mm(struct mm_struct *mm, bool local) on_each_cpu_mask(&others, ipi_remote_fence_i, NULL, 1); } +ziccid:; + preempt_enable(); } -- 2.39.5
