Replace fence instructions with Zalasr instructions during smp_load_acquire() and smp_store_release() operations.
|----------------------------------|
| | __smp_store_release |
| |-----------------------------|
| | zalasr | !zalasr |
| rl |-----------------------------|
| | s{b|h|w|d}.rl | fence rw, w |
| | | s{b|h|w|d} |
|----------------------------------|
| | __smp_load_acquire |
| |-----------------------------|
| | zalasr | !zalasr |
| aq |-----------------------------|
| | l{b|h|w|d}.aq | l{b|h|w|d} |
| | | fence r, rw |
|----------------------------------|
Signed-off-by: Xu Lu <[email protected]>
---
arch/riscv/include/asm/barrier.h | 91 ++++++++++++++++++++++++++++----
1 file changed, 80 insertions(+), 11 deletions(-)
diff --git a/arch/riscv/include/asm/barrier.h b/arch/riscv/include/asm/barrier.h
index b8c5726d86acb..9eaf94a028096 100644
--- a/arch/riscv/include/asm/barrier.h
+++ b/arch/riscv/include/asm/barrier.h
@@ -51,19 +51,88 @@
*/
#define smp_mb__after_spinlock() RISCV_FENCE(iorw, iorw)
-#define __smp_store_release(p, v) \
-do { \
- compiletime_assert_atomic_type(*p); \
- RISCV_FENCE(rw, w); \
- WRITE_ONCE(*p, v); \
+extern void __bad_size_call_parameter(void);
+
+#define __smp_store_release(p, v)
\
+do {
\
+ typeof(p) __p = (p);
\
+ union { typeof(*p) __val; char __c[1]; } __u =
\
+ { .__val = (__force typeof(*p)) (v) };
\
+ compiletime_assert_atomic_type(*p);
\
+ switch (sizeof(*p)) {
\
+ case 1:
\
+ asm volatile(ALTERNATIVE("fence rw, w;\t\nsb %0, 0(%1)\t\n",
\
+ SB_RL(%0, %1) "\t\nnop\t\n",
\
+ 0, RISCV_ISA_EXT_ZALASR, 1)
\
+ : : "r" (*(__u8 *)__u.__c), "r" (__p)
\
+ : "memory");
\
+ break;
\
+ case 2:
\
+ asm volatile(ALTERNATIVE("fence rw, w;\t\nsh %0, 0(%1)\t\n",
\
+ SH_RL(%0, %1) "\t\nnop\t\n",
\
+ 0, RISCV_ISA_EXT_ZALASR, 1)
\
+ : : "r" (*(__u16 *)__u.__c), "r" (__p)
\
+ : "memory");
\
+ break;
\
+ case 4:
\
+ asm volatile(ALTERNATIVE("fence rw, w;\t\nsw %0, 0(%1)\t\n",
\
+ SW_RL(%0, %1) "\t\nnop\t\n",
\
+ 0, RISCV_ISA_EXT_ZALASR, 1)
\
+ : : "r" (*(__u32 *)__u.__c), "r" (__p)
\
+ : "memory");
\
+ break;
\
+ case 8:
\
+ asm volatile(ALTERNATIVE("fence rw, w;\t\nsd %0, 0(%1)\t\n",
\
+ SD_RL(%0, %1) "\t\nnop\t\n",
\
+ 0, RISCV_ISA_EXT_ZALASR, 1)
\
+ : : "r" (*(__u64 *)__u.__c), "r" (__p)
\
+ : "memory");
\
+ break;
\
+ default:
\
+ __bad_size_call_parameter();
\
+ break;
\
+ }
\
} while (0)
-#define __smp_load_acquire(p) \
-({ \
- typeof(*p) ___p1 = READ_ONCE(*p); \
- compiletime_assert_atomic_type(*p); \
- RISCV_FENCE(r, rw); \
- ___p1; \
+#define __smp_load_acquire(p)
\
+({
\
+ union { typeof(*p) __val; char __c[1]; } __u;
\
+ typeof(p) __p = (p);
\
+ compiletime_assert_atomic_type(*p);
\
+ switch (sizeof(*p)) {
\
+ case 1:
\
+ asm volatile(ALTERNATIVE("lb %0, 0(%1)\t\nfence r, rw\t\n",
\
+ LB_AQ(%0, %1) "\t\nnop\t\n",
\
+ 0, RISCV_ISA_EXT_ZALASR, 1)
\
+ : "=r" (*(__u8 *)__u.__c) : "r" (__p)
\
+ : "memory");
\
+ break;
\
+ case 2:
\
+ asm volatile(ALTERNATIVE("lh %0, 0(%1)\t\nfence r, rw\t\n",
\
+ LH_AQ(%0, %1) "\t\nnop\t\n",
\
+ 0, RISCV_ISA_EXT_ZALASR, 1)
\
+ : "=r" (*(__u16 *)__u.__c) : "r" (__p)
\
+ : "memory");
\
+ break;
\
+ case 4:
\
+ asm volatile(ALTERNATIVE("lw %0, 0(%1)\t\nfence r, rw\t\n",
\
+ LW_AQ(%0, %1) "\t\nnop\t\n",
\
+ 0, RISCV_ISA_EXT_ZALASR, 1)
\
+ : "=r" (*(__u32 *)__u.__c) : "r" (__p)
\
+ : "memory");
\
+ break;
\
+ case 8:
\
+ asm volatile(ALTERNATIVE("ld %0, 0(%1)\t\nfence r, rw\t\n",
\
+ LD_AQ(%0, %1) "\t\nnop\t\n",
\
+ 0, RISCV_ISA_EXT_ZALASR, 1)
\
+ : "=r" (*(__u64 *)__u.__c) : "r" (__p)
\
+ : "memory");
\
+ break;
\
+ default:
\
+ __bad_size_call_parameter();
\
+ break;
\
+ }
\
+ __u.__val;
\
})
#ifdef CONFIG_RISCV_ISA_ZAWRS
--
2.20.1
