https://github.com/Lukacma updated https://github.com/llvm/llvm-project/pull/227326
>From ea8313ea0030b8838d8b212b9771d025ffd11b3c Mon Sep 17 00:00:00 2001 From: Marian Lukac <[email protected]> Date: Wed, 16 Sep 2026 12:37:14 +0000 Subject: [PATCH 1/4] [AArch64] Add CMH hints to store_with_hint intrinsic --- clang/lib/Headers/arm_acle.h | 3 + clang/test/CodeGen/builtins-arm64.c | 19 ++++- clang/test/Sema/builtins-arm64.c | 3 +- .../include/llvm/Support/AArch64MemoryHints.h | 11 ++- .../Target/AArch64/AArch64ISelDAGToDAG.cpp | 18 +++++ .../lib/Target/AArch64/AArch64InstrAtomics.td | 9 +++ .../Atomics/aarch64-atomic-store-hint.ll | 80 ++++++++++++++++++- .../Atomics/aarch64-relaxed-store-hint.ll | 18 +++-- 8 files changed, 150 insertions(+), 11 deletions(-) diff --git a/clang/lib/Headers/arm_acle.h b/clang/lib/Headers/arm_acle.h index 715fe851b0c32..395b760e7dbf0 100644 --- a/clang/lib/Headers/arm_acle.h +++ b/clang/lib/Headers/arm_acle.h @@ -746,6 +746,9 @@ __arm_st64bv0(void *__addr, data512_t __value) { #if defined(__ARM_64BIT_STATE) && __ARM_64BIT_STATE #define HINT_STSHH_KEEP 0 #define HINT_STSHH_STRM 1 +#define HINT_STCPH 2 +#define HINT_SHUH 3 +#define HINT_SHUH_PH 4 #define __arm_atomic_store_with_hint(ptr, data, memory_order, hint) \ __builtin_arm_atomic_store_with_hint(ptr, data, memory_order, hint) #endif diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c index fb297cd3eec2c..9c3d810d55974 100644 --- a/clang/test/CodeGen/builtins-arm64.c +++ b/clang/test/CodeGen/builtins-arm64.c @@ -230,9 +230,18 @@ void atomic_store_with_hint(int64_t *a, int64_t b) { __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELEASE, HINT_STSHH_STRM); // CHECK: store atomic i64 {{.*}}, ptr {{.*}} release, align 8, !mem.cache_hint ![[M3:[0-9]+]] + __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, HINT_STCPH); + // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8, !mem.cache_hint ![[M5:[0-9]+]] + + __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELEASE, HINT_SHUH); + // CHECK: store atomic i64 {{.*}}, ptr {{.*}} release, align 8, !mem.cache_hint ![[M7:[0-9]+]] + + __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_SEQ_CST, HINT_SHUH_PH); + // CHECK: store atomic i64 {{.*}}, ptr {{.*}} seq_cst, align 8, !mem.cache_hint ![[M9:[0-9]+]] + // Invalid hint should be dropped - __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 2); // Invalid Hint - // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8 + __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 5); // Invalid Hint + // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8{{$}} } // CHECK: ![[M0]] = !{!"1:2:3:4:5"} @@ -240,3 +249,9 @@ void atomic_store_with_hint(int64_t *a, int64_t b) { // CHECK: ![[M2]] = !{!"aarch64.mem_hint", i32 0} // CHECK: ![[M3]] = !{i32 1, ![[M4:[0-9]+]]} // CHECK: ![[M4]] = !{!"aarch64.mem_hint", i32 1} +// CHECK: ![[M5]] = !{i32 1, ![[M6:[0-9]+]]} +// CHECK: ![[M6]] = !{!"aarch64.mem_hint", i32 2} +// CHECK: ![[M7]] = !{i32 1, ![[M8:[0-9]+]]} +// CHECK: ![[M8]] = !{!"aarch64.mem_hint", i32 3} +// CHECK: ![[M9]] = !{i32 1, ![[M10:[0-9]+]]} +// CHECK: ![[M10]] = !{!"aarch64.mem_hint", i32 4} diff --git a/clang/test/Sema/builtins-arm64.c b/clang/test/Sema/builtins-arm64.c index 9a6e0715c4c78..a79fb9177cec8 100644 --- a/clang/test/Sema/builtins-arm64.c +++ b/clang/test/Sema/builtins-arm64.c @@ -73,7 +73,8 @@ void test_atomic_store_hint(char *c_ptr, __int128 *inv_ptr, float *f_ptr, __builtin_arm_atomic_store_with_hint(c_ptr, c_data, inv_int, 0); // expected-error {{invalid memory order argument to atomic hint operation ('int' invalid)}} __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 2, 0); // expected-error {{invalid memory order argument to atomic hint operation (2 invalid)}} - __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 3); // expected-warning {{unrecognised hint type argument to atomic hint operation (3)}} + __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 5); // expected-warning {{unrecognised hint type argument to atomic hint operation (5)}} + __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, inv_int); // expected-error {{invalid hint type argument to atomic hint operation ('int')}} __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, "h"); // expected-error {{incompatible pointer to integer conversion passing 'char *' to parameter of type 'int'}} diff --git a/llvm/include/llvm/Support/AArch64MemoryHints.h b/llvm/include/llvm/Support/AArch64MemoryHints.h index b917191499cac..f47eb0c945d1b 100644 --- a/llvm/include/llvm/Support/AArch64MemoryHints.h +++ b/llvm/include/llvm/Support/AArch64MemoryHints.h @@ -14,11 +14,14 @@ enum class AArch64MemoryHint { NONE = 0, STSHH_KEEP = 1, STSHH_STRM = 2, + STCPH = 3, + SHUH = 4, + SHUH_PH = 5, }; template <typename Int> inline bool isValidAArch64MemoryHintValue(Int I) { return (Int)AArch64MemoryHint::STSHH_KEEP <= I && - I <= (Int)AArch64MemoryHint::STSHH_STRM; + I <= (Int)AArch64MemoryHint::SHUH_PH; } template <typename Int> inline AArch64MemoryHint toAArch64MemoryHint(Int I) { @@ -27,6 +30,12 @@ template <typename Int> inline AArch64MemoryHint toAArch64MemoryHint(Int I) { return AArch64MemoryHint::STSHH_KEEP; case 1: return AArch64MemoryHint::STSHH_STRM; + case 2: + return AArch64MemoryHint::STCPH; + case 3: + return AArch64MemoryHint::SHUH; + case 4: + return AArch64MemoryHint::SHUH_PH; default: return AArch64MemoryHint::NONE; } diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp index 48c0e37151b9c..aa07673673ce2 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -520,6 +520,9 @@ class AArch64DAGToDAGISel : public SelectionDAGISel { AArch64MemoryHint decodeMemoryHintFlags(MachineMemOperand *MMO) const; bool isAtomicSTSHH_KEEP(SDNode *N) const; bool isAtomicSTSHH_STRM(SDNode *N) const; + bool isAtomicSTCPH(SDNode *N) const; + bool isAtomicSHUH(SDNode *N) const; + bool isAtomicSHUH_PH(SDNode *N) const; bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift, bool Negate); @@ -4647,6 +4650,21 @@ bool AArch64DAGToDAGISel::isAtomicSTSHH_STRM(SDNode *N) const { AArch64MemoryHint::STSHH_STRM; } +bool AArch64DAGToDAGISel::isAtomicSTCPH(SDNode *N) const { + return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == + AArch64MemoryHint::STCPH; +} + +bool AArch64DAGToDAGISel::isAtomicSHUH(SDNode *N) const { + return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == + AArch64MemoryHint::SHUH; +} + +bool AArch64DAGToDAGISel::isAtomicSHUH_PH(SDNode *N) const { + return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == + AArch64MemoryHint::SHUH_PH; +} + bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift, bool Negate) { if (!isa<ConstantSDNode>(N)) diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td index 7e0e75c7a3c6b..56e4821fc20b8 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td +++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td @@ -425,11 +425,20 @@ multiclass AtomicHintPatterns<int Relaxed, int Hint, code Pred> { let AddedComplexity = 15 in { defm : AtomicHintPatternsAddrMode<48, [{ return isAtomicSTSHH_KEEP(N); }]>; defm : AtomicHintPatternsAddrMode<49, [{ return isAtomicSTSHH_STRM(N); }]>; + defm : AtomicHintPatternsAddrMode<52, [{ return isAtomicSTCPH(N); }]>; + defm : AtomicHintPatternsAddrMode<50, [{ return isAtomicSHUH(N); }]>; + defm : AtomicHintPatternsAddrMode<51, [{ return isAtomicSHUH_PH(N); }]>; defm : AtomicHintPatterns<1, 48, [{ return isAtomicSTSHH_KEEP(N); }]>; defm : AtomicHintPatterns<0, 48, [{ return isAtomicSTSHH_KEEP(N); }]>; defm : AtomicHintPatterns<1, 49, [{ return isAtomicSTSHH_STRM(N); }]>; defm : AtomicHintPatterns<0, 49, [{ return isAtomicSTSHH_STRM(N); }]>; + defm : AtomicHintPatterns<1, 52, [{ return isAtomicSTCPH(N); }]>; + defm : AtomicHintPatterns<0, 52, [{ return isAtomicSTCPH(N); }]>; + defm : AtomicHintPatterns<1, 50, [{ return isAtomicSHUH(N); }]>; + defm : AtomicHintPatterns<0, 50, [{ return isAtomicSHUH(N); }]>; + defm : AtomicHintPatterns<1, 51, [{ return isAtomicSHUH_PH(N); }]>; + defm : AtomicHintPatterns<0, 51, [{ return isAtomicSHUH_PH(N); }]>; } //===---------------------------------- diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll index bfa9fc49cf752..b015d020dc07f 100644 --- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll +++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll @@ -318,6 +318,78 @@ define void @test_atomic_store_stream_seqcst_double(ptr %ptr, double %val) nounw ret void } +; +; STCPH +; + +define void @test_atomic_store_stcph_relaxed_i8(ptr %ptr, i8 %val) nounwind { +; CHECK-LABEL: test_atomic_store_stcph_relaxed_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: stcph +; CHECK-NEXT: strb w1, [x0] +; CHECK-NEXT: ret + store atomic i8 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !4 + ret void +} + +define void @test_atomic_store_stcph_release_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: test_atomic_store_stcph_release_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: stcph +; CHECK-NEXT: stlr x1, [x0] +; CHECK-NEXT: ret + store atomic i64 %val, ptr %ptr release, align 8, !mem.cache_hint !4 + ret void +} + +; +; SHUH +; + +define void @test_atomic_store_shuh_relaxed_i8(ptr %ptr, i8 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_relaxed_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh{{$}} +; CHECK-NEXT: strb w1, [x0] +; CHECK-NEXT: ret + store atomic i8 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !6 + ret void +} + +define void @test_atomic_store_shuh_release_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_release_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh{{$}} +; CHECK-NEXT: stlr x1, [x0] +; CHECK-NEXT: ret + store atomic i64 %val, ptr %ptr release, align 8, !mem.cache_hint !6 + ret void +} + +; +; SHUH PH +; + +define void @test_atomic_store_shuh_ph_relaxed_i8(ptr %ptr, i8 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_ph_relaxed_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh ph +; CHECK-NEXT: strb w1, [x0] +; CHECK-NEXT: ret + store atomic i8 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !8 + ret void +} + +define void @test_atomic_store_shuh_ph_release_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_ph_release_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh ph +; CHECK-NEXT: stlr x1, [x0] +; CHECK-NEXT: ret + store atomic i64 %val, ptr %ptr release, align 8, !mem.cache_hint !8 + ret void +} + ; ; Invalid Hint ; @@ -327,7 +399,7 @@ define void @test_atomic_store_invalid_hint(ptr %ptr, i8 %val) nounwind { ; CHECK: // %bb.0: ; CHECK-NEXT: stlrb w1, [x0] ; CHECK-NEXT: ret - store atomic i8 %val, ptr %ptr release, align 8, !mem.cache_hint !4 + store atomic i8 %val, ptr %ptr release, align 8, !mem.cache_hint !10 ret void } @@ -337,3 +409,9 @@ define void @test_atomic_store_invalid_hint(ptr %ptr, i8 %val) nounwind { !3 = !{!"aarch64.mem_hint", i32 1} !4 = !{i32 1, !5} !5 = !{!"aarch64.mem_hint", i32 2} +!6 = !{i32 1, !7} +!7 = !{!"aarch64.mem_hint", i32 3} +!8 = !{i32 1, !9} +!9 = !{!"aarch64.mem_hint", i32 4} +!10 = !{i32 1, !11} +!11 = !{!"aarch64.mem_hint", i32 5} diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll index 5f5bc43039778..b862ae7f4a15d 100644 --- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll +++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll @@ -11,11 +11,11 @@ define void @relaxed_store_hint_roW_i8(ptr %ptr, i32 %offset, i8 %val) nounwind { ; CHECK-LABEL: relaxed_store_hint_roW_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: stshh keep +; CHECK-NEXT: stcph ; CHECK-NEXT: strb w2, [x0, w1, sxtw] ; CHECK-NEXT: ret %addr = getelementptr i8, ptr %ptr, i32 %offset - store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0 + store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !4 ret void } @@ -124,11 +124,11 @@ define void @relaxed_store_hint_roW_double(ptr %ptr, i32 %offset, double %val) n define void @relaxed_store_hint_roX_i8(ptr %ptr, i64 %offset, i8 %val) nounwind { ; CHECK-LABEL: relaxed_store_hint_roX_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: stshh strm +; CHECK-NEXT: shuh{{$}} ; CHECK-NEXT: strb w2, [x0, x1] ; CHECK-NEXT: ret %addr = getelementptr i8, ptr %ptr, i64 %offset - store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2 + store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !6 ret void } @@ -237,11 +237,11 @@ define void @relaxed_store_hint_roX_double(ptr %ptr, i64 %offset, double %val) n define void @relaxed_store_hint_uimm_i8(ptr %ptr, i8 %val) nounwind { ; CHECK-LABEL: relaxed_store_hint_uimm_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: stshh strm +; CHECK-NEXT: shuh ph ; CHECK-NEXT: strb w1, [x0, #4095] ; CHECK-NEXT: ret %addr = getelementptr i8, ptr %ptr, i32 4095 - store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2 + store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !8 ret void } @@ -460,3 +460,9 @@ define void @relaxed_store_hint_imm_double(ptr %ptr, double %val) nounwind { !1 = !{!"aarch64.mem_hint", i32 0} !2 = !{i32 1, !3} !3 = !{!"aarch64.mem_hint", i32 1} +!4 = !{i32 1, !5} +!5 = !{!"aarch64.mem_hint", i32 2} +!6 = !{i32 1, !7} +!7 = !{!"aarch64.mem_hint", i32 3} +!8 = !{i32 1, !9} +!9 = !{!"aarch64.mem_hint", i32 4} >From 9181f1536f6916aa58893abad6ba0a1dba0c323a Mon Sep 17 00:00:00 2001 From: Marian Lukac <[email protected]> Date: Wed, 30 Sep 2026 11:39:27 +0000 Subject: [PATCH 2/4] refactor isAtomic* functions and address review comments --- clang/test/CodeGen/builtins-arm64.c | 2 +- .../Target/AArch64/AArch64ISelDAGToDAG.cpp | 32 +++---------------- .../lib/Target/AArch64/AArch64InstrAtomics.td | 32 +++++++++---------- .../Atomics/aarch64-atomic-store-hint.ll | 30 +++++++++++++++++ .../Atomics/aarch64-relaxed-store-hint.ll | 2 +- 5 files changed, 52 insertions(+), 46 deletions(-) diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c index 9c3d810d55974..b100236258580 100644 --- a/clang/test/CodeGen/builtins-arm64.c +++ b/clang/test/CodeGen/builtins-arm64.c @@ -241,7 +241,7 @@ void atomic_store_with_hint(int64_t *a, int64_t b) { // Invalid hint should be dropped __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 5); // Invalid Hint - // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8{{$}} + // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8 } // CHECK: ![[M0]] = !{!"1:2:3:4:5"} diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp index aa07673673ce2..2416936309923 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -518,11 +518,7 @@ class AArch64DAGToDAGISel : public SelectionDAGISel { bool SelectCMP_SWAP(SDNode *N); AArch64MemoryHint decodeMemoryHintFlags(MachineMemOperand *MMO) const; - bool isAtomicSTSHH_KEEP(SDNode *N) const; - bool isAtomicSTSHH_STRM(SDNode *N) const; - bool isAtomicSTCPH(SDNode *N) const; - bool isAtomicSHUH(SDNode *N) const; - bool isAtomicSHUH_PH(SDNode *N) const; + bool isAtomicMemoryHint(SDNode *N, AArch64MemoryHint Hint) const; bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift, bool Negate); @@ -4640,29 +4636,9 @@ AArch64DAGToDAGISel::decodeMemoryHintFlags(MachineMemOperand *MMO) const { return toAArch64MemoryHint(MemoryHint); } -bool AArch64DAGToDAGISel::isAtomicSTSHH_KEEP(SDNode *N) const { - return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == - AArch64MemoryHint::STSHH_KEEP; -} - -bool AArch64DAGToDAGISel::isAtomicSTSHH_STRM(SDNode *N) const { - return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == - AArch64MemoryHint::STSHH_STRM; -} - -bool AArch64DAGToDAGISel::isAtomicSTCPH(SDNode *N) const { - return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == - AArch64MemoryHint::STCPH; -} - -bool AArch64DAGToDAGISel::isAtomicSHUH(SDNode *N) const { - return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == - AArch64MemoryHint::SHUH; -} - -bool AArch64DAGToDAGISel::isAtomicSHUH_PH(SDNode *N) const { - return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == - AArch64MemoryHint::SHUH_PH; +bool AArch64DAGToDAGISel::isAtomicMemoryHint(SDNode *N, + AArch64MemoryHint Hint) const { + return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == Hint; } bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td index 56e4821fc20b8..09464116b2bfb 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td +++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td @@ -423,22 +423,22 @@ multiclass AtomicHintPatterns<int Relaxed, int Hint, code Pred> { } let AddedComplexity = 15 in { - defm : AtomicHintPatternsAddrMode<48, [{ return isAtomicSTSHH_KEEP(N); }]>; - defm : AtomicHintPatternsAddrMode<49, [{ return isAtomicSTSHH_STRM(N); }]>; - defm : AtomicHintPatternsAddrMode<52, [{ return isAtomicSTCPH(N); }]>; - defm : AtomicHintPatternsAddrMode<50, [{ return isAtomicSHUH(N); }]>; - defm : AtomicHintPatternsAddrMode<51, [{ return isAtomicSHUH_PH(N); }]>; - - defm : AtomicHintPatterns<1, 48, [{ return isAtomicSTSHH_KEEP(N); }]>; - defm : AtomicHintPatterns<0, 48, [{ return isAtomicSTSHH_KEEP(N); }]>; - defm : AtomicHintPatterns<1, 49, [{ return isAtomicSTSHH_STRM(N); }]>; - defm : AtomicHintPatterns<0, 49, [{ return isAtomicSTSHH_STRM(N); }]>; - defm : AtomicHintPatterns<1, 52, [{ return isAtomicSTCPH(N); }]>; - defm : AtomicHintPatterns<0, 52, [{ return isAtomicSTCPH(N); }]>; - defm : AtomicHintPatterns<1, 50, [{ return isAtomicSHUH(N); }]>; - defm : AtomicHintPatterns<0, 50, [{ return isAtomicSHUH(N); }]>; - defm : AtomicHintPatterns<1, 51, [{ return isAtomicSHUH_PH(N); }]>; - defm : AtomicHintPatterns<0, 51, [{ return isAtomicSHUH_PH(N); }]>; + defm : AtomicHintPatternsAddrMode<48, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STSHH_KEEP); }]>; + defm : AtomicHintPatternsAddrMode<49, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STSHH_STRM); }]>; + defm : AtomicHintPatternsAddrMode<52, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STCPH); }]>; + defm : AtomicHintPatternsAddrMode<50, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH); }]>; + defm : AtomicHintPatternsAddrMode<51, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH_PH); }]>; + + defm : AtomicHintPatterns<1, 48, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STSHH_KEEP); }]>; + defm : AtomicHintPatterns<0, 48, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STSHH_KEEP); }]>; + defm : AtomicHintPatterns<1, 49, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STSHH_STRM); }]>; + defm : AtomicHintPatterns<0, 49, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STSHH_STRM); }]>; + defm : AtomicHintPatterns<1, 52, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STCPH); }]>; + defm : AtomicHintPatterns<0, 52, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STCPH); }]>; + defm : AtomicHintPatterns<1, 50, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH); }]>; + defm : AtomicHintPatterns<0, 50, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH); }]>; + defm : AtomicHintPatterns<1, 51, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH_PH); }]>; + defm : AtomicHintPatterns<0, 51, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH_PH); }]>; } //===---------------------------------- diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll index b015d020dc07f..c989a5e40e091 100644 --- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll +++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll @@ -342,6 +342,16 @@ define void @test_atomic_store_stcph_release_i64(ptr %ptr, i64 %val) nounwind { ret void } +define void @test_atomic_store_stcph_seqcst_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: test_atomic_store_stcph_seqcst_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: stcph +; CHECK-NEXT: stlr x1, [x0] +; CHECK-NEXT: ret + store atomic i64 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !4 + ret void +} + ; ; SHUH ; @@ -366,6 +376,16 @@ define void @test_atomic_store_shuh_release_i64(ptr %ptr, i64 %val) nounwind { ret void } +define void @test_atomic_store_shuh_seqcst_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_seqcst_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh +; CHECK-NEXT: stlr x1, [x0] +; CHECK-NEXT: ret + store atomic i64 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !6 + ret void +} + ; ; SHUH PH ; @@ -390,6 +410,16 @@ define void @test_atomic_store_shuh_ph_release_i64(ptr %ptr, i64 %val) nounwind ret void } +define void @test_atomic_store_shuh_ph_seqcst_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_ph_seqcst_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh ph +; CHECK-NEXT: stlr x1, [x0] +; CHECK-NEXT: ret + store atomic i64 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !8 + ret void +} + ; ; Invalid Hint ; diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll index b862ae7f4a15d..74d50efa988bd 100644 --- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll +++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll @@ -124,7 +124,7 @@ define void @relaxed_store_hint_roW_double(ptr %ptr, i32 %offset, double %val) n define void @relaxed_store_hint_roX_i8(ptr %ptr, i64 %offset, i8 %val) nounwind { ; CHECK-LABEL: relaxed_store_hint_roX_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: shuh{{$}} +; CHECK-NEXT: shuh ; CHECK-NEXT: strb w2, [x0, x1] ; CHECK-NEXT: ret %addr = getelementptr i8, ptr %ptr, i64 %offset >From 52d0edcdfbd5ec15dc236be83bd56fe8e9ff35ea Mon Sep 17 00:00:00 2001 From: Marian Lukac <[email protected]> Date: Wed, 30 Sep 2026 11:44:39 +0000 Subject: [PATCH 3/4] formatting --- llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp index 2416936309923..43d3027d5bafa 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -4637,7 +4637,7 @@ AArch64DAGToDAGISel::decodeMemoryHintFlags(MachineMemOperand *MMO) const { } bool AArch64DAGToDAGISel::isAtomicMemoryHint(SDNode *N, - AArch64MemoryHint Hint) const { + AArch64MemoryHint Hint) const { return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == Hint; } >From 5fda50e9abfae92a04e8ec4ec33d839dd77cc2fa Mon Sep 17 00:00:00 2001 From: Marian Lukac <[email protected]> Date: Thu, 1 Oct 2026 10:55:30 +0000 Subject: [PATCH 4/4] Add tests --- .../Atomics/aarch64-atomic-store-hint.ll | 300 +++++++++++++++++- 1 file changed, 297 insertions(+), 3 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll index c989a5e40e091..2528a4079b032 100644 --- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll +++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll @@ -319,7 +319,7 @@ define void @test_atomic_store_stream_seqcst_double(ptr %ptr, double %val) nounw } ; -; STCPH +; STCPH, Relaxed ; define void @test_atomic_store_stcph_relaxed_i8(ptr %ptr, i8 %val) nounwind { @@ -332,6 +332,70 @@ define void @test_atomic_store_stcph_relaxed_i8(ptr %ptr, i8 %val) nounwind { ret void } +define void @test_atomic_store_stcph_relaxed_i16(ptr %ptr, i16 %val) nounwind { +; CHECK-LABEL: test_atomic_store_stcph_relaxed_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: stcph +; CHECK-NEXT: strh w1, [x0] +; CHECK-NEXT: ret + store atomic i16 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !4 + ret void +} + +define void @test_atomic_store_stcph_relaxed_i32(ptr %ptr, i32 %val) nounwind { +; CHECK-LABEL: test_atomic_store_stcph_relaxed_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: stcph +; CHECK-NEXT: str w1, [x0] +; CHECK-NEXT: ret + store atomic i32 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !4 + ret void +} + +define void @test_atomic_store_stcph_relaxed_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: test_atomic_store_stcph_relaxed_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: stcph +; CHECK-NEXT: str x1, [x0] +; CHECK-NEXT: ret + store atomic i64 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !4 + ret void +} + +; +; STCPH, Release +; + +define void @test_atomic_store_stcph_release_i8(ptr %ptr, i8 %val) nounwind { +; CHECK-LABEL: test_atomic_store_stcph_release_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: stcph +; CHECK-NEXT: stlrb w1, [x0] +; CHECK-NEXT: ret + store atomic i8 %val, ptr %ptr release, align 8, !mem.cache_hint !4 + ret void +} + +define void @test_atomic_store_stcph_release_i16(ptr %ptr, i16 %val) nounwind { +; CHECK-LABEL: test_atomic_store_stcph_release_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: stcph +; CHECK-NEXT: stlrh w1, [x0] +; CHECK-NEXT: ret + store atomic i16 %val, ptr %ptr release, align 8, !mem.cache_hint !4 + ret void +} + +define void @test_atomic_store_stcph_release_i32(ptr %ptr, i32 %val) nounwind { +; CHECK-LABEL: test_atomic_store_stcph_release_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: stcph +; CHECK-NEXT: stlr w1, [x0] +; CHECK-NEXT: ret + store atomic i32 %val, ptr %ptr release, align 8, !mem.cache_hint !4 + ret void +} + define void @test_atomic_store_stcph_release_i64(ptr %ptr, i64 %val) nounwind { ; CHECK-LABEL: test_atomic_store_stcph_release_i64: ; CHECK: // %bb.0: @@ -342,6 +406,40 @@ define void @test_atomic_store_stcph_release_i64(ptr %ptr, i64 %val) nounwind { ret void } +; +; STCPH, SequentiallyConsistent +; + +define void @test_atomic_store_stcph_seqcst_i8(ptr %ptr, i8 %val) nounwind { +; CHECK-LABEL: test_atomic_store_stcph_seqcst_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: stcph +; CHECK-NEXT: stlrb w1, [x0] +; CHECK-NEXT: ret + store atomic i8 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !4 + ret void +} + +define void @test_atomic_store_stcph_seqcst_i16(ptr %ptr, i16 %val) nounwind { +; CHECK-LABEL: test_atomic_store_stcph_seqcst_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: stcph +; CHECK-NEXT: stlrh w1, [x0] +; CHECK-NEXT: ret + store atomic i16 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !4 + ret void +} + +define void @test_atomic_store_stcph_seqcst_i32(ptr %ptr, i32 %val) nounwind { +; CHECK-LABEL: test_atomic_store_stcph_seqcst_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: stcph +; CHECK-NEXT: stlr w1, [x0] +; CHECK-NEXT: ret + store atomic i32 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !4 + ret void +} + define void @test_atomic_store_stcph_seqcst_i64(ptr %ptr, i64 %val) nounwind { ; CHECK-LABEL: test_atomic_store_stcph_seqcst_i64: ; CHECK: // %bb.0: @@ -353,7 +451,7 @@ define void @test_atomic_store_stcph_seqcst_i64(ptr %ptr, i64 %val) nounwind { } ; -; SHUH +; SHUH, Relaxed ; define void @test_atomic_store_shuh_relaxed_i8(ptr %ptr, i8 %val) nounwind { @@ -366,6 +464,70 @@ define void @test_atomic_store_shuh_relaxed_i8(ptr %ptr, i8 %val) nounwind { ret void } +define void @test_atomic_store_shuh_relaxed_i16(ptr %ptr, i16 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_relaxed_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh +; CHECK-NEXT: strh w1, [x0] +; CHECK-NEXT: ret + store atomic i16 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !6 + ret void +} + +define void @test_atomic_store_shuh_relaxed_i32(ptr %ptr, i32 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_relaxed_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh +; CHECK-NEXT: str w1, [x0] +; CHECK-NEXT: ret + store atomic i32 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !6 + ret void +} + +define void @test_atomic_store_shuh_relaxed_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_relaxed_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh +; CHECK-NEXT: str x1, [x0] +; CHECK-NEXT: ret + store atomic i64 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !6 + ret void +} + +; +; SHUH, Release +; + +define void @test_atomic_store_shuh_release_i8(ptr %ptr, i8 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_release_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh +; CHECK-NEXT: stlrb w1, [x0] +; CHECK-NEXT: ret + store atomic i8 %val, ptr %ptr release, align 8, !mem.cache_hint !6 + ret void +} + +define void @test_atomic_store_shuh_release_i16(ptr %ptr, i16 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_release_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh +; CHECK-NEXT: stlrh w1, [x0] +; CHECK-NEXT: ret + store atomic i16 %val, ptr %ptr release, align 8, !mem.cache_hint !6 + ret void +} + +define void @test_atomic_store_shuh_release_i32(ptr %ptr, i32 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_release_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh +; CHECK-NEXT: stlr w1, [x0] +; CHECK-NEXT: ret + store atomic i32 %val, ptr %ptr release, align 8, !mem.cache_hint !6 + ret void +} + define void @test_atomic_store_shuh_release_i64(ptr %ptr, i64 %val) nounwind { ; CHECK-LABEL: test_atomic_store_shuh_release_i64: ; CHECK: // %bb.0: @@ -376,6 +538,40 @@ define void @test_atomic_store_shuh_release_i64(ptr %ptr, i64 %val) nounwind { ret void } +; +; SHUH, SequentiallyConsistent +; + +define void @test_atomic_store_shuh_seqcst_i8(ptr %ptr, i8 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_seqcst_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh +; CHECK-NEXT: stlrb w1, [x0] +; CHECK-NEXT: ret + store atomic i8 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !6 + ret void +} + +define void @test_atomic_store_shuh_seqcst_i16(ptr %ptr, i16 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_seqcst_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh +; CHECK-NEXT: stlrh w1, [x0] +; CHECK-NEXT: ret + store atomic i16 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !6 + ret void +} + +define void @test_atomic_store_shuh_seqcst_i32(ptr %ptr, i32 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_seqcst_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh +; CHECK-NEXT: stlr w1, [x0] +; CHECK-NEXT: ret + store atomic i32 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !6 + ret void +} + define void @test_atomic_store_shuh_seqcst_i64(ptr %ptr, i64 %val) nounwind { ; CHECK-LABEL: test_atomic_store_shuh_seqcst_i64: ; CHECK: // %bb.0: @@ -387,7 +583,7 @@ define void @test_atomic_store_shuh_seqcst_i64(ptr %ptr, i64 %val) nounwind { } ; -; SHUH PH +; SHUH PH, Relaxed ; define void @test_atomic_store_shuh_ph_relaxed_i8(ptr %ptr, i8 %val) nounwind { @@ -400,6 +596,70 @@ define void @test_atomic_store_shuh_ph_relaxed_i8(ptr %ptr, i8 %val) nounwind { ret void } +define void @test_atomic_store_shuh_ph_relaxed_i16(ptr %ptr, i16 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_ph_relaxed_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh ph +; CHECK-NEXT: strh w1, [x0] +; CHECK-NEXT: ret + store atomic i16 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !8 + ret void +} + +define void @test_atomic_store_shuh_ph_relaxed_i32(ptr %ptr, i32 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_ph_relaxed_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh ph +; CHECK-NEXT: str w1, [x0] +; CHECK-NEXT: ret + store atomic i32 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !8 + ret void +} + +define void @test_atomic_store_shuh_ph_relaxed_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_ph_relaxed_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh ph +; CHECK-NEXT: str x1, [x0] +; CHECK-NEXT: ret + store atomic i64 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !8 + ret void +} + +; +; SHUH PH, Release +; + +define void @test_atomic_store_shuh_ph_release_i8(ptr %ptr, i8 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_ph_release_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh ph +; CHECK-NEXT: stlrb w1, [x0] +; CHECK-NEXT: ret + store atomic i8 %val, ptr %ptr release, align 8, !mem.cache_hint !8 + ret void +} + +define void @test_atomic_store_shuh_ph_release_i16(ptr %ptr, i16 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_ph_release_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh ph +; CHECK-NEXT: stlrh w1, [x0] +; CHECK-NEXT: ret + store atomic i16 %val, ptr %ptr release, align 8, !mem.cache_hint !8 + ret void +} + +define void @test_atomic_store_shuh_ph_release_i32(ptr %ptr, i32 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_ph_release_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh ph +; CHECK-NEXT: stlr w1, [x0] +; CHECK-NEXT: ret + store atomic i32 %val, ptr %ptr release, align 8, !mem.cache_hint !8 + ret void +} + define void @test_atomic_store_shuh_ph_release_i64(ptr %ptr, i64 %val) nounwind { ; CHECK-LABEL: test_atomic_store_shuh_ph_release_i64: ; CHECK: // %bb.0: @@ -410,6 +670,40 @@ define void @test_atomic_store_shuh_ph_release_i64(ptr %ptr, i64 %val) nounwind ret void } +; +; SHUH PH, SequentiallyConsistent +; + +define void @test_atomic_store_shuh_ph_seqcst_i8(ptr %ptr, i8 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_ph_seqcst_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh ph +; CHECK-NEXT: stlrb w1, [x0] +; CHECK-NEXT: ret + store atomic i8 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !8 + ret void +} + +define void @test_atomic_store_shuh_ph_seqcst_i16(ptr %ptr, i16 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_ph_seqcst_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh ph +; CHECK-NEXT: stlrh w1, [x0] +; CHECK-NEXT: ret + store atomic i16 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !8 + ret void +} + +define void @test_atomic_store_shuh_ph_seqcst_i32(ptr %ptr, i32 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_ph_seqcst_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh ph +; CHECK-NEXT: stlr w1, [x0] +; CHECK-NEXT: ret + store atomic i32 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !8 + ret void +} + define void @test_atomic_store_shuh_ph_seqcst_i64(ptr %ptr, i64 %val) nounwind { ; CHECK-LABEL: test_atomic_store_shuh_ph_seqcst_i64: ; CHECK: // %bb.0: _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
