https://github.com/Lukacma updated https://github.com/llvm/llvm-project/pull/227711
>From ea8313ea0030b8838d8b212b9771d025ffd11b3c Mon Sep 17 00:00:00 2001 From: Marian Lukac <[email protected]> Date: Wed, 16 Sep 2026 12:37:14 +0000 Subject: [PATCH 1/6] [AArch64] Add CMH hints to store_with_hint intrinsic --- clang/lib/Headers/arm_acle.h | 3 + clang/test/CodeGen/builtins-arm64.c | 19 ++++- clang/test/Sema/builtins-arm64.c | 3 +- .../include/llvm/Support/AArch64MemoryHints.h | 11 ++- .../Target/AArch64/AArch64ISelDAGToDAG.cpp | 18 +++++ .../lib/Target/AArch64/AArch64InstrAtomics.td | 9 +++ .../Atomics/aarch64-atomic-store-hint.ll | 80 ++++++++++++++++++- .../Atomics/aarch64-relaxed-store-hint.ll | 18 +++-- 8 files changed, 150 insertions(+), 11 deletions(-) diff --git a/clang/lib/Headers/arm_acle.h b/clang/lib/Headers/arm_acle.h index 715fe851b0c32..395b760e7dbf0 100644 --- a/clang/lib/Headers/arm_acle.h +++ b/clang/lib/Headers/arm_acle.h @@ -746,6 +746,9 @@ __arm_st64bv0(void *__addr, data512_t __value) { #if defined(__ARM_64BIT_STATE) && __ARM_64BIT_STATE #define HINT_STSHH_KEEP 0 #define HINT_STSHH_STRM 1 +#define HINT_STCPH 2 +#define HINT_SHUH 3 +#define HINT_SHUH_PH 4 #define __arm_atomic_store_with_hint(ptr, data, memory_order, hint) \ __builtin_arm_atomic_store_with_hint(ptr, data, memory_order, hint) #endif diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c index fb297cd3eec2c..9c3d810d55974 100644 --- a/clang/test/CodeGen/builtins-arm64.c +++ b/clang/test/CodeGen/builtins-arm64.c @@ -230,9 +230,18 @@ void atomic_store_with_hint(int64_t *a, int64_t b) { __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELEASE, HINT_STSHH_STRM); // CHECK: store atomic i64 {{.*}}, ptr {{.*}} release, align 8, !mem.cache_hint ![[M3:[0-9]+]] + __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, HINT_STCPH); + // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8, !mem.cache_hint ![[M5:[0-9]+]] + + __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELEASE, HINT_SHUH); + // CHECK: store atomic i64 {{.*}}, ptr {{.*}} release, align 8, !mem.cache_hint ![[M7:[0-9]+]] + + __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_SEQ_CST, HINT_SHUH_PH); + // CHECK: store atomic i64 {{.*}}, ptr {{.*}} seq_cst, align 8, !mem.cache_hint ![[M9:[0-9]+]] + // Invalid hint should be dropped - __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 2); // Invalid Hint - // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8 + __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 5); // Invalid Hint + // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8{{$}} } // CHECK: ![[M0]] = !{!"1:2:3:4:5"} @@ -240,3 +249,9 @@ void atomic_store_with_hint(int64_t *a, int64_t b) { // CHECK: ![[M2]] = !{!"aarch64.mem_hint", i32 0} // CHECK: ![[M3]] = !{i32 1, ![[M4:[0-9]+]]} // CHECK: ![[M4]] = !{!"aarch64.mem_hint", i32 1} +// CHECK: ![[M5]] = !{i32 1, ![[M6:[0-9]+]]} +// CHECK: ![[M6]] = !{!"aarch64.mem_hint", i32 2} +// CHECK: ![[M7]] = !{i32 1, ![[M8:[0-9]+]]} +// CHECK: ![[M8]] = !{!"aarch64.mem_hint", i32 3} +// CHECK: ![[M9]] = !{i32 1, ![[M10:[0-9]+]]} +// CHECK: ![[M10]] = !{!"aarch64.mem_hint", i32 4} diff --git a/clang/test/Sema/builtins-arm64.c b/clang/test/Sema/builtins-arm64.c index 9a6e0715c4c78..a79fb9177cec8 100644 --- a/clang/test/Sema/builtins-arm64.c +++ b/clang/test/Sema/builtins-arm64.c @@ -73,7 +73,8 @@ void test_atomic_store_hint(char *c_ptr, __int128 *inv_ptr, float *f_ptr, __builtin_arm_atomic_store_with_hint(c_ptr, c_data, inv_int, 0); // expected-error {{invalid memory order argument to atomic hint operation ('int' invalid)}} __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 2, 0); // expected-error {{invalid memory order argument to atomic hint operation (2 invalid)}} - __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 3); // expected-warning {{unrecognised hint type argument to atomic hint operation (3)}} + __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 5); // expected-warning {{unrecognised hint type argument to atomic hint operation (5)}} + __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, inv_int); // expected-error {{invalid hint type argument to atomic hint operation ('int')}} __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, "h"); // expected-error {{incompatible pointer to integer conversion passing 'char *' to parameter of type 'int'}} diff --git a/llvm/include/llvm/Support/AArch64MemoryHints.h b/llvm/include/llvm/Support/AArch64MemoryHints.h index b917191499cac..f47eb0c945d1b 100644 --- a/llvm/include/llvm/Support/AArch64MemoryHints.h +++ b/llvm/include/llvm/Support/AArch64MemoryHints.h @@ -14,11 +14,14 @@ enum class AArch64MemoryHint { NONE = 0, STSHH_KEEP = 1, STSHH_STRM = 2, + STCPH = 3, + SHUH = 4, + SHUH_PH = 5, }; template <typename Int> inline bool isValidAArch64MemoryHintValue(Int I) { return (Int)AArch64MemoryHint::STSHH_KEEP <= I && - I <= (Int)AArch64MemoryHint::STSHH_STRM; + I <= (Int)AArch64MemoryHint::SHUH_PH; } template <typename Int> inline AArch64MemoryHint toAArch64MemoryHint(Int I) { @@ -27,6 +30,12 @@ template <typename Int> inline AArch64MemoryHint toAArch64MemoryHint(Int I) { return AArch64MemoryHint::STSHH_KEEP; case 1: return AArch64MemoryHint::STSHH_STRM; + case 2: + return AArch64MemoryHint::STCPH; + case 3: + return AArch64MemoryHint::SHUH; + case 4: + return AArch64MemoryHint::SHUH_PH; default: return AArch64MemoryHint::NONE; } diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp index 48c0e37151b9c..aa07673673ce2 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -520,6 +520,9 @@ class AArch64DAGToDAGISel : public SelectionDAGISel { AArch64MemoryHint decodeMemoryHintFlags(MachineMemOperand *MMO) const; bool isAtomicSTSHH_KEEP(SDNode *N) const; bool isAtomicSTSHH_STRM(SDNode *N) const; + bool isAtomicSTCPH(SDNode *N) const; + bool isAtomicSHUH(SDNode *N) const; + bool isAtomicSHUH_PH(SDNode *N) const; bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift, bool Negate); @@ -4647,6 +4650,21 @@ bool AArch64DAGToDAGISel::isAtomicSTSHH_STRM(SDNode *N) const { AArch64MemoryHint::STSHH_STRM; } +bool AArch64DAGToDAGISel::isAtomicSTCPH(SDNode *N) const { + return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == + AArch64MemoryHint::STCPH; +} + +bool AArch64DAGToDAGISel::isAtomicSHUH(SDNode *N) const { + return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == + AArch64MemoryHint::SHUH; +} + +bool AArch64DAGToDAGISel::isAtomicSHUH_PH(SDNode *N) const { + return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == + AArch64MemoryHint::SHUH_PH; +} + bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift, bool Negate) { if (!isa<ConstantSDNode>(N)) diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td index 7e0e75c7a3c6b..56e4821fc20b8 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td +++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td @@ -425,11 +425,20 @@ multiclass AtomicHintPatterns<int Relaxed, int Hint, code Pred> { let AddedComplexity = 15 in { defm : AtomicHintPatternsAddrMode<48, [{ return isAtomicSTSHH_KEEP(N); }]>; defm : AtomicHintPatternsAddrMode<49, [{ return isAtomicSTSHH_STRM(N); }]>; + defm : AtomicHintPatternsAddrMode<52, [{ return isAtomicSTCPH(N); }]>; + defm : AtomicHintPatternsAddrMode<50, [{ return isAtomicSHUH(N); }]>; + defm : AtomicHintPatternsAddrMode<51, [{ return isAtomicSHUH_PH(N); }]>; defm : AtomicHintPatterns<1, 48, [{ return isAtomicSTSHH_KEEP(N); }]>; defm : AtomicHintPatterns<0, 48, [{ return isAtomicSTSHH_KEEP(N); }]>; defm : AtomicHintPatterns<1, 49, [{ return isAtomicSTSHH_STRM(N); }]>; defm : AtomicHintPatterns<0, 49, [{ return isAtomicSTSHH_STRM(N); }]>; + defm : AtomicHintPatterns<1, 52, [{ return isAtomicSTCPH(N); }]>; + defm : AtomicHintPatterns<0, 52, [{ return isAtomicSTCPH(N); }]>; + defm : AtomicHintPatterns<1, 50, [{ return isAtomicSHUH(N); }]>; + defm : AtomicHintPatterns<0, 50, [{ return isAtomicSHUH(N); }]>; + defm : AtomicHintPatterns<1, 51, [{ return isAtomicSHUH_PH(N); }]>; + defm : AtomicHintPatterns<0, 51, [{ return isAtomicSHUH_PH(N); }]>; } //===---------------------------------- diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll index bfa9fc49cf752..b015d020dc07f 100644 --- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll +++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll @@ -318,6 +318,78 @@ define void @test_atomic_store_stream_seqcst_double(ptr %ptr, double %val) nounw ret void } +; +; STCPH +; + +define void @test_atomic_store_stcph_relaxed_i8(ptr %ptr, i8 %val) nounwind { +; CHECK-LABEL: test_atomic_store_stcph_relaxed_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: stcph +; CHECK-NEXT: strb w1, [x0] +; CHECK-NEXT: ret + store atomic i8 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !4 + ret void +} + +define void @test_atomic_store_stcph_release_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: test_atomic_store_stcph_release_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: stcph +; CHECK-NEXT: stlr x1, [x0] +; CHECK-NEXT: ret + store atomic i64 %val, ptr %ptr release, align 8, !mem.cache_hint !4 + ret void +} + +; +; SHUH +; + +define void @test_atomic_store_shuh_relaxed_i8(ptr %ptr, i8 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_relaxed_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh{{$}} +; CHECK-NEXT: strb w1, [x0] +; CHECK-NEXT: ret + store atomic i8 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !6 + ret void +} + +define void @test_atomic_store_shuh_release_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_release_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh{{$}} +; CHECK-NEXT: stlr x1, [x0] +; CHECK-NEXT: ret + store atomic i64 %val, ptr %ptr release, align 8, !mem.cache_hint !6 + ret void +} + +; +; SHUH PH +; + +define void @test_atomic_store_shuh_ph_relaxed_i8(ptr %ptr, i8 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_ph_relaxed_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh ph +; CHECK-NEXT: strb w1, [x0] +; CHECK-NEXT: ret + store atomic i8 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !8 + ret void +} + +define void @test_atomic_store_shuh_ph_release_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_ph_release_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh ph +; CHECK-NEXT: stlr x1, [x0] +; CHECK-NEXT: ret + store atomic i64 %val, ptr %ptr release, align 8, !mem.cache_hint !8 + ret void +} + ; ; Invalid Hint ; @@ -327,7 +399,7 @@ define void @test_atomic_store_invalid_hint(ptr %ptr, i8 %val) nounwind { ; CHECK: // %bb.0: ; CHECK-NEXT: stlrb w1, [x0] ; CHECK-NEXT: ret - store atomic i8 %val, ptr %ptr release, align 8, !mem.cache_hint !4 + store atomic i8 %val, ptr %ptr release, align 8, !mem.cache_hint !10 ret void } @@ -337,3 +409,9 @@ define void @test_atomic_store_invalid_hint(ptr %ptr, i8 %val) nounwind { !3 = !{!"aarch64.mem_hint", i32 1} !4 = !{i32 1, !5} !5 = !{!"aarch64.mem_hint", i32 2} +!6 = !{i32 1, !7} +!7 = !{!"aarch64.mem_hint", i32 3} +!8 = !{i32 1, !9} +!9 = !{!"aarch64.mem_hint", i32 4} +!10 = !{i32 1, !11} +!11 = !{!"aarch64.mem_hint", i32 5} diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll index 5f5bc43039778..b862ae7f4a15d 100644 --- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll +++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll @@ -11,11 +11,11 @@ define void @relaxed_store_hint_roW_i8(ptr %ptr, i32 %offset, i8 %val) nounwind { ; CHECK-LABEL: relaxed_store_hint_roW_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: stshh keep +; CHECK-NEXT: stcph ; CHECK-NEXT: strb w2, [x0, w1, sxtw] ; CHECK-NEXT: ret %addr = getelementptr i8, ptr %ptr, i32 %offset - store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0 + store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !4 ret void } @@ -124,11 +124,11 @@ define void @relaxed_store_hint_roW_double(ptr %ptr, i32 %offset, double %val) n define void @relaxed_store_hint_roX_i8(ptr %ptr, i64 %offset, i8 %val) nounwind { ; CHECK-LABEL: relaxed_store_hint_roX_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: stshh strm +; CHECK-NEXT: shuh{{$}} ; CHECK-NEXT: strb w2, [x0, x1] ; CHECK-NEXT: ret %addr = getelementptr i8, ptr %ptr, i64 %offset - store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2 + store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !6 ret void } @@ -237,11 +237,11 @@ define void @relaxed_store_hint_roX_double(ptr %ptr, i64 %offset, double %val) n define void @relaxed_store_hint_uimm_i8(ptr %ptr, i8 %val) nounwind { ; CHECK-LABEL: relaxed_store_hint_uimm_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: stshh strm +; CHECK-NEXT: shuh ph ; CHECK-NEXT: strb w1, [x0, #4095] ; CHECK-NEXT: ret %addr = getelementptr i8, ptr %ptr, i32 4095 - store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2 + store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !8 ret void } @@ -460,3 +460,9 @@ define void @relaxed_store_hint_imm_double(ptr %ptr, double %val) nounwind { !1 = !{!"aarch64.mem_hint", i32 0} !2 = !{i32 1, !3} !3 = !{!"aarch64.mem_hint", i32 1} +!4 = !{i32 1, !5} +!5 = !{!"aarch64.mem_hint", i32 2} +!6 = !{i32 1, !7} +!7 = !{!"aarch64.mem_hint", i32 3} +!8 = !{i32 1, !9} +!9 = !{!"aarch64.mem_hint", i32 4} >From 9181f1536f6916aa58893abad6ba0a1dba0c323a Mon Sep 17 00:00:00 2001 From: Marian Lukac <[email protected]> Date: Wed, 30 Sep 2026 11:39:27 +0000 Subject: [PATCH 2/6] refactor isAtomic* functions and address review comments --- clang/test/CodeGen/builtins-arm64.c | 2 +- .../Target/AArch64/AArch64ISelDAGToDAG.cpp | 32 +++---------------- .../lib/Target/AArch64/AArch64InstrAtomics.td | 32 +++++++++---------- .../Atomics/aarch64-atomic-store-hint.ll | 30 +++++++++++++++++ .../Atomics/aarch64-relaxed-store-hint.ll | 2 +- 5 files changed, 52 insertions(+), 46 deletions(-) diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c index 9c3d810d55974..b100236258580 100644 --- a/clang/test/CodeGen/builtins-arm64.c +++ b/clang/test/CodeGen/builtins-arm64.c @@ -241,7 +241,7 @@ void atomic_store_with_hint(int64_t *a, int64_t b) { // Invalid hint should be dropped __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 5); // Invalid Hint - // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8{{$}} + // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8 } // CHECK: ![[M0]] = !{!"1:2:3:4:5"} diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp index aa07673673ce2..2416936309923 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -518,11 +518,7 @@ class AArch64DAGToDAGISel : public SelectionDAGISel { bool SelectCMP_SWAP(SDNode *N); AArch64MemoryHint decodeMemoryHintFlags(MachineMemOperand *MMO) const; - bool isAtomicSTSHH_KEEP(SDNode *N) const; - bool isAtomicSTSHH_STRM(SDNode *N) const; - bool isAtomicSTCPH(SDNode *N) const; - bool isAtomicSHUH(SDNode *N) const; - bool isAtomicSHUH_PH(SDNode *N) const; + bool isAtomicMemoryHint(SDNode *N, AArch64MemoryHint Hint) const; bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift, bool Negate); @@ -4640,29 +4636,9 @@ AArch64DAGToDAGISel::decodeMemoryHintFlags(MachineMemOperand *MMO) const { return toAArch64MemoryHint(MemoryHint); } -bool AArch64DAGToDAGISel::isAtomicSTSHH_KEEP(SDNode *N) const { - return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == - AArch64MemoryHint::STSHH_KEEP; -} - -bool AArch64DAGToDAGISel::isAtomicSTSHH_STRM(SDNode *N) const { - return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == - AArch64MemoryHint::STSHH_STRM; -} - -bool AArch64DAGToDAGISel::isAtomicSTCPH(SDNode *N) const { - return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == - AArch64MemoryHint::STCPH; -} - -bool AArch64DAGToDAGISel::isAtomicSHUH(SDNode *N) const { - return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == - AArch64MemoryHint::SHUH; -} - -bool AArch64DAGToDAGISel::isAtomicSHUH_PH(SDNode *N) const { - return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == - AArch64MemoryHint::SHUH_PH; +bool AArch64DAGToDAGISel::isAtomicMemoryHint(SDNode *N, + AArch64MemoryHint Hint) const { + return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == Hint; } bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td index 56e4821fc20b8..09464116b2bfb 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td +++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td @@ -423,22 +423,22 @@ multiclass AtomicHintPatterns<int Relaxed, int Hint, code Pred> { } let AddedComplexity = 15 in { - defm : AtomicHintPatternsAddrMode<48, [{ return isAtomicSTSHH_KEEP(N); }]>; - defm : AtomicHintPatternsAddrMode<49, [{ return isAtomicSTSHH_STRM(N); }]>; - defm : AtomicHintPatternsAddrMode<52, [{ return isAtomicSTCPH(N); }]>; - defm : AtomicHintPatternsAddrMode<50, [{ return isAtomicSHUH(N); }]>; - defm : AtomicHintPatternsAddrMode<51, [{ return isAtomicSHUH_PH(N); }]>; - - defm : AtomicHintPatterns<1, 48, [{ return isAtomicSTSHH_KEEP(N); }]>; - defm : AtomicHintPatterns<0, 48, [{ return isAtomicSTSHH_KEEP(N); }]>; - defm : AtomicHintPatterns<1, 49, [{ return isAtomicSTSHH_STRM(N); }]>; - defm : AtomicHintPatterns<0, 49, [{ return isAtomicSTSHH_STRM(N); }]>; - defm : AtomicHintPatterns<1, 52, [{ return isAtomicSTCPH(N); }]>; - defm : AtomicHintPatterns<0, 52, [{ return isAtomicSTCPH(N); }]>; - defm : AtomicHintPatterns<1, 50, [{ return isAtomicSHUH(N); }]>; - defm : AtomicHintPatterns<0, 50, [{ return isAtomicSHUH(N); }]>; - defm : AtomicHintPatterns<1, 51, [{ return isAtomicSHUH_PH(N); }]>; - defm : AtomicHintPatterns<0, 51, [{ return isAtomicSHUH_PH(N); }]>; + defm : AtomicHintPatternsAddrMode<48, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STSHH_KEEP); }]>; + defm : AtomicHintPatternsAddrMode<49, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STSHH_STRM); }]>; + defm : AtomicHintPatternsAddrMode<52, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STCPH); }]>; + defm : AtomicHintPatternsAddrMode<50, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH); }]>; + defm : AtomicHintPatternsAddrMode<51, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH_PH); }]>; + + defm : AtomicHintPatterns<1, 48, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STSHH_KEEP); }]>; + defm : AtomicHintPatterns<0, 48, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STSHH_KEEP); }]>; + defm : AtomicHintPatterns<1, 49, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STSHH_STRM); }]>; + defm : AtomicHintPatterns<0, 49, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STSHH_STRM); }]>; + defm : AtomicHintPatterns<1, 52, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STCPH); }]>; + defm : AtomicHintPatterns<0, 52, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STCPH); }]>; + defm : AtomicHintPatterns<1, 50, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH); }]>; + defm : AtomicHintPatterns<0, 50, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH); }]>; + defm : AtomicHintPatterns<1, 51, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH_PH); }]>; + defm : AtomicHintPatterns<0, 51, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH_PH); }]>; } //===---------------------------------- diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll index b015d020dc07f..c989a5e40e091 100644 --- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll +++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll @@ -342,6 +342,16 @@ define void @test_atomic_store_stcph_release_i64(ptr %ptr, i64 %val) nounwind { ret void } +define void @test_atomic_store_stcph_seqcst_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: test_atomic_store_stcph_seqcst_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: stcph +; CHECK-NEXT: stlr x1, [x0] +; CHECK-NEXT: ret + store atomic i64 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !4 + ret void +} + ; ; SHUH ; @@ -366,6 +376,16 @@ define void @test_atomic_store_shuh_release_i64(ptr %ptr, i64 %val) nounwind { ret void } +define void @test_atomic_store_shuh_seqcst_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_seqcst_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh +; CHECK-NEXT: stlr x1, [x0] +; CHECK-NEXT: ret + store atomic i64 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !6 + ret void +} + ; ; SHUH PH ; @@ -390,6 +410,16 @@ define void @test_atomic_store_shuh_ph_release_i64(ptr %ptr, i64 %val) nounwind ret void } +define void @test_atomic_store_shuh_ph_seqcst_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: test_atomic_store_shuh_ph_seqcst_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh ph +; CHECK-NEXT: stlr x1, [x0] +; CHECK-NEXT: ret + store atomic i64 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !8 + ret void +} + ; ; Invalid Hint ; diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll index b862ae7f4a15d..74d50efa988bd 100644 --- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll +++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll @@ -124,7 +124,7 @@ define void @relaxed_store_hint_roW_double(ptr %ptr, i32 %offset, double %val) n define void @relaxed_store_hint_roX_i8(ptr %ptr, i64 %offset, i8 %val) nounwind { ; CHECK-LABEL: relaxed_store_hint_roX_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: shuh{{$}} +; CHECK-NEXT: shuh ; CHECK-NEXT: strb w2, [x0, x1] ; CHECK-NEXT: ret %addr = getelementptr i8, ptr %ptr, i64 %offset >From 52d0edcdfbd5ec15dc236be83bd56fe8e9ff35ea Mon Sep 17 00:00:00 2001 From: Marian Lukac <[email protected]> Date: Wed, 30 Sep 2026 11:44:39 +0000 Subject: [PATCH 3/6] formatting --- llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp index 2416936309923..43d3027d5bafa 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -4637,7 +4637,7 @@ AArch64DAGToDAGISel::decodeMemoryHintFlags(MachineMemOperand *MMO) const { } bool AArch64DAGToDAGISel::isAtomicMemoryHint(SDNode *N, - AArch64MemoryHint Hint) const { + AArch64MemoryHint Hint) const { return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == Hint; } >From 201de51d24648d961d02490f497635aed95eff6e Mon Sep 17 00:00:00 2001 From: Marian Lukac <[email protected]> Date: Wed, 30 Sep 2026 12:24:37 +0000 Subject: [PATCH 4/6] [AArch64] Implement the atomic fetch with hint intrinsic --- clang/include/clang/Basic/BuiltinsAArch64.td | 5 + .../clang/Basic/DiagnosticSemaKinds.td | 4 +- clang/include/clang/Sema/SemaARM.h | 2 +- clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 94 ++++++++-- clang/lib/Headers/arm_acle.h | 12 +- clang/lib/Sema/SemaARM.cpp | 51 +++-- .../AArch64/atomic-store-hint-template.cpp | 31 +++ clang/test/CodeGen/arm_acle.c | 111 +++++++++++ clang/test/CodeGen/builtins-arm64.c | 53 ++++++ clang/test/Sema/builtins-arm64.c | 30 +++ llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 21 +++ .../Target/AArch64/AArch64ISelLowering.cpp | 6 +- .../lib/Target/AArch64/AArch64InstrAtomics.td | 75 +++++++- .../Atomics/aarch64-atomic-store-hint.ll | 176 +++++++++++++++++- 14 files changed, 629 insertions(+), 42 deletions(-) diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td index 30aa3d526cbbb..39ea7102b5d65 100644 --- a/clang/include/clang/Basic/BuiltinsAArch64.td +++ b/clang/include/clang/Basic/BuiltinsAArch64.td @@ -173,6 +173,11 @@ let Attributes = [NoThrow], Features = "ls64" in { let Attributes = [NoThrow, CustomTypeChecking] in { def atomic_store_with_hint : AArch64Builtin<"void(...)">; + def atomic_fetch_add_with_hint : AArch64Builtin<"void(...)">; + def atomic_fetch_sub_with_hint : AArch64Builtin<"void(...)">; + def atomic_fetch_and_with_hint : AArch64Builtin<"void(...)">; + def atomic_fetch_xor_with_hint : AArch64Builtin<"void(...)">; + def atomic_fetch_or_with_hint : AArch64Builtin<"void(...)">; } // Armv9.3-A Guarded Control Stack diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index d293a9798da6a..f1c9b7e64f87f 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -9618,7 +9618,7 @@ def err_atomic_builtin_must_be_pointer_intptr : Error< def err_atomic_hint_builtin_must_be_pointer : Error< "address argument to atomic hint builtin must be a pointer to a scalar " - "integral or floating-point type of 8, 16, 32, or 64 bits (%0 invalid)">; + "%select{integral|integral or floating-point}1 type of 8, 16, 32, or 64 bits (%0 invalid)">; def err_atomic_hint_has_invalid_hint_type : Error< "invalid hint type argument to atomic hint operation (%0)">; @@ -9691,7 +9691,7 @@ def err_atomic_op_needs_atomic_fp : Error<"address argument to atomic operation must be a pointer to " "%select{|atomic }0floating point type (%1 invalid)">; def err_atomic_op_hint_data_size : Error< - "address argument to atomic store with hint must be of size 8, 16, 32 or 64 bits">; + "address argument to atomic %select{fetch|store}0 with hint must be of size 8, 16, 32 or 64 bits">; def err_atomic_hint_has_invalid_memory_order : Error< "invalid memory order argument to atomic hint operation (%0 invalid)">; def warn_atomic_hint_has_invalid_hint_type : Warning< diff --git a/clang/include/clang/Sema/SemaARM.h b/clang/include/clang/Sema/SemaARM.h index b0a01c40ffece..76a06f4f4221a 100644 --- a/clang/include/clang/Sema/SemaARM.h +++ b/clang/include/clang/Sema/SemaARM.h @@ -70,7 +70,7 @@ class SemaARM : public SemaBase { bool BuiltinARMSpecialReg(unsigned BuiltinID, CallExpr *TheCall, int ArgNum, unsigned ExpectedFieldNum, bool AllowName); bool BuiltinARMMemoryTaggingCall(unsigned BuiltinID, CallExpr *TheCall); - bool BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, CallExpr *TheCall); + bool BuiltinARMAtomicHintCall(unsigned BuiltinID, CallExpr *TheCall); bool MveAliasValid(unsigned BuiltinID, llvm::StringRef AliasName); bool CdeAliasValid(unsigned BuiltinID, llvm::StringRef AliasName); diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp index e20fa60ee132b..3293c263527d4 100644 --- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp +++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp @@ -2066,6 +2066,29 @@ static Value *EmitRangePrefetchBuiltin(CodeGenFunction &CGF, unsigned BuiltinID, Ops); } +static void AttachAtomicHintMetadata(CodeGenFunction &CGF, const CallExpr *E, + Instruction *Atomic, + unsigned PtrOperand) { + CodeGen::CGBuilderTy &Builder = CGF.Builder; + LLVMContext &Ctx = CGF.CGM.getLLVMContext(); + Expr::EvalResult Result; + if (!E->getArg(3)->EvaluateAsInt(Result, CGF.getContext())) + llvm_unreachable( + "Expected integer hint argument to atomic operation with hint."); + unsigned HintArg = Result.Val.getInt().getExtValue(); + + if (toAArch64MemoryHint(HintArg) == AArch64MemoryHint::NONE) + return; + + MDNode *MemHint = MDNode::get( + Ctx, {MDString::get(Ctx, "aarch64.mem_hint"), + llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg))}); + MDNode *HintNode = MDNode::get( + Ctx, {llvm::ConstantAsMetadata::get(Builder.getInt32(PtrOperand)), + MemHint}); + Atomic->setMetadata(llvm::LLVMContext::MD_mem_cache_hint, HintNode); +} + static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E) { @@ -2105,26 +2128,53 @@ static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF, break; } Store->setAtomic(Ordering); + AttachAtomicHintMetadata(CGF, E, Store, 1); + return Store; +} - if (!E->getArg(3)->EvaluateAsInt(Result, CGM.getContext())) +static Value *EmitAtomicFetchWithHintBuiltin(CodeGenFunction &CGF, + const CallExpr *E, + AtomicRMWInst::BinOp Op) { + CodeGen::CGBuilderTy &Builder = CGF.Builder; + Expr::EvalResult Result; + if (!E->getArg(2)->EvaluateAsInt(Result, CGF.getContext())) llvm_unreachable( - "Expected integer hint argument to atomic store with hint."); - unsigned HintArg = Result.Val.getInt().getExtValue(); + "Expected integer policy argument to atomic fetch with hint."); - // Attach the hint if valid - if (toAArch64MemoryHint(HintArg) != AArch64MemoryHint::NONE) { - LLVMContext &Ctx = CGM.getLLVMContext(); - MDNode *MemHint = MDNode::get( - Ctx, {MDString::get(Ctx, "aarch64.mem_hint"), - llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg))}); - MDNode *HintNode = MDNode::get( - CGM.getLLVMContext(), - {llvm::ConstantAsMetadata::get(Builder.getInt32(1)), MemHint}); + AtomicOrdering Ordering; + unsigned OrderingArg = Result.Val.getInt().getExtValue(); + assert(isValidAtomicOrderingCABI(OrderingArg) && "Invalid atomic ordering"); - Store->setMetadata(llvm::LLVMContext::MD_mem_cache_hint, HintNode); + switch (static_cast<AtomicOrderingCABI>(OrderingArg)) { + case AtomicOrderingCABI::relaxed: + Ordering = AtomicOrdering::Monotonic; + break; + case AtomicOrderingCABI::consume: + case AtomicOrderingCABI::acquire: + Ordering = AtomicOrdering::Acquire; + break; + case AtomicOrderingCABI::release: + Ordering = AtomicOrdering::Release; + break; + case AtomicOrderingCABI::acq_rel: + Ordering = AtomicOrdering::AcquireRelease; + break; + case AtomicOrderingCABI::seq_cst: + Ordering = AtomicOrdering::SequentiallyConsistent; + break; } - return Store; + const Expr *Ptr = E->getArg(0); + Address Addr = CGF.EmitPointerWithAlignment(Ptr); + Addr = Addr.withElementType( + CGF.ConvertTypeForMem(Ptr->getType()->getPointeeType())); + const Expr *Data = E->getArg(1); + Value *DataVal = CGF.EmitToMemory(CGF.EmitScalarExpr(Data), Data->getType()); + + AtomicRMWInst *RMW = Builder.CreateAtomicRMW(Op, Addr, DataVal, Ordering); + RMW->setVolatile(Ptr->getType()->getPointeeType().isVolatileQualified()); + AttachAtomicHintMetadata(CGF, E, RMW, 0); + return CGF.EmitFromMemory(RMW, E->getType()); } /// Return true if BuiltinID is an overloaded Neon intrinsic with an extra @@ -4902,8 +4952,22 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, BuiltinID == AArch64::BI__builtin_arm_range_prefetch_x) return EmitRangePrefetchBuiltin(*this, BuiltinID, E); - if (BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint) + switch (BuiltinID) { + default: + break; + case AArch64::BI__builtin_arm_atomic_store_with_hint: return EmitAtomicStoreWithHintBuiltin(*this, BuiltinID, E); + case AArch64::BI__builtin_arm_atomic_fetch_add_with_hint: + return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::Add); + case AArch64::BI__builtin_arm_atomic_fetch_sub_with_hint: + return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::Sub); + case AArch64::BI__builtin_arm_atomic_fetch_and_with_hint: + return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::And); + case AArch64::BI__builtin_arm_atomic_fetch_xor_with_hint: + return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::Xor); + case AArch64::BI__builtin_arm_atomic_fetch_or_with_hint: + return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::Or); + } // Memory Tagging Extensions (MTE) Intrinsics Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic; diff --git a/clang/lib/Headers/arm_acle.h b/clang/lib/Headers/arm_acle.h index 395b760e7dbf0..cfbe490d995bf 100644 --- a/clang/lib/Headers/arm_acle.h +++ b/clang/lib/Headers/arm_acle.h @@ -742,7 +742,7 @@ __arm_st64bv0(void *__addr, data512_t __value) { } #endif -/* Atomic store with hints */ +/* Atomic intrinsics with hints */ #if defined(__ARM_64BIT_STATE) && __ARM_64BIT_STATE #define HINT_STSHH_KEEP 0 #define HINT_STSHH_STRM 1 @@ -751,6 +751,16 @@ __arm_st64bv0(void *__addr, data512_t __value) { #define HINT_SHUH_PH 4 #define __arm_atomic_store_with_hint(ptr, data, memory_order, hint) \ __builtin_arm_atomic_store_with_hint(ptr, data, memory_order, hint) +#define __arm_atomic_fetch_add_with_hint(ptr, data, memory_order, hint) \ + __builtin_arm_atomic_fetch_add_with_hint(ptr, data, memory_order, hint) +#define __arm_atomic_fetch_sub_with_hint(ptr, data, memory_order, hint) \ + __builtin_arm_atomic_fetch_sub_with_hint(ptr, data, memory_order, hint) +#define __arm_atomic_fetch_and_with_hint(ptr, data, memory_order, hint) \ + __builtin_arm_atomic_fetch_and_with_hint(ptr, data, memory_order, hint) +#define __arm_atomic_fetch_xor_with_hint(ptr, data, memory_order, hint) \ + __builtin_arm_atomic_fetch_xor_with_hint(ptr, data, memory_order, hint) +#define __arm_atomic_fetch_or_with_hint(ptr, data, memory_order, hint) \ + __builtin_arm_atomic_fetch_or_with_hint(ptr, data, memory_order, hint) #endif /* 11.1 Special register intrinsics */ diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp index a5037fce03c6b..2f5a9f985a81d 100644 --- a/clang/lib/Sema/SemaARM.cpp +++ b/clang/lib/Sema/SemaARM.cpp @@ -323,14 +323,16 @@ bool SemaARM::BuiltinARMSpecialReg(unsigned BuiltinID, CallExpr *TheCall, return false; } -bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, - CallExpr *TheCall) { +bool SemaARM::BuiltinARMAtomicHintCall(unsigned BuiltinID, CallExpr *TheCall) { if (SemaRef.checkArgCount(TheCall, 4)) return true; - // Arg 0 should be the pointer type. The pointee type must be a - // scalar integral or floating-point type of 8, 16, 32 or 64 bits. + bool IsStore = BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint; ASTContext &Context = getASTContext(); + + // Arg 0 should be the pointer type. The pointee type must be a + // scalar integral type of 8, 16, 32 or 64 bits. Stores also support + // floating-point types. auto PtrArgRes = SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(0)); if (PtrArgRes.isInvalid()) @@ -340,7 +342,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, if (!PtrTy) return Diag(TheCall->getBeginLoc(), diag::err_atomic_hint_builtin_must_be_pointer) - << PtrArg->getType() << 0 << PtrArg->getSourceRange(); + << PtrArg->getType() << IsStore << PtrArg->getSourceRange(); TheCall->setArg(0, PtrArg); QualType PtrQT = Context.getCanonicalType(PtrTy->getPointeeType()); @@ -350,10 +352,11 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, << PtrQT << PtrArg->getSourceRange(); PtrQT = PtrQT.getUnqualifiedType(); - if (!PtrQT->isIntegralType(Context) && !PtrQT->isFloatingType() && - !PtrQT->isMFloat8Type()) + if (!PtrQT->isIntegralType(Context) && + !(IsStore && (PtrQT->isFloatingType() || PtrQT->isMFloat8Type()))) return Diag(TheCall->getBeginLoc(), - diag::err_atomic_op_needs_atomic_int_or_fp) + IsStore ? diag::err_atomic_op_needs_atomic_int_or_fp + : diag::err_atomic_op_needs_atomic_int) << 0 << PtrQT << PtrArg->getSourceRange(); if (PtrQT->isBitIntType()) @@ -364,9 +367,12 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, unsigned TySize = Context.getTypeSize(PtrQT); if (TySize != 8 && TySize != 16 && TySize != 32 && TySize != 64) return Diag(TheCall->getBeginLoc(), diag::err_atomic_op_hint_data_size) - << PtrArg->getSourceRange(); + << IsStore << PtrArg->getSourceRange(); + + if (!IsStore) + TheCall->setType(PtrQT); - // Arg 1 is the data to be stored. The type must match the pointee + // Arg 1 is the data operand. The type must match the pointee // type found above. auto DataArgRes = SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(1)); @@ -382,7 +388,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, diag::err_typecheck_call_different_arg_types) << PtrQT << DataQT; - // Arg 2 is the memory order, which must be relaxed, release or seq_cst + // Arg 2 is the memory order. Stores only allow relaxed, release or seq_cst. auto MemOrdArg = SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(2)); if (MemOrdArg.isInvalid()) @@ -407,7 +413,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, << *MemOrdAP << MemOrd->getSourceRange(); auto AtomicOrdering = static_cast<llvm::AtomicOrderingCABI>(Ordering); - if (AtomicOrdering != llvm::AtomicOrderingCABI::relaxed && + if (IsStore && AtomicOrdering != llvm::AtomicOrderingCABI::relaxed && AtomicOrdering != llvm::AtomicOrderingCABI::release && AtomicOrdering != llvm::AtomicOrderingCABI::seq_cst) return Diag(TheCall->getBeginLoc(), @@ -416,7 +422,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, } // Arg 3 is the hint type. Only values represented by AArch64MemoryHint - // are valid. + // are valid. Fetch operations only allow HINT_SHUH and HINT_SHUH_PH. auto HintArg = SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(3)); if (HintArg.isInvalid()) @@ -433,8 +439,14 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, diag::err_atomic_hint_has_invalid_hint_type) << Hint->getType() << Hint->getSourceRange(); - if (llvm::toAArch64MemoryHint(HintAP->getZExtValue()) == - llvm::AArch64MemoryHint::NONE) { + auto MemoryHint = llvm::toAArch64MemoryHint(HintAP->getZExtValue()); + if (!IsStore && MemoryHint != llvm::AArch64MemoryHint::SHUH && + MemoryHint != llvm::AArch64MemoryHint::SHUH_PH) + return Diag(TheCall->getBeginLoc(), + diag::err_atomic_hint_has_invalid_hint_type) + << *HintAP << Hint->getSourceRange(); + + if (MemoryHint == llvm::AArch64MemoryHint::NONE) { Diag(TheCall->getBeginLoc(), diag::warn_atomic_hint_has_invalid_hint_type) << *HintAP << Hint->getSourceRange(); return false; @@ -1291,8 +1303,13 @@ bool SemaARM::CheckAArch64BuiltinFunctionCall(const TargetInfo &TI, BuiltinID == AArch64::BI__builtin_arm_wsrp) return BuiltinARMSpecialReg(BuiltinID, TheCall, 0, 5, true); - if (BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint) - return BuiltinARMAtomicStoreHintCall(BuiltinID, TheCall); + if (BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint || + BuiltinID == AArch64::BI__builtin_arm_atomic_fetch_add_with_hint || + BuiltinID == AArch64::BI__builtin_arm_atomic_fetch_sub_with_hint || + BuiltinID == AArch64::BI__builtin_arm_atomic_fetch_and_with_hint || + BuiltinID == AArch64::BI__builtin_arm_atomic_fetch_xor_with_hint || + BuiltinID == AArch64::BI__builtin_arm_atomic_fetch_or_with_hint) + return BuiltinARMAtomicHintCall(BuiltinID, TheCall); // Only check the valid encoding range. Any constant in this range would be // converted to a register of the form S2_2_C3_C4_5. Let the hardware throw diff --git a/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp b/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp index ecf1e10a09b5d..17e99be63adcd 100644 --- a/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp +++ b/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp @@ -55,9 +55,40 @@ void test_atomic_store_hint_array_consexpr_hint(int *ptr, int value) { __builtin_arm_atomic_store_with_hint(ptr, value, __ATOMIC_RELAXED, HintVal{}); } +template <unsigned Order> +int test_atomic_fetch_hint_template_order(int *ptr, int val) { + return __arm_atomic_fetch_add_with_hint(ptr, val, Order, 3); +} + +template int test_atomic_fetch_hint_template_order<__ATOMIC_ACQUIRE>(int *, int); + +template <unsigned Hint> +int test_atomic_fetch_hint_template_hint(int *ptr, int val) { + return __arm_atomic_fetch_xor_with_hint(ptr, val, __ATOMIC_SEQ_CST, Hint); +} + +template int test_atomic_fetch_hint_template_hint<4>(int *, int); + +// CHECK-LABEL: define weak_odr noundef i32 @_Z37test_atomic_fetch_hint_template_orderILj2EEiPii( +// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VAL:%.*]]) #[[ATTR0]] comdat { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = atomicrmw add ptr [[PTR]], i32 [[VAL]] acquire, align 4, !mem.cache_hint [[META8:![0-9]+]] +// CHECK-NEXT: ret i32 [[TMP0]] +// +// +// CHECK-LABEL: define weak_odr noundef i32 @_Z36test_atomic_fetch_hint_template_hintILj4EEiPii( +// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VAL:%.*]]) #[[ATTR0]] comdat { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = atomicrmw xor ptr [[PTR]], i32 [[VAL]] seq_cst, align 4, !mem.cache_hint [[META10:![0-9]+]] +// CHECK-NEXT: ret i32 [[TMP0]] +// //. // CHECK: [[META4]] = !{i32 1, [[META5:![0-9]+]]} // CHECK: [[META5]] = !{!"aarch64.mem_hint", i32 0} // CHECK: [[META6]] = !{i32 1, [[META7:![0-9]+]]} // CHECK: [[META7]] = !{!"aarch64.mem_hint", i32 1} +// CHECK: [[META8]] = !{i32 0, [[META9:![0-9]+]]} +// CHECK: [[META9]] = !{!"aarch64.mem_hint", i32 3} +// CHECK: [[META10]] = !{i32 0, [[META11:![0-9]+]]} +// CHECK: [[META11]] = !{!"aarch64.mem_hint", i32 4} //. diff --git a/clang/test/CodeGen/arm_acle.c b/clang/test/CodeGen/arm_acle.c index 60ec3e48e9498..2acb0373632c5 100644 --- a/clang/test/CodeGen/arm_acle.c +++ b/clang/test/CodeGen/arm_acle.c @@ -1989,9 +1989,120 @@ void test_atomic_store_hint_array_arg() { __builtin_arm_atomic_store_with_hint(storage, 0, __ATOMIC_RELAXED, HINT_STSHH_STRM); } +// AArch64-LABEL: @test_atomic_fetch_hint_char( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[PTR:%.*]], i8 [[DATA:%.*]] monotonic, align 1, !mem.cache_hint [[FETCH0:![0-9]+]] +// AArch64-NEXT: ret i8 [[OLD]] +// +char test_atomic_fetch_hint_char(char *ptr, char data) { + return __arm_atomic_fetch_add_with_hint(ptr, data, __ATOMIC_RELAXED, 3); +} + +// AArch64-LABEL: @test_atomic_fetch_hint_uchar( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[OLD:%.*]] = atomicrmw sub ptr [[PTR:%.*]], i8 [[DATA:%.*]] release, align 1, !mem.cache_hint [[FETCH1:![0-9]+]] +// AArch64-NEXT: ret i8 [[OLD]] +// +unsigned char test_atomic_fetch_hint_uchar(unsigned char *ptr, unsigned char data) { + return __arm_atomic_fetch_sub_with_hint(ptr, data, __ATOMIC_RELEASE, 4); +} + +// AArch64-LABEL: @test_atomic_fetch_hint_short( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[OLD:%.*]] = atomicrmw and ptr [[PTR:%.*]], i16 [[DATA:%.*]] acquire, align 2, !mem.cache_hint [[FETCH0]] +// AArch64-NEXT: ret i16 [[OLD]] +// +short test_atomic_fetch_hint_short(short *ptr, short data) { + return __arm_atomic_fetch_and_with_hint(ptr, data, __ATOMIC_ACQUIRE, 3); +} + +// AArch64-LABEL: @test_atomic_fetch_hint_ushort( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[OLD:%.*]] = atomicrmw xor ptr [[PTR:%.*]], i16 [[DATA:%.*]] acq_rel, align 2, !mem.cache_hint [[FETCH1]] +// AArch64-NEXT: ret i16 [[OLD]] +// +unsigned short test_atomic_fetch_hint_ushort(unsigned short *ptr, unsigned short data) { + return __arm_atomic_fetch_xor_with_hint(ptr, data, __ATOMIC_ACQ_REL, 4); +} + +// AArch64-LABEL: @test_atomic_fetch_hint_int( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[OLD:%.*]] = atomicrmw or ptr [[PTR:%.*]], i32 [[DATA:%.*]] seq_cst, align 4, !mem.cache_hint [[FETCH0]] +// AArch64-NEXT: ret i32 [[OLD]] +// +int test_atomic_fetch_hint_int(int *ptr, int data) { + return __arm_atomic_fetch_or_with_hint(ptr, data, __ATOMIC_SEQ_CST, 3); +} + +// AArch64-LABEL: @test_atomic_fetch_hint_unsigned( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[PTR:%.*]], i32 [[DATA:%.*]] acquire, align 4, !mem.cache_hint [[FETCH1]] +// AArch64-NEXT: ret i32 [[OLD]] +// +unsigned test_atomic_fetch_hint_unsigned(unsigned *ptr, unsigned data) { + return __arm_atomic_fetch_add_with_hint(ptr, data, __ATOMIC_CONSUME, 4); +} + +// AArch64-LABEL: @test_atomic_fetch_hint_long( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[OLD:%.*]] = atomicrmw sub ptr [[PTR:%.*]], i64 [[DATA:%.*]] monotonic, align 8, !mem.cache_hint [[FETCH0]] +// AArch64-NEXT: ret i64 [[OLD]] +// +long test_atomic_fetch_hint_long(long *ptr, long data) { + return __arm_atomic_fetch_sub_with_hint(ptr, data, __ATOMIC_RELAXED, 3); +} + +// AArch64-LABEL: @test_atomic_fetch_hint_bool( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[STOREDV:%.*]] = zext i1 [[DATA:%.*]] to i8 +// AArch64-NEXT: [[LOADEDV:%.*]] = icmp ne i8 [[STOREDV]], 0 +// AArch64-NEXT: [[STOREDV1:%.*]] = zext i1 [[LOADEDV]] to i8 +// AArch64-NEXT: [[OLD:%.*]] = atomicrmw xor ptr [[PTR:%.*]], i8 [[STOREDV1]] monotonic, align 1, !mem.cache_hint [[FETCH0]] +// AArch64-NEXT: [[LOADEDV1:%.*]] = icmp ne i8 [[OLD]], 0 +// AArch64-NEXT: ret i1 [[LOADEDV1]] +// +bool test_atomic_fetch_hint_bool(bool *ptr, bool data) { + return __arm_atomic_fetch_xor_with_hint(ptr, data, __ATOMIC_RELAXED, 3); +} + +// AArch64-LABEL: @test_atomic_fetch_hint_volatile( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[OLD:%.*]] = atomicrmw volatile or ptr [[PTR:%.*]], i32 [[DATA:%.*]] acq_rel, align 4, !mem.cache_hint [[FETCH0]] +// AArch64-NEXT: ret i32 [[OLD]] +// +int test_atomic_fetch_hint_volatile(volatile int *ptr, int data) { + return __arm_atomic_fetch_or_with_hint(ptr, data, __ATOMIC_ACQ_REL, 3); +} + +// AArch64-LABEL: @test_atomic_fetch_hint_typedef( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[PTR:%.*]], i32 [[DATA:%.*]] release, align 4, !mem.cache_hint [[FETCH1]] +// AArch64-NEXT: ret i32 [[OLD]] +// +aliased_int test_atomic_fetch_hint_typedef(aliased_int *ptr, const int data) { + return __arm_atomic_fetch_add_with_hint(ptr, data, __ATOMIC_RELEASE, 4); +} + +// AArch64-LABEL: @test_atomic_fetch_hint_array_arg( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[STORAGE:%.*]] = alloca [1 x i32], align 4 +// AArch64-NEXT: call void @llvm.memset.p0.i64(ptr align 4 [[STORAGE]], i8 0, i64 4, i1 false) +// AArch64-NEXT: [[ARRAYDECAY:%.*]] = getelementptr inbounds [1 x i32], ptr [[STORAGE]], i64 0, i64 0 +// AArch64-NEXT: [[OLD:%.*]] = atomicrmw sub ptr [[ARRAYDECAY]], i32 1 monotonic, align 4, !mem.cache_hint [[FETCH1]] +// AArch64-NEXT: ret i32 [[OLD]] +// +int test_atomic_fetch_hint_array_arg(void) { + int storage[1] = {0}; + return __arm_atomic_fetch_sub_with_hint(storage, 1, __ATOMIC_RELAXED, 4); +} + // AArch64: [[HINT1]] = !{i32 1, [[HINT2:![0-9]+]]} // AArch64-NEXT: [[HINT2]] = !{!"aarch64.mem_hint", i32 0} // AArch64-NEXT: [[HINT3]] = !{i32 1, [[HINT4:![0-9]+]]} // AArch64-NEXT: [[HINT4]] = !{!"aarch64.mem_hint", i32 1} +// AArch64: [[FETCH0]] = !{i32 0, [[FETCH_HINT0:![0-9]+]]} +// AArch64-NEXT: [[FETCH_HINT0]] = !{!"aarch64.mem_hint", i32 3} +// AArch64-NEXT: [[FETCH1]] = !{i32 0, [[FETCH_HINT1:![0-9]+]]} +// AArch64-NEXT: [[FETCH_HINT1]] = !{!"aarch64.mem_hint", i32 4} #endif diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c index b100236258580..33960feb6cdb2 100644 --- a/clang/test/CodeGen/builtins-arm64.c +++ b/clang/test/CodeGen/builtins-arm64.c @@ -242,6 +242,57 @@ void atomic_store_with_hint(int64_t *a, int64_t b) { // Invalid hint should be dropped __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 5); // Invalid Hint // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8 + // CHECK-NOT: !mem.cache_hint + // CHECK-NEXT: ret void +} + +// CHECK-LABEL: @atomic_fetch_add_with_hint( +// CHECK: [[OLD:%.*]] = atomicrmw add ptr {{.*}}, i64 {{.*}} monotonic, align 8, !mem.cache_hint ![[F1:[0-9]+]] +// CHECK-NEXT: ret i64 [[OLD]] +int64_t atomic_fetch_add_with_hint(int64_t *a, int64_t b) { + return __builtin_arm_atomic_fetch_add_with_hint(a, b, __ATOMIC_RELAXED, 3); +} + +// CHECK-LABEL: @atomic_fetch_sub_with_hint( +// CHECK: [[OLD:%.*]] = atomicrmw sub ptr {{.*}}, i64 {{.*}} release, align 8, !mem.cache_hint ![[F3:[0-9]+]] +// CHECK-NEXT: ret i64 [[OLD]] +int64_t atomic_fetch_sub_with_hint(int64_t *a, int64_t b) { + return __builtin_arm_atomic_fetch_sub_with_hint(a, b, __ATOMIC_RELEASE, 4); +} + +// CHECK-LABEL: @atomic_fetch_and_with_hint( +// CHECK: [[OLD:%.*]] = atomicrmw and ptr {{.*}}, i64 {{.*}} acquire, align 8, !mem.cache_hint ![[F1]] +// CHECK-NEXT: ret i64 [[OLD]] +int64_t atomic_fetch_and_with_hint(int64_t *a, int64_t b) { + return __builtin_arm_atomic_fetch_and_with_hint(a, b, __ATOMIC_ACQUIRE, 3); +} + +// CHECK-LABEL: @atomic_fetch_xor_with_hint( +// CHECK: [[OLD:%.*]] = atomicrmw xor ptr {{.*}}, i64 {{.*}} acq_rel, align 8, !mem.cache_hint ![[F1]] +// CHECK-NEXT: ret i64 [[OLD]] +int64_t atomic_fetch_xor_with_hint(int64_t *a, int64_t b) { + return __builtin_arm_atomic_fetch_xor_with_hint(a, b, __ATOMIC_ACQ_REL, 3); +} + +// CHECK-LABEL: @atomic_fetch_or_with_hint( +// CHECK: [[OLD:%.*]] = atomicrmw or ptr {{.*}}, i64 {{.*}} seq_cst, align 8, !mem.cache_hint ![[F3]] +// CHECK-NEXT: ret i64 [[OLD]] +int64_t atomic_fetch_or_with_hint(int64_t *a, int64_t b) { + return __builtin_arm_atomic_fetch_or_with_hint(a, b, __ATOMIC_SEQ_CST, 4); +} + +// CHECK-LABEL: @atomic_fetch_with_hint_consume( +// CHECK: [[OLD:%.*]] = atomicrmw add ptr {{.*}}, i64 {{.*}} acquire, align 8, !mem.cache_hint ![[F1]] +// CHECK-NEXT: ret i64 [[OLD]] +int64_t atomic_fetch_with_hint_consume(int64_t *a, int64_t b) { + return __builtin_arm_atomic_fetch_add_with_hint(a, b, __ATOMIC_CONSUME, 3); +} + +// CHECK-LABEL: @atomic_fetch_add_with_hint_ph( +// CHECK: [[OLD:%.*]] = atomicrmw add ptr {{.*}}, i64 {{.*}} monotonic, align 8, !mem.cache_hint ![[F3]] +// CHECK-NEXT: ret i64 [[OLD]] +int64_t atomic_fetch_add_with_hint_ph(int64_t *a, int64_t b) { + return __builtin_arm_atomic_fetch_add_with_hint(a, b, __ATOMIC_RELAXED, 4); } // CHECK: ![[M0]] = !{!"1:2:3:4:5"} @@ -255,3 +306,5 @@ void atomic_store_with_hint(int64_t *a, int64_t b) { // CHECK: ![[M8]] = !{!"aarch64.mem_hint", i32 3} // CHECK: ![[M9]] = !{i32 1, ![[M10:[0-9]+]]} // CHECK: ![[M10]] = !{!"aarch64.mem_hint", i32 4} +// CHECK: ![[F1]] = !{i32 0, ![[M8]]} +// CHECK: ![[F3]] = !{i32 0, ![[M10]]} diff --git a/clang/test/Sema/builtins-arm64.c b/clang/test/Sema/builtins-arm64.c index a79fb9177cec8..ba0cf233df319 100644 --- a/clang/test/Sema/builtins-arm64.c +++ b/clang/test/Sema/builtins-arm64.c @@ -80,3 +80,33 @@ void test_atomic_store_hint(char *c_ptr, __int128 *inv_ptr, float *f_ptr, __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, "h"); // expected-error {{incompatible pointer to integer conversion passing 'char *' to parameter of type 'int'}} // expected-error@-1 {{invalid hint type argument to atomic hint operation ('int')}} } + +void test_atomic_fetch_hint(char *c_ptr, char c_data, const char *const_c_ptr, + float *f_ptr, float f_data, __int128 *wide_ptr, + __int128 wide_data, unsigned _BitInt(8) *bit_ptr, + unsigned _BitInt(8) bit_data, int variable, + int **ptr_ptr, _Atomic int *atomic_ptr) { + __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, 0); // expected-error {{too few arguments to function call, expected 4, have 3}} + __builtin_arm_atomic_fetch_sub_with_hint(c_ptr, c_data, 0, 3, 0); // expected-error {{too many arguments to function call, expected 4, have 5}} + + __builtin_arm_atomic_fetch_and_with_hint(0, c_data, 0, 3); // expected-error {{address argument to atomic hint builtin must be a pointer to a scalar integral type of 8, 16, 32, or 64 bits ('int' invalid)}} + __builtin_arm_atomic_fetch_xor_with_hint(c_ptr, f_data, 0, 3); // expected-error {{arguments are of different types ('char' vs 'float')}} + __builtin_arm_atomic_fetch_or_with_hint(const_c_ptr, c_data, 0, 3); // expected-error {{address argument to atomic operation must be a pointer to non-const type ('const char' invalid)}} + __builtin_arm_atomic_fetch_add_with_hint(f_ptr, f_data, 0, 3); // expected-error {{address argument to atomic operation must be a pointer to integer ('float' invalid)}} + __builtin_arm_atomic_fetch_sub_with_hint(wide_ptr, wide_data, 0, 3); // expected-error {{address argument to atomic fetch with hint must be of size 8, 16, 32 or 64 bits}} + __builtin_arm_atomic_fetch_and_with_hint(bit_ptr, bit_data, 0, 3); // expected-error {{argument to atomic builtin of type '_BitInt' is not supported}} + __builtin_arm_atomic_fetch_xor_with_hint(ptr_ptr, 0, 0, 3); // expected-error {{address argument to atomic operation must be a pointer to integer ('int *' invalid)}} + __builtin_arm_atomic_fetch_or_with_hint(atomic_ptr, 0, 0, 3); // expected-error {{address argument to atomic operation must be a pointer to integer ('_Atomic(int)' invalid)}} + + __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, variable, 3); // expected-error {{invalid memory order argument to atomic hint operation ('int' invalid)}} + __builtin_arm_atomic_fetch_sub_with_hint(c_ptr, c_data, -1, 3); // expected-error {{invalid memory order argument to atomic hint operation (-1 invalid)}} + __builtin_arm_atomic_fetch_and_with_hint(c_ptr, c_data, 6, 3); // expected-error {{invalid memory order argument to atomic hint operation (6 invalid)}} + + __builtin_arm_atomic_fetch_xor_with_hint(c_ptr, c_data, 0, variable); // expected-error {{invalid hint type argument to atomic hint operation ('int')}} + __builtin_arm_atomic_fetch_or_with_hint(c_ptr, c_data, 0, "h"); // expected-error {{incompatible pointer to integer conversion passing 'char *' to parameter of type 'int'}} + // expected-error@-1 {{invalid hint type argument to atomic hint operation ('int')}} + __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, 0, 0); // expected-error {{invalid hint type argument to atomic hint operation (0)}} + __builtin_arm_atomic_fetch_sub_with_hint(c_ptr, c_data, 0, 1); // expected-error {{invalid hint type argument to atomic hint operation (1)}} + __builtin_arm_atomic_fetch_and_with_hint(c_ptr, c_data, 0, 2); // expected-error {{invalid hint type argument to atomic hint operation (2)}} + __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, 0, 5); // expected-error {{invalid hint type argument to atomic hint operation (5)}} +} diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index 5a502f785e0f4..73e1a0e9b3a52 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -295,6 +295,9 @@ class AArch64AsmPrinter : public AsmPrinter { void emitAtomicHintPseudoExpansionRO(const MachineInstr *MI); void emitAtomicHintPseudoExpansionImm(const MachineInstr *MI); + // Emit expansion of atomic fetch with hint pseudo instructions + void emitAtomicFetchHintPseudoExpansion(const MachineInstr *MI); + void EmitToStreamer(MCStreamer &S, const MCInst &Inst); void EmitToStreamer(const MCInst &Inst) { EmitToStreamer(*OutStreamer, Inst); @@ -3409,6 +3412,20 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansionImm( EmitToStreamer(*OutStreamer, Store); } +void AArch64AsmPrinter::emitAtomicFetchHintPseudoExpansion( + const MachineInstr *MI) { + EmitToStreamer( + MCInstBuilder(AArch64::HINT).addImm(MI->getOperand(4).getImm())); + + MCInst Fetch; + Fetch.setOpcode(MI->getOperand(3).getImm()); + Fetch.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); + Fetch.addOperand(MCOperand::createReg(MI->getOperand(1).getReg())); + Fetch.addOperand(MCOperand::createReg(MI->getOperand(2).getReg())); + Fetch.setFlags(MI->getFlags()); + EmitToStreamer(*OutStreamer, Fetch); +} + // Simple pseudo-instructions have their lowering (with expansion to real // instructions) auto-generated. #include "AArch64GenMCPseudoLowering.inc" @@ -4132,6 +4149,10 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { case AArch64::ATOMIC_STORE_HINT_X: emitAtomicHintPseudoExpansion(MI); return; + case AArch64::ATOMIC_FETCH_HINT_W: + case AArch64::ATOMIC_FETCH_HINT_X: + emitAtomicFetchHintPseudoExpansion(MI); + return; case AArch64::ATOMIC_STORE_HINT_BroW: case AArch64::ATOMIC_STORE_HINT_HroW: case AArch64::ATOMIC_STORE_HINT_WroW: diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 2ddc7fe934430..71e3ad19cbe04 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -34099,11 +34099,15 @@ bool AArch64TargetLowering::fallBackToDAGISel(const Instruction &Inst) const { } // The !mem.cache_hint metadata is not supported by GISel and will be dropped. - // TODO: Remove this and handle atomic store hints in GISel once supported. + // TODO: Remove this and handle atomic hints in GISel once supported. if (auto *Store = dyn_cast<StoreInst>(&Inst)) { if (Store->isAtomic() && getMemCacheHintMetadata(*Store, 1)) return true; } + if (auto *RMW = dyn_cast<AtomicRMWInst>(&Inst)) { + if (getMemCacheHintMetadata(*RMW)) + return true; + } return false; } diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td index 09464116b2bfb..7568b6f530db8 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td +++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td @@ -283,7 +283,7 @@ def : Pat<(relaxed_store<atomic_store_64> (STURDi FPR64Op:$val, GPR64sp:$Rn, simm9:$offset)>; //===---------------------------------- -// Atomic store with hint pseudos +// Atomic operations with hint pseudos //===---------------------------------- let Size = 8, isCodeGenOnly = 1, hasSideEffects = 1, mayStore = 1 in { @@ -441,6 +441,79 @@ let AddedComplexity = 15 in { defm : AtomicHintPatterns<0, 51, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH_PH); }]>; } +// Keep the hint and the LSE instruction together until assembly emission. +// The opcode operand lets all operations and orderings share these pseudos. +let Size = 8, isCodeGenOnly = 1, hasSideEffects = 1, mayLoad = 1, + mayStore = 1 in { + class BaseFetchHintPseudo<RegisterClass dsttype, RegisterClass srctype> + : Pseudo<(outs dsttype:$result), + (ins srctype:$data, GPR64sp:$addr, i32imm:$opcode, + i32imm:$hint), []>, Sched<[WriteAtomic]>; + + def ATOMIC_FETCH_HINT_W : BaseFetchHintPseudo<GPR32common, GPR32>; + def ATOMIC_FETCH_HINT_X : BaseFetchHintPseudo<GPR64common, GPR64>; +} + +class atomic_hint_fetch<PatFrag Base, code Pred> + : PatFrag<(ops node:$ptr, node:$val), + (Base node:$ptr, node:$val), Pred> { + // TODO: Change once mem.cache_hint supported in GISel + let GISelPredicateCode = [{ return false; }]; +} + +// Pass the LSE opcode as an immediate so all operations and orderings can use +// the same two fetch pseudos. +class AtomicFetchOpcode<string Inst> : SDNodeXForm<imm, + "return CurDAG->getTargetConstant(AArch64::" # Inst # + ", SDLoc(N), MVT::i32);">; + +multiclass AtomicFetchHintPatternsForOpcode<string Inst, string Op, + RegisterClass RC, Instruction Pseudo> { + defvar Base = !cast<PatFrag>(Op); + defvar Opcode = AtomicFetchOpcode<Inst>; + def : Pat<(atomic_hint_fetch<Base, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH); }]> + GPR64sp:$addr, RC:$data), + (Pseudo RC:$data, GPR64sp:$addr, (Opcode (i32 0)), (i32 50))>; + def : Pat<(atomic_hint_fetch<Base, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH_PH); }]> + GPR64sp:$addr, RC:$data), + (Pseudo RC:$data, GPR64sp:$addr, (Opcode (i32 0)), (i32 51))>; +} + +multiclass AtomicFetchHintPatternsOrd<string Inst, string Suffix, string Op, + RegisterClass RC, Instruction Pseudo> { + defm : AtomicFetchHintPatternsForOpcode<Inst # Suffix, Op # "_monotonic", + RC, Pseudo>; + defm : AtomicFetchHintPatternsForOpcode<Inst # "A" # Suffix, Op # "_acquire", + RC, Pseudo>; + defm : AtomicFetchHintPatternsForOpcode<Inst # "L" # Suffix, Op # "_release", + RC, Pseudo>; + defm : AtomicFetchHintPatternsForOpcode<Inst # "AL" # Suffix, Op # "_acq_rel", + RC, Pseudo>; + defm : AtomicFetchHintPatternsForOpcode<Inst # "AL" # Suffix, Op # "_seq_cst", + RC, Pseudo>; +} + +multiclass AtomicFetchHintPatterns<string Inst, string Op> { + defm : AtomicFetchHintPatternsOrd<Inst, "X", Op # "_i64", GPR64, + ATOMIC_FETCH_HINT_X>; + defm : AtomicFetchHintPatternsOrd<Inst, "W", Op # "_i32", GPR32, + ATOMIC_FETCH_HINT_W>; + defm : AtomicFetchHintPatternsOrd<Inst, "H", Op # "_i16", GPR32, + ATOMIC_FETCH_HINT_W>; + defm : AtomicFetchHintPatternsOrd<Inst, "B", Op # "_i8", GPR32, + ATOMIC_FETCH_HINT_W>; +} + +// SelectionDAG lowers SUB to ADD with a negated value and AND to CLR with a +// complemented value. Hinted atomics use SelectionDAG, so separate SUB/AND +// patterns are not needed here (unlike the GlobalISel patterns below). +let Predicates = [HasLSE], AddedComplexity = 15 in { + defm : AtomicFetchHintPatterns<"LDADD", "atomic_load_add">; + defm : AtomicFetchHintPatterns<"LDSET", "atomic_load_or">; + defm : AtomicFetchHintPatterns<"LDEOR", "atomic_load_xor">; + defm : AtomicFetchHintPatterns<"LDCLR", "atomic_load_clr">; +} + //===---------------------------------- // Low-level exclusive operations //===---------------------------------- diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll index c989a5e40e091..b227f27633ce4 100644 --- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll +++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll @@ -1,6 +1,9 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 -; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NOLSE +; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NOLSE +; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+lse -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LSE +; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+lse -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LSE +; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+outline-atomics -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,OUTLINE ; ; STSHH: Keep, Relaxed @@ -359,7 +362,7 @@ define void @test_atomic_store_stcph_seqcst_i64(ptr %ptr, i64 %val) nounwind { define void @test_atomic_store_shuh_relaxed_i8(ptr %ptr, i8 %val) nounwind { ; CHECK-LABEL: test_atomic_store_shuh_relaxed_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: shuh{{$}} +; CHECK-NEXT: shuh ; CHECK-NEXT: strb w1, [x0] ; CHECK-NEXT: ret store atomic i8 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !6 @@ -369,7 +372,7 @@ define void @test_atomic_store_shuh_relaxed_i8(ptr %ptr, i8 %val) nounwind { define void @test_atomic_store_shuh_release_i64(ptr %ptr, i64 %val) nounwind { ; CHECK-LABEL: test_atomic_store_shuh_release_i64: ; CHECK: // %bb.0: -; CHECK-NEXT: shuh{{$}} +; CHECK-NEXT: shuh ; CHECK-NEXT: stlr x1, [x0] ; CHECK-NEXT: ret store atomic i64 %val, ptr %ptr release, align 8, !mem.cache_hint !6 @@ -386,6 +389,104 @@ define void @test_atomic_store_shuh_seqcst_i64(ptr %ptr, i64 %val) nounwind { ret void } +define i8 @test_atomic_fetch_add_shuh_relaxed_i8(ptr %ptr, i8 %val) nounwind { +; NOLSE-LABEL: test_atomic_fetch_add_shuh_relaxed_i8: +; NOLSE: // %bb.0: +; NOLSE-NEXT: .LBB33_1: // %atomicrmw.start +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldxrb w8, [x0] +; NOLSE-NEXT: add w9, w8, w1 +; NOLSE-NEXT: stxrb w10, w9, [x0] +; NOLSE-NEXT: cbnz w10, .LBB33_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end +; NOLSE-NEXT: mov w0, w8 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomic_fetch_add_shuh_relaxed_i8: +; LSE: // %bb.0: +; LSE-NEXT: shuh +; LSE-NEXT: ldaddb w1, w0, [x0] +; LSE-NEXT: ret +; +; OUTLINE-LABEL: test_atomic_fetch_add_shuh_relaxed_i8: +; OUTLINE: // %bb.0: +; OUTLINE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-NEXT: mov x8, x0 +; OUTLINE-NEXT: mov w0, w1 +; OUTLINE-NEXT: mov x1, x8 +; OUTLINE-NEXT: bl __aarch64_ldadd1_relax +; OUTLINE-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-NEXT: ret + %old = atomicrmw add ptr %ptr, i8 %val monotonic, align 1, !mem.cache_hint !12 + ret i8 %old +} + +define i8 @test_atomic_fetch_and_shuh_acq_rel_i8(ptr %ptr, i8 %val) nounwind { +; NOLSE-LABEL: test_atomic_fetch_and_shuh_acq_rel_i8: +; NOLSE: // %bb.0: +; NOLSE-NEXT: .LBB34_1: // %atomicrmw.start +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxrb w8, [x0] +; NOLSE-NEXT: and w9, w8, w1 +; NOLSE-NEXT: stlxrb w10, w9, [x0] +; NOLSE-NEXT: cbnz w10, .LBB34_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end +; NOLSE-NEXT: mov w0, w8 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomic_fetch_and_shuh_acq_rel_i8: +; LSE: // %bb.0: +; LSE-NEXT: mvn w8, w1 +; LSE-NEXT: shuh +; LSE-NEXT: ldclralb w8, w0, [x0] +; LSE-NEXT: ret +; +; OUTLINE-LABEL: test_atomic_fetch_and_shuh_acq_rel_i8: +; OUTLINE: // %bb.0: +; OUTLINE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-NEXT: mov x8, x0 +; OUTLINE-NEXT: mvn w0, w1 +; OUTLINE-NEXT: mov x1, x8 +; OUTLINE-NEXT: bl __aarch64_ldclr1_acq_rel +; OUTLINE-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-NEXT: ret + %old = atomicrmw and ptr %ptr, i8 %val acq_rel, align 1, !mem.cache_hint !12 + ret i8 %old +} + +; Sequentially consistent 64-bit fetch-xor: retain acquire semantics when the +; result is unused. +define void @test_atomic_fetch_xor_shuh_seqcst_dead_i64(ptr %ptr, i64 %val) nounwind { +; NOLSE-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i64: +; NOLSE: // %bb.0: +; NOLSE-NEXT: .LBB35_1: // %atomicrmw.start +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxr x8, [x0] +; NOLSE-NEXT: eor x8, x8, x1 +; NOLSE-NEXT: stlxr w9, x8, [x0] +; NOLSE-NEXT: cbnz w9, .LBB35_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i64: +; LSE: // %bb.0: +; LSE-NEXT: shuh +; LSE-NEXT: ldeoral x1, x8, [x0] +; LSE-NEXT: ret +; +; OUTLINE-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i64: +; OUTLINE: // %bb.0: +; OUTLINE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-NEXT: mov x8, x0 +; OUTLINE-NEXT: mov x0, x1 +; OUTLINE-NEXT: mov x1, x8 +; OUTLINE-NEXT: bl __aarch64_ldeor8_acq_rel +; OUTLINE-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-NEXT: ret + %old = atomicrmw xor ptr %ptr, i64 %val seq_cst, align 8, !mem.cache_hint !12 + ret void +} + ; ; SHUH PH ; @@ -420,6 +521,71 @@ define void @test_atomic_store_shuh_ph_seqcst_i64(ptr %ptr, i64 %val) nounwind { ret void } +define i32 @test_atomic_fetch_sub_shuh_ph_acquire_i32(ptr %ptr, i32 %val) nounwind { +; NOLSE-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i32: +; NOLSE: // %bb.0: +; NOLSE-NEXT: .LBB39_1: // %atomicrmw.start +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldaxr w8, [x0] +; NOLSE-NEXT: sub w9, w8, w1 +; NOLSE-NEXT: stxr w10, w9, [x0] +; NOLSE-NEXT: cbnz w10, .LBB39_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end +; NOLSE-NEXT: mov w0, w8 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i32: +; LSE: // %bb.0: +; LSE-NEXT: neg w8, w1 +; LSE-NEXT: shuh ph +; LSE-NEXT: ldadda w8, w0, [x0] +; LSE-NEXT: ret +; +; OUTLINE-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i32: +; OUTLINE: // %bb.0: +; OUTLINE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-NEXT: mov x8, x0 +; OUTLINE-NEXT: neg w0, w1 +; OUTLINE-NEXT: mov x1, x8 +; OUTLINE-NEXT: bl __aarch64_ldadd4_acq +; OUTLINE-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-NEXT: ret + %old = atomicrmw sub ptr %ptr, i32 %val acquire, align 4, !mem.cache_hint !13 + ret i32 %old +} + +define i16 @test_atomic_fetch_or_shuh_ph_release_i16(ptr %ptr, i16 %val) nounwind { +; NOLSE-LABEL: test_atomic_fetch_or_shuh_ph_release_i16: +; NOLSE: // %bb.0: +; NOLSE-NEXT: .LBB40_1: // %atomicrmw.start +; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1 +; NOLSE-NEXT: ldxrh w8, [x0] +; NOLSE-NEXT: orr w9, w8, w1 +; NOLSE-NEXT: stlxrh w10, w9, [x0] +; NOLSE-NEXT: cbnz w10, .LBB40_1 +; NOLSE-NEXT: // %bb.2: // %atomicrmw.end +; NOLSE-NEXT: mov w0, w8 +; NOLSE-NEXT: ret +; +; LSE-LABEL: test_atomic_fetch_or_shuh_ph_release_i16: +; LSE: // %bb.0: +; LSE-NEXT: shuh ph +; LSE-NEXT: ldsetlh w1, w0, [x0] +; LSE-NEXT: ret +; +; OUTLINE-LABEL: test_atomic_fetch_or_shuh_ph_release_i16: +; OUTLINE: // %bb.0: +; OUTLINE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-NEXT: mov x8, x0 +; OUTLINE-NEXT: mov w0, w1 +; OUTLINE-NEXT: mov x1, x8 +; OUTLINE-NEXT: bl __aarch64_ldset2_rel +; OUTLINE-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-NEXT: ret + %old = atomicrmw or ptr %ptr, i16 %val release, align 2, !mem.cache_hint !13 + ret i16 %old +} + ; ; Invalid Hint ; @@ -445,3 +611,5 @@ define void @test_atomic_store_invalid_hint(ptr %ptr, i8 %val) nounwind { !9 = !{!"aarch64.mem_hint", i32 4} !10 = !{i32 1, !11} !11 = !{!"aarch64.mem_hint", i32 5} +!12 = !{i32 0, !7} +!13 = !{i32 0, !9} >From 7abd7203682803a6b89a05ae7272b4b5add7ca4e Mon Sep 17 00:00:00 2001 From: Marian Lukac <[email protected]> Date: Wed, 30 Sep 2026 13:28:54 +0000 Subject: [PATCH 5/6] Formatting --- clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp index 3293c263527d4..8ec2929eb8a81 100644 --- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp +++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp @@ -2067,8 +2067,7 @@ static Value *EmitRangePrefetchBuiltin(CodeGenFunction &CGF, unsigned BuiltinID, } static void AttachAtomicHintMetadata(CodeGenFunction &CGF, const CallExpr *E, - Instruction *Atomic, - unsigned PtrOperand) { + Instruction *Atomic, unsigned PtrOperand) { CodeGen::CGBuilderTy &Builder = CGF.Builder; LLVMContext &Ctx = CGF.CGM.getLLVMContext(); Expr::EvalResult Result; @@ -2084,8 +2083,8 @@ static void AttachAtomicHintMetadata(CodeGenFunction &CGF, const CallExpr *E, Ctx, {MDString::get(Ctx, "aarch64.mem_hint"), llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg))}); MDNode *HintNode = MDNode::get( - Ctx, {llvm::ConstantAsMetadata::get(Builder.getInt32(PtrOperand)), - MemHint}); + Ctx, + {llvm::ConstantAsMetadata::get(Builder.getInt32(PtrOperand)), MemHint}); Atomic->setMetadata(llvm::LLVMContext::MD_mem_cache_hint, HintNode); } >From 3dfb40c2134002494f1555edc415b0b2a06d33a8 Mon Sep 17 00:00:00 2001 From: Marian Lukac <[email protected]> Date: Wed, 7 Oct 2026 10:34:02 +0000 Subject: [PATCH 6/6] Address comments --- clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 5 +++- clang/lib/Sema/SemaARM.cpp | 10 ++----- clang/test/CodeGen/builtins-arm64.c | 13 ++++++++ clang/test/Sema/builtins-arm64.c | 8 ++--- .../include/llvm/Support/AArch64MemoryHints.h | 5 ---- .../lib/Target/AArch64/AArch64InstrAtomics.td | 30 +++++++------------ 6 files changed, 34 insertions(+), 37 deletions(-) diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp index 8ec2929eb8a81..02a30f8ec5b25 100644 --- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp +++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp @@ -2076,7 +2076,10 @@ static void AttachAtomicHintMetadata(CodeGenFunction &CGF, const CallExpr *E, "Expected integer hint argument to atomic operation with hint."); unsigned HintArg = Result.Val.getInt().getExtValue(); - if (toAArch64MemoryHint(HintArg) == AArch64MemoryHint::NONE) + auto MemoryHint = toAArch64MemoryHint(HintArg); + if (MemoryHint == AArch64MemoryHint::NONE || + (isa<AtomicRMWInst>(Atomic) && MemoryHint != AArch64MemoryHint::SHUH && + MemoryHint != AArch64MemoryHint::SHUH_PH)) return; MDNode *MemHint = MDNode::get( diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp index 2f5a9f985a81d..a1e4df5300d2e 100644 --- a/clang/lib/Sema/SemaARM.cpp +++ b/clang/lib/Sema/SemaARM.cpp @@ -440,13 +440,9 @@ bool SemaARM::BuiltinARMAtomicHintCall(unsigned BuiltinID, CallExpr *TheCall) { << Hint->getType() << Hint->getSourceRange(); auto MemoryHint = llvm::toAArch64MemoryHint(HintAP->getZExtValue()); - if (!IsStore && MemoryHint != llvm::AArch64MemoryHint::SHUH && - MemoryHint != llvm::AArch64MemoryHint::SHUH_PH) - return Diag(TheCall->getBeginLoc(), - diag::err_atomic_hint_has_invalid_hint_type) - << *HintAP << Hint->getSourceRange(); - - if (MemoryHint == llvm::AArch64MemoryHint::NONE) { + if (MemoryHint == llvm::AArch64MemoryHint::NONE || + (!IsStore && MemoryHint != llvm::AArch64MemoryHint::SHUH && + MemoryHint != llvm::AArch64MemoryHint::SHUH_PH)) { Diag(TheCall->getBeginLoc(), diag::warn_atomic_hint_has_invalid_hint_type) << *HintAP << Hint->getSourceRange(); return false; diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c index 33960feb6cdb2..ed83126ce5420 100644 --- a/clang/test/CodeGen/builtins-arm64.c +++ b/clang/test/CodeGen/builtins-arm64.c @@ -295,6 +295,19 @@ int64_t atomic_fetch_add_with_hint_ph(int64_t *a, int64_t b) { return __builtin_arm_atomic_fetch_add_with_hint(a, b, __ATOMIC_RELAXED, 4); } +// CHECK-LABEL: @atomic_fetch_with_invalid_hint( +// CHECK: atomicrmw add ptr {{.*}}, i64 {{.*}} monotonic, align 8{{$}} +// CHECK-NEXT: atomicrmw sub ptr {{.*}}, i64 {{.*}} monotonic, align 8{{$}} +// CHECK-NEXT: atomicrmw and ptr {{.*}}, i64 {{.*}} monotonic, align 8{{$}} +// CHECK-NEXT: [[OLD:%.*]] = atomicrmw add ptr {{.*}}, i64 {{.*}} monotonic, align 8{{$}} +// CHECK-NEXT: ret i64 [[OLD]] +int64_t atomic_fetch_with_invalid_hint(int64_t *a, int64_t b) { + __builtin_arm_atomic_fetch_add_with_hint(a, b, __ATOMIC_RELAXED, HINT_STSHH_KEEP); + __builtin_arm_atomic_fetch_sub_with_hint(a, b, __ATOMIC_RELAXED, HINT_STSHH_STRM); + __builtin_arm_atomic_fetch_and_with_hint(a, b, __ATOMIC_RELAXED, HINT_STCPH); + return __builtin_arm_atomic_fetch_add_with_hint(a, b, __ATOMIC_RELAXED, 5); +} + // CHECK: ![[M0]] = !{!"1:2:3:4:5"} // CHECK: ![[M1]] = !{i32 1, ![[M2:[0-9]+]]} // CHECK: ![[M2]] = !{!"aarch64.mem_hint", i32 0} diff --git a/clang/test/Sema/builtins-arm64.c b/clang/test/Sema/builtins-arm64.c index ba0cf233df319..96774a02fca64 100644 --- a/clang/test/Sema/builtins-arm64.c +++ b/clang/test/Sema/builtins-arm64.c @@ -105,8 +105,8 @@ void test_atomic_fetch_hint(char *c_ptr, char c_data, const char *const_c_ptr, __builtin_arm_atomic_fetch_xor_with_hint(c_ptr, c_data, 0, variable); // expected-error {{invalid hint type argument to atomic hint operation ('int')}} __builtin_arm_atomic_fetch_or_with_hint(c_ptr, c_data, 0, "h"); // expected-error {{incompatible pointer to integer conversion passing 'char *' to parameter of type 'int'}} // expected-error@-1 {{invalid hint type argument to atomic hint operation ('int')}} - __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, 0, 0); // expected-error {{invalid hint type argument to atomic hint operation (0)}} - __builtin_arm_atomic_fetch_sub_with_hint(c_ptr, c_data, 0, 1); // expected-error {{invalid hint type argument to atomic hint operation (1)}} - __builtin_arm_atomic_fetch_and_with_hint(c_ptr, c_data, 0, 2); // expected-error {{invalid hint type argument to atomic hint operation (2)}} - __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, 0, 5); // expected-error {{invalid hint type argument to atomic hint operation (5)}} + __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, 0, 0); // expected-warning {{unrecognised hint type argument to atomic hint operation (0)}} + __builtin_arm_atomic_fetch_sub_with_hint(c_ptr, c_data, 0, 1); // expected-warning {{unrecognised hint type argument to atomic hint operation (1)}} + __builtin_arm_atomic_fetch_and_with_hint(c_ptr, c_data, 0, 2); // expected-warning {{unrecognised hint type argument to atomic hint operation (2)}} + __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, 0, 5); // expected-warning {{unrecognised hint type argument to atomic hint operation (5)}} } diff --git a/llvm/include/llvm/Support/AArch64MemoryHints.h b/llvm/include/llvm/Support/AArch64MemoryHints.h index f47eb0c945d1b..aefd1c1076462 100644 --- a/llvm/include/llvm/Support/AArch64MemoryHints.h +++ b/llvm/include/llvm/Support/AArch64MemoryHints.h @@ -19,11 +19,6 @@ enum class AArch64MemoryHint { SHUH_PH = 5, }; -template <typename Int> inline bool isValidAArch64MemoryHintValue(Int I) { - return (Int)AArch64MemoryHint::STSHH_KEEP <= I && - I <= (Int)AArch64MemoryHint::SHUH_PH; -} - template <typename Int> inline AArch64MemoryHint toAArch64MemoryHint(Int I) { switch (I) { case 0: diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td index 7568b6f530db8..dce5d7d84e2d2 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td +++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td @@ -464,8 +464,7 @@ class atomic_hint_fetch<PatFrag Base, code Pred> // Pass the LSE opcode as an immediate so all operations and orderings can use // the same two fetch pseudos. class AtomicFetchOpcode<string Inst> : SDNodeXForm<imm, - "return CurDAG->getTargetConstant(AArch64::" # Inst # - ", SDLoc(N), MVT::i32);">; + "return CurDAG->getTargetConstant(AArch64::" # Inst # ", SDLoc(N), MVT::i32);">; multiclass AtomicFetchHintPatternsForOpcode<string Inst, string Op, RegisterClass RC, Instruction Pseudo> { @@ -481,27 +480,18 @@ multiclass AtomicFetchHintPatternsForOpcode<string Inst, string Op, multiclass AtomicFetchHintPatternsOrd<string Inst, string Suffix, string Op, RegisterClass RC, Instruction Pseudo> { - defm : AtomicFetchHintPatternsForOpcode<Inst # Suffix, Op # "_monotonic", - RC, Pseudo>; - defm : AtomicFetchHintPatternsForOpcode<Inst # "A" # Suffix, Op # "_acquire", - RC, Pseudo>; - defm : AtomicFetchHintPatternsForOpcode<Inst # "L" # Suffix, Op # "_release", - RC, Pseudo>; - defm : AtomicFetchHintPatternsForOpcode<Inst # "AL" # Suffix, Op # "_acq_rel", - RC, Pseudo>; - defm : AtomicFetchHintPatternsForOpcode<Inst # "AL" # Suffix, Op # "_seq_cst", - RC, Pseudo>; + defm : AtomicFetchHintPatternsForOpcode<Inst # Suffix, Op # "_monotonic", RC, Pseudo>; + defm : AtomicFetchHintPatternsForOpcode<Inst # "A" # Suffix, Op # "_acquire", RC, Pseudo>; + defm : AtomicFetchHintPatternsForOpcode<Inst # "L" # Suffix, Op # "_release", RC, Pseudo>; + defm : AtomicFetchHintPatternsForOpcode<Inst # "AL" # Suffix, Op # "_acq_rel", RC, Pseudo>; + defm : AtomicFetchHintPatternsForOpcode<Inst # "AL" # Suffix, Op # "_seq_cst", RC, Pseudo>; } multiclass AtomicFetchHintPatterns<string Inst, string Op> { - defm : AtomicFetchHintPatternsOrd<Inst, "X", Op # "_i64", GPR64, - ATOMIC_FETCH_HINT_X>; - defm : AtomicFetchHintPatternsOrd<Inst, "W", Op # "_i32", GPR32, - ATOMIC_FETCH_HINT_W>; - defm : AtomicFetchHintPatternsOrd<Inst, "H", Op # "_i16", GPR32, - ATOMIC_FETCH_HINT_W>; - defm : AtomicFetchHintPatternsOrd<Inst, "B", Op # "_i8", GPR32, - ATOMIC_FETCH_HINT_W>; + defm : AtomicFetchHintPatternsOrd<Inst, "X", Op # "_i64", GPR64, ATOMIC_FETCH_HINT_X>; + defm : AtomicFetchHintPatternsOrd<Inst, "W", Op # "_i32", GPR32, ATOMIC_FETCH_HINT_W>; + defm : AtomicFetchHintPatternsOrd<Inst, "H", Op # "_i16", GPR32, ATOMIC_FETCH_HINT_W>; + defm : AtomicFetchHintPatternsOrd<Inst, "B", Op # "_i8", GPR32, ATOMIC_FETCH_HINT_W>; } // SelectionDAG lowers SUB to ADD with a negated value and AND to CLR with a _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
