Author: Lukacma Date: 2026-10-14T10:28:04Z New Revision: 336ad6e9bb9e0d140831fa1e4bc36e03ec88a808
URL: https://github.com/llvm/llvm-project/commit/336ad6e9bb9e0d140831fa1e4bc36e03ec88a808 DIFF: https://github.com/llvm/llvm-project/commit/336ad6e9bb9e0d140831fa1e4bc36e03ec88a808.diff LOG: [AArch64] Implement the atomic fetch with hint intrinsic (#227711) This patch extends the work started in #198316 and introduces fetch hint intrinsic described in [ACLE](https://arm-software.github.io/acle/main/acle.html#atomic-fetch-with-hints-intrinsics) : ``` type __arm_atomic_fetch_add_with_hint(type *ptr, type data, int memory_order, int hint); type __arm_atomic_fetch_sub_with_hint(type *ptr, type data, int memory_order, int hint); type __arm_atomic_fetch_and_with_hint(type *ptr, type data, int memory_order, int hint); type __arm_atomic_fetch_xor_with_hint(type *ptr, type data, int memory_order, int hint); type __arm_atomic_fetch_or_with_hint(type *ptr, type data, int memory_order, int hint); ``` These intrinsics support scalar integral types of 8, 16, 32, and 64 bits and only work with HINT_SHUH, HINT_SHUH_PH . Added: clang/test/CodeGen/AArch64/atomic-hint-template.cpp llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-fetch-hint.ll Modified: clang/include/clang/Basic/BuiltinsAArch64.td clang/include/clang/Basic/DiagnosticSemaKinds.td clang/include/clang/Sema/SemaARM.h clang/lib/CodeGen/TargetBuiltins/ARM.cpp clang/lib/Headers/arm_acle.h clang/lib/Sema/SemaARM.cpp clang/test/CodeGen/arm_acle.c clang/test/CodeGen/builtins-arm64.c clang/test/Sema/builtins-arm64.c llvm/include/llvm/Support/AArch64MemoryHints.h llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp llvm/lib/Target/AArch64/AArch64ISelLowering.cpp llvm/lib/Target/AArch64/AArch64InstrAtomics.td llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll Removed: clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp ################################################################################ diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td index 729045d8eb5176..84ad46119682a6 100644 --- a/clang/include/clang/Basic/BuiltinsAArch64.td +++ b/clang/include/clang/Basic/BuiltinsAArch64.td @@ -179,6 +179,11 @@ let Attributes = [NoThrow], Features = "ls64" in { let Attributes = [NoThrow, CustomTypeChecking] in { def atomic_store_with_hint : AArch64Builtin<"void(...)">; + def atomic_fetch_add_with_hint : AArch64Builtin<"void(...)">; + def atomic_fetch_sub_with_hint : AArch64Builtin<"void(...)">; + def atomic_fetch_and_with_hint : AArch64Builtin<"void(...)">; + def atomic_fetch_xor_with_hint : AArch64Builtin<"void(...)">; + def atomic_fetch_or_with_hint : AArch64Builtin<"void(...)">; } // Armv9.3-A Guarded Control Stack diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index 1054ab293a96b3..162106c84d120d 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -9636,7 +9636,7 @@ def err_atomic_builtin_must_be_pointer_intptr : Error< def err_atomic_hint_builtin_must_be_pointer : Error< "address argument to atomic hint builtin must be a pointer to a scalar " - "integral or floating-point type of 8, 16, 32, or 64 bits (%0 invalid)">; + "%select{integral|integral or floating-point}1 type of 8, 16, 32, or 64 bits (%0 invalid)">; def err_atomic_hint_has_invalid_hint_type : Error< "invalid hint type argument to atomic hint operation (%0)">; @@ -9709,7 +9709,7 @@ def err_atomic_op_needs_atomic_fp : Error<"address argument to atomic operation must be a pointer to " "%select{|atomic }0floating point type (%1 invalid)">; def err_atomic_op_hint_data_size : Error< - "address argument to atomic store with hint must be of size 8, 16, 32 or 64 bits">; + "address argument to atomic %select{fetch|store}0 with hint must be of size 8, 16, 32 or 64 bits">; def err_atomic_hint_has_invalid_memory_order : Error< "invalid memory order argument to atomic hint operation (%0 invalid)">; def warn_atomic_hint_has_invalid_hint_type : Warning< diff --git a/clang/include/clang/Sema/SemaARM.h b/clang/include/clang/Sema/SemaARM.h index b0a01c40ffecef..76a06f4f4221a5 100644 --- a/clang/include/clang/Sema/SemaARM.h +++ b/clang/include/clang/Sema/SemaARM.h @@ -70,7 +70,7 @@ class SemaARM : public SemaBase { bool BuiltinARMSpecialReg(unsigned BuiltinID, CallExpr *TheCall, int ArgNum, unsigned ExpectedFieldNum, bool AllowName); bool BuiltinARMMemoryTaggingCall(unsigned BuiltinID, CallExpr *TheCall); - bool BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, CallExpr *TheCall); + bool BuiltinARMAtomicHintCall(unsigned BuiltinID, CallExpr *TheCall); bool MveAliasValid(unsigned BuiltinID, llvm::StringRef AliasName); bool CdeAliasValid(unsigned BuiltinID, llvm::StringRef AliasName); diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp index fcd765aa64d0e8..4a7bdba6d917ee 100644 --- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp +++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp @@ -2067,6 +2067,31 @@ static Value *EmitRangePrefetchBuiltin(CodeGenFunction &CGF, unsigned BuiltinID, Ops); } +static void AttachAtomicHintMetadata(CodeGenFunction &CGF, const CallExpr *E, + Instruction *Atomic, unsigned PtrOperand) { + CodeGen::CGBuilderTy &Builder = CGF.Builder; + LLVMContext &Ctx = CGF.CGM.getLLVMContext(); + Expr::EvalResult Result; + if (!E->getArg(3)->EvaluateAsInt(Result, CGF.getContext())) + llvm_unreachable( + "Expected integer hint argument to atomic operation with hint."); + unsigned HintArg = Result.Val.getInt().getExtValue(); + + auto MemoryHint = toAArch64MemoryHint(HintArg); + if (MemoryHint == AArch64MemoryHint::NONE || + (isa<AtomicRMWInst>(Atomic) && MemoryHint != AArch64MemoryHint::SHUH && + MemoryHint != AArch64MemoryHint::SHUH_PH)) + return; + + MDNode *MemHint = MDNode::get( + Ctx, {MDString::get(Ctx, "aarch64.mem_hint"), + llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg))}); + MDNode *HintNode = MDNode::get( + Ctx, + {llvm::ConstantAsMetadata::get(Builder.getInt32(PtrOperand)), MemHint}); + Atomic->setMetadata(llvm::LLVMContext::MD_mem_cache_hint, HintNode); +} + static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E) { @@ -2106,26 +2131,53 @@ static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF, break; } Store->setAtomic(Ordering); + AttachAtomicHintMetadata(CGF, E, Store, 1); + return Store; +} - if (!E->getArg(3)->EvaluateAsInt(Result, CGM.getContext())) +static Value *EmitAtomicFetchWithHintBuiltin(CodeGenFunction &CGF, + const CallExpr *E, + AtomicRMWInst::BinOp Op) { + CodeGen::CGBuilderTy &Builder = CGF.Builder; + Expr::EvalResult Result; + if (!E->getArg(2)->EvaluateAsInt(Result, CGF.getContext())) llvm_unreachable( - "Expected integer hint argument to atomic store with hint."); - unsigned HintArg = Result.Val.getInt().getExtValue(); + "Expected integer policy argument to atomic fetch with hint."); - // Attach the hint if valid - if (toAArch64MemoryHint(HintArg) != AArch64MemoryHint::NONE) { - LLVMContext &Ctx = CGM.getLLVMContext(); - MDNode *MemHint = MDNode::get( - Ctx, {MDString::get(Ctx, "aarch64.mem_hint"), - llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg))}); - MDNode *HintNode = MDNode::get( - CGM.getLLVMContext(), - {llvm::ConstantAsMetadata::get(Builder.getInt32(1)), MemHint}); + AtomicOrdering Ordering; + unsigned OrderingArg = Result.Val.getInt().getExtValue(); + assert(isValidAtomicOrderingCABI(OrderingArg) && "Invalid atomic ordering"); - Store->setMetadata(llvm::LLVMContext::MD_mem_cache_hint, HintNode); + switch (static_cast<AtomicOrderingCABI>(OrderingArg)) { + case AtomicOrderingCABI::relaxed: + Ordering = AtomicOrdering::Monotonic; + break; + case AtomicOrderingCABI::consume: + case AtomicOrderingCABI::acquire: + Ordering = AtomicOrdering::Acquire; + break; + case AtomicOrderingCABI::release: + Ordering = AtomicOrdering::Release; + break; + case AtomicOrderingCABI::acq_rel: + Ordering = AtomicOrdering::AcquireRelease; + break; + case AtomicOrderingCABI::seq_cst: + Ordering = AtomicOrdering::SequentiallyConsistent; + break; } - return Store; + const Expr *Ptr = E->getArg(0); + Address Addr = CGF.EmitPointerWithAlignment(Ptr); + Addr = Addr.withElementType( + CGF.ConvertTypeForMem(Ptr->getType()->getPointeeType())); + const Expr *Data = E->getArg(1); + Value *DataVal = CGF.EmitToMemory(CGF.EmitScalarExpr(Data), Data->getType()); + + AtomicRMWInst *RMW = Builder.CreateAtomicRMW(Op, Addr, DataVal, Ordering); + RMW->setVolatile(Ptr->getType()->getPointeeType().isVolatileQualified()); + AttachAtomicHintMetadata(CGF, E, RMW, 0); + return CGF.EmitFromMemory(RMW, E->getType()); } Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, @@ -4871,8 +4923,22 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, BuiltinID == AArch64::BI__builtin_arm_range_prefetch_x) return EmitRangePrefetchBuiltin(*this, BuiltinID, E); - if (BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint) + switch (BuiltinID) { + default: + break; + case AArch64::BI__builtin_arm_atomic_store_with_hint: return EmitAtomicStoreWithHintBuiltin(*this, BuiltinID, E); + case AArch64::BI__builtin_arm_atomic_fetch_add_with_hint: + return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::Add); + case AArch64::BI__builtin_arm_atomic_fetch_sub_with_hint: + return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::Sub); + case AArch64::BI__builtin_arm_atomic_fetch_and_with_hint: + return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::And); + case AArch64::BI__builtin_arm_atomic_fetch_xor_with_hint: + return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::Xor); + case AArch64::BI__builtin_arm_atomic_fetch_or_with_hint: + return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::Or); + } // Memory Tagging Extensions (MTE) Intrinsics Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic; diff --git a/clang/lib/Headers/arm_acle.h b/clang/lib/Headers/arm_acle.h index 395b760e7dbf09..cfbe490d995bfa 100644 --- a/clang/lib/Headers/arm_acle.h +++ b/clang/lib/Headers/arm_acle.h @@ -742,7 +742,7 @@ __arm_st64bv0(void *__addr, data512_t __value) { } #endif -/* Atomic store with hints */ +/* Atomic intrinsics with hints */ #if defined(__ARM_64BIT_STATE) && __ARM_64BIT_STATE #define HINT_STSHH_KEEP 0 #define HINT_STSHH_STRM 1 @@ -751,6 +751,16 @@ __arm_st64bv0(void *__addr, data512_t __value) { #define HINT_SHUH_PH 4 #define __arm_atomic_store_with_hint(ptr, data, memory_order, hint) \ __builtin_arm_atomic_store_with_hint(ptr, data, memory_order, hint) +#define __arm_atomic_fetch_add_with_hint(ptr, data, memory_order, hint) \ + __builtin_arm_atomic_fetch_add_with_hint(ptr, data, memory_order, hint) +#define __arm_atomic_fetch_sub_with_hint(ptr, data, memory_order, hint) \ + __builtin_arm_atomic_fetch_sub_with_hint(ptr, data, memory_order, hint) +#define __arm_atomic_fetch_and_with_hint(ptr, data, memory_order, hint) \ + __builtin_arm_atomic_fetch_and_with_hint(ptr, data, memory_order, hint) +#define __arm_atomic_fetch_xor_with_hint(ptr, data, memory_order, hint) \ + __builtin_arm_atomic_fetch_xor_with_hint(ptr, data, memory_order, hint) +#define __arm_atomic_fetch_or_with_hint(ptr, data, memory_order, hint) \ + __builtin_arm_atomic_fetch_or_with_hint(ptr, data, memory_order, hint) #endif /* 11.1 Special register intrinsics */ diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp index 2e8b90def62663..9938e187f7aad5 100644 --- a/clang/lib/Sema/SemaARM.cpp +++ b/clang/lib/Sema/SemaARM.cpp @@ -323,14 +323,16 @@ bool SemaARM::BuiltinARMSpecialReg(unsigned BuiltinID, CallExpr *TheCall, return false; } -bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, - CallExpr *TheCall) { +bool SemaARM::BuiltinARMAtomicHintCall(unsigned BuiltinID, CallExpr *TheCall) { if (SemaRef.checkArgCount(TheCall, 4)) return true; - // Arg 0 should be the pointer type. The pointee type must be a - // scalar integral or floating-point type of 8, 16, 32 or 64 bits. + bool IsStore = BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint; ASTContext &Context = getASTContext(); + + // Arg 0 should be the pointer type. The pointee type must be a + // scalar integral type of 8, 16, 32 or 64 bits. Stores also support + // floating-point types. auto PtrArgRes = SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(0)); if (PtrArgRes.isInvalid()) @@ -340,7 +342,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, if (!PtrTy) return Diag(TheCall->getBeginLoc(), diag::err_atomic_hint_builtin_must_be_pointer) - << PtrArg->getType() << 0 << PtrArg->getSourceRange(); + << PtrArg->getType() << IsStore << PtrArg->getSourceRange(); TheCall->setArg(0, PtrArg); QualType PtrQT = Context.getCanonicalType(PtrTy->getPointeeType()); @@ -350,10 +352,11 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, << PtrQT << PtrArg->getSourceRange(); PtrQT = PtrQT.getUnqualifiedType(); - if (!PtrQT->isIntegralType(Context) && !PtrQT->isFloatingType() && - !PtrQT->isMFloat8Type()) + if (!PtrQT->isIntegralType(Context) && + !(IsStore && (PtrQT->isFloatingType() || PtrQT->isMFloat8Type()))) return Diag(TheCall->getBeginLoc(), - diag::err_atomic_op_needs_atomic_int_or_fp) + IsStore ? diag::err_atomic_op_needs_atomic_int_or_fp + : diag::err_atomic_op_needs_atomic_int) << 0 << PtrQT << PtrArg->getSourceRange(); if (PtrQT->isBitIntType()) @@ -364,9 +367,12 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, unsigned TySize = Context.getTypeSize(PtrQT); if (TySize != 8 && TySize != 16 && TySize != 32 && TySize != 64) return Diag(TheCall->getBeginLoc(), diag::err_atomic_op_hint_data_size) - << PtrArg->getSourceRange(); + << IsStore << PtrArg->getSourceRange(); + + if (!IsStore) + TheCall->setType(PtrQT); - // Arg 1 is the data to be stored. The type must match the pointee + // Arg 1 is the data operand. The type must match the pointee // type found above. auto DataArgRes = SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(1)); @@ -382,7 +388,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, diag::err_typecheck_call_ diff erent_arg_types) << PtrQT << DataQT; - // Arg 2 is the memory order, which must be relaxed, release or seq_cst + // Arg 2 is the memory order. Stores only allow relaxed, release or seq_cst. auto MemOrdArg = SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(2)); if (MemOrdArg.isInvalid()) @@ -407,7 +413,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, << *MemOrdAP << MemOrd->getSourceRange(); auto AtomicOrdering = static_cast<llvm::AtomicOrderingCABI>(Ordering); - if (AtomicOrdering != llvm::AtomicOrderingCABI::relaxed && + if (IsStore && AtomicOrdering != llvm::AtomicOrderingCABI::relaxed && AtomicOrdering != llvm::AtomicOrderingCABI::release && AtomicOrdering != llvm::AtomicOrderingCABI::seq_cst) return Diag(TheCall->getBeginLoc(), @@ -416,7 +422,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, } // Arg 3 is the hint type. Only values represented by AArch64MemoryHint - // are valid. + // are valid. Fetch operations only allow HINT_SHUH and HINT_SHUH_PH. auto HintArg = SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(3)); if (HintArg.isInvalid()) @@ -433,8 +439,10 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, diag::err_atomic_hint_has_invalid_hint_type) << Hint->getType() << Hint->getSourceRange(); - if (llvm::toAArch64MemoryHint(HintAP->getZExtValue()) == - llvm::AArch64MemoryHint::NONE) { + auto MemoryHint = llvm::toAArch64MemoryHint(HintAP->getZExtValue()); + if (MemoryHint == llvm::AArch64MemoryHint::NONE || + (!IsStore && MemoryHint != llvm::AArch64MemoryHint::SHUH && + MemoryHint != llvm::AArch64MemoryHint::SHUH_PH)) { Diag(TheCall->getBeginLoc(), diag::warn_atomic_hint_has_invalid_hint_type) << *HintAP << Hint->getSourceRange(); return false; @@ -1314,8 +1322,13 @@ bool SemaARM::CheckAArch64BuiltinFunctionCall(const TargetInfo &TI, BuiltinID == AArch64::BI__builtin_arm_wsrp) return BuiltinARMSpecialReg(BuiltinID, TheCall, 0, 5, true); - if (BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint) - return BuiltinARMAtomicStoreHintCall(BuiltinID, TheCall); + if (BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint || + BuiltinID == AArch64::BI__builtin_arm_atomic_fetch_add_with_hint || + BuiltinID == AArch64::BI__builtin_arm_atomic_fetch_sub_with_hint || + BuiltinID == AArch64::BI__builtin_arm_atomic_fetch_and_with_hint || + BuiltinID == AArch64::BI__builtin_arm_atomic_fetch_xor_with_hint || + BuiltinID == AArch64::BI__builtin_arm_atomic_fetch_or_with_hint) + return BuiltinARMAtomicHintCall(BuiltinID, TheCall); // Only check the valid encoding range. Any constant in this range would be // converted to a register of the form S2_2_C3_C4_5. Let the hardware throw diff --git a/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp b/clang/test/CodeGen/AArch64/atomic-hint-template.cpp similarity index 66% rename from clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp rename to clang/test/CodeGen/AArch64/atomic-hint-template.cpp index ecf1e10a09b5da..0f64aacd6d9694 100644 --- a/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp +++ b/clang/test/CodeGen/AArch64/atomic-hint-template.cpp @@ -55,9 +55,37 @@ void test_atomic_store_hint_array_consexpr_hint(int *ptr, int value) { __builtin_arm_atomic_store_with_hint(ptr, value, __ATOMIC_RELAXED, HintVal{}); } +template <unsigned Order> +int test_atomic_fetch_hint_template_order(int *ptr, int val) { +// CHECK-LABEL: define weak_odr noundef i32 @_Z37test_atomic_fetch_hint_template_orderILj2EEiPii( +// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VAL:%.*]]) #[[ATTR0]] comdat { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = atomicrmw add ptr [[PTR]], i32 [[VAL]] acquire, align 4, !mem.cache_hint [[META8:![0-9]+]] +// CHECK-NEXT: ret i32 [[TMP0]] + return __arm_atomic_fetch_add_with_hint(ptr, val, Order, 3); +} + +template int test_atomic_fetch_hint_template_order<__ATOMIC_ACQUIRE>(int *, int); + +template <unsigned Hint> +int test_atomic_fetch_hint_template_hint(int *ptr, int val) { +// CHECK-LABEL: define weak_odr noundef i32 @_Z36test_atomic_fetch_hint_template_hintILj4EEiPii( +// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VAL:%.*]]) #[[ATTR0]] comdat { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = atomicrmw xor ptr [[PTR]], i32 [[VAL]] seq_cst, align 4, !mem.cache_hint [[META10:![0-9]+]] +// CHECK-NEXT: ret i32 [[TMP0]] + return __arm_atomic_fetch_xor_with_hint(ptr, val, __ATOMIC_SEQ_CST, Hint); +} + +template int test_atomic_fetch_hint_template_hint<4>(int *, int); + //. // CHECK: [[META4]] = !{i32 1, [[META5:![0-9]+]]} // CHECK: [[META5]] = !{!"aarch64.mem_hint", i32 0} // CHECK: [[META6]] = !{i32 1, [[META7:![0-9]+]]} // CHECK: [[META7]] = !{!"aarch64.mem_hint", i32 1} +// CHECK: [[META8]] = !{i32 0, [[META9:![0-9]+]]} +// CHECK: [[META9]] = !{!"aarch64.mem_hint", i32 3} +// CHECK: [[META10]] = !{i32 0, [[META11:![0-9]+]]} +// CHECK: [[META11]] = !{!"aarch64.mem_hint", i32 4} //. diff --git a/clang/test/CodeGen/arm_acle.c b/clang/test/CodeGen/arm_acle.c index 60ec3e48e94980..2acb0373632c52 100644 --- a/clang/test/CodeGen/arm_acle.c +++ b/clang/test/CodeGen/arm_acle.c @@ -1989,9 +1989,120 @@ void test_atomic_store_hint_array_arg() { __builtin_arm_atomic_store_with_hint(storage, 0, __ATOMIC_RELAXED, HINT_STSHH_STRM); } +// AArch64-LABEL: @test_atomic_fetch_hint_char( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[PTR:%.*]], i8 [[DATA:%.*]] monotonic, align 1, !mem.cache_hint [[FETCH0:![0-9]+]] +// AArch64-NEXT: ret i8 [[OLD]] +// +char test_atomic_fetch_hint_char(char *ptr, char data) { + return __arm_atomic_fetch_add_with_hint(ptr, data, __ATOMIC_RELAXED, 3); +} + +// AArch64-LABEL: @test_atomic_fetch_hint_uchar( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[OLD:%.*]] = atomicrmw sub ptr [[PTR:%.*]], i8 [[DATA:%.*]] release, align 1, !mem.cache_hint [[FETCH1:![0-9]+]] +// AArch64-NEXT: ret i8 [[OLD]] +// +unsigned char test_atomic_fetch_hint_uchar(unsigned char *ptr, unsigned char data) { + return __arm_atomic_fetch_sub_with_hint(ptr, data, __ATOMIC_RELEASE, 4); +} + +// AArch64-LABEL: @test_atomic_fetch_hint_short( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[OLD:%.*]] = atomicrmw and ptr [[PTR:%.*]], i16 [[DATA:%.*]] acquire, align 2, !mem.cache_hint [[FETCH0]] +// AArch64-NEXT: ret i16 [[OLD]] +// +short test_atomic_fetch_hint_short(short *ptr, short data) { + return __arm_atomic_fetch_and_with_hint(ptr, data, __ATOMIC_ACQUIRE, 3); +} + +// AArch64-LABEL: @test_atomic_fetch_hint_ushort( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[OLD:%.*]] = atomicrmw xor ptr [[PTR:%.*]], i16 [[DATA:%.*]] acq_rel, align 2, !mem.cache_hint [[FETCH1]] +// AArch64-NEXT: ret i16 [[OLD]] +// +unsigned short test_atomic_fetch_hint_ushort(unsigned short *ptr, unsigned short data) { + return __arm_atomic_fetch_xor_with_hint(ptr, data, __ATOMIC_ACQ_REL, 4); +} + +// AArch64-LABEL: @test_atomic_fetch_hint_int( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[OLD:%.*]] = atomicrmw or ptr [[PTR:%.*]], i32 [[DATA:%.*]] seq_cst, align 4, !mem.cache_hint [[FETCH0]] +// AArch64-NEXT: ret i32 [[OLD]] +// +int test_atomic_fetch_hint_int(int *ptr, int data) { + return __arm_atomic_fetch_or_with_hint(ptr, data, __ATOMIC_SEQ_CST, 3); +} + +// AArch64-LABEL: @test_atomic_fetch_hint_unsigned( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[PTR:%.*]], i32 [[DATA:%.*]] acquire, align 4, !mem.cache_hint [[FETCH1]] +// AArch64-NEXT: ret i32 [[OLD]] +// +unsigned test_atomic_fetch_hint_unsigned(unsigned *ptr, unsigned data) { + return __arm_atomic_fetch_add_with_hint(ptr, data, __ATOMIC_CONSUME, 4); +} + +// AArch64-LABEL: @test_atomic_fetch_hint_long( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[OLD:%.*]] = atomicrmw sub ptr [[PTR:%.*]], i64 [[DATA:%.*]] monotonic, align 8, !mem.cache_hint [[FETCH0]] +// AArch64-NEXT: ret i64 [[OLD]] +// +long test_atomic_fetch_hint_long(long *ptr, long data) { + return __arm_atomic_fetch_sub_with_hint(ptr, data, __ATOMIC_RELAXED, 3); +} + +// AArch64-LABEL: @test_atomic_fetch_hint_bool( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[STOREDV:%.*]] = zext i1 [[DATA:%.*]] to i8 +// AArch64-NEXT: [[LOADEDV:%.*]] = icmp ne i8 [[STOREDV]], 0 +// AArch64-NEXT: [[STOREDV1:%.*]] = zext i1 [[LOADEDV]] to i8 +// AArch64-NEXT: [[OLD:%.*]] = atomicrmw xor ptr [[PTR:%.*]], i8 [[STOREDV1]] monotonic, align 1, !mem.cache_hint [[FETCH0]] +// AArch64-NEXT: [[LOADEDV1:%.*]] = icmp ne i8 [[OLD]], 0 +// AArch64-NEXT: ret i1 [[LOADEDV1]] +// +bool test_atomic_fetch_hint_bool(bool *ptr, bool data) { + return __arm_atomic_fetch_xor_with_hint(ptr, data, __ATOMIC_RELAXED, 3); +} + +// AArch64-LABEL: @test_atomic_fetch_hint_volatile( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[OLD:%.*]] = atomicrmw volatile or ptr [[PTR:%.*]], i32 [[DATA:%.*]] acq_rel, align 4, !mem.cache_hint [[FETCH0]] +// AArch64-NEXT: ret i32 [[OLD]] +// +int test_atomic_fetch_hint_volatile(volatile int *ptr, int data) { + return __arm_atomic_fetch_or_with_hint(ptr, data, __ATOMIC_ACQ_REL, 3); +} + +// AArch64-LABEL: @test_atomic_fetch_hint_typedef( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[PTR:%.*]], i32 [[DATA:%.*]] release, align 4, !mem.cache_hint [[FETCH1]] +// AArch64-NEXT: ret i32 [[OLD]] +// +aliased_int test_atomic_fetch_hint_typedef(aliased_int *ptr, const int data) { + return __arm_atomic_fetch_add_with_hint(ptr, data, __ATOMIC_RELEASE, 4); +} + +// AArch64-LABEL: @test_atomic_fetch_hint_array_arg( +// AArch64-NEXT: entry: +// AArch64-NEXT: [[STORAGE:%.*]] = alloca [1 x i32], align 4 +// AArch64-NEXT: call void @llvm.memset.p0.i64(ptr align 4 [[STORAGE]], i8 0, i64 4, i1 false) +// AArch64-NEXT: [[ARRAYDECAY:%.*]] = getelementptr inbounds [1 x i32], ptr [[STORAGE]], i64 0, i64 0 +// AArch64-NEXT: [[OLD:%.*]] = atomicrmw sub ptr [[ARRAYDECAY]], i32 1 monotonic, align 4, !mem.cache_hint [[FETCH1]] +// AArch64-NEXT: ret i32 [[OLD]] +// +int test_atomic_fetch_hint_array_arg(void) { + int storage[1] = {0}; + return __arm_atomic_fetch_sub_with_hint(storage, 1, __ATOMIC_RELAXED, 4); +} + // AArch64: [[HINT1]] = !{i32 1, [[HINT2:![0-9]+]]} // AArch64-NEXT: [[HINT2]] = !{!"aarch64.mem_hint", i32 0} // AArch64-NEXT: [[HINT3]] = !{i32 1, [[HINT4:![0-9]+]]} // AArch64-NEXT: [[HINT4]] = !{!"aarch64.mem_hint", i32 1} +// AArch64: [[FETCH0]] = !{i32 0, [[FETCH_HINT0:![0-9]+]]} +// AArch64-NEXT: [[FETCH_HINT0]] = !{!"aarch64.mem_hint", i32 3} +// AArch64-NEXT: [[FETCH1]] = !{i32 0, [[FETCH_HINT1:![0-9]+]]} +// AArch64-NEXT: [[FETCH_HINT1]] = !{!"aarch64.mem_hint", i32 4} #endif diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c index b1002362585803..ed83126ce54203 100644 --- a/clang/test/CodeGen/builtins-arm64.c +++ b/clang/test/CodeGen/builtins-arm64.c @@ -242,6 +242,70 @@ void atomic_store_with_hint(int64_t *a, int64_t b) { // Invalid hint should be dropped __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 5); // Invalid Hint // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8 + // CHECK-NOT: !mem.cache_hint + // CHECK-NEXT: ret void +} + +// CHECK-LABEL: @atomic_fetch_add_with_hint( +// CHECK: [[OLD:%.*]] = atomicrmw add ptr {{.*}}, i64 {{.*}} monotonic, align 8, !mem.cache_hint ![[F1:[0-9]+]] +// CHECK-NEXT: ret i64 [[OLD]] +int64_t atomic_fetch_add_with_hint(int64_t *a, int64_t b) { + return __builtin_arm_atomic_fetch_add_with_hint(a, b, __ATOMIC_RELAXED, 3); +} + +// CHECK-LABEL: @atomic_fetch_sub_with_hint( +// CHECK: [[OLD:%.*]] = atomicrmw sub ptr {{.*}}, i64 {{.*}} release, align 8, !mem.cache_hint ![[F3:[0-9]+]] +// CHECK-NEXT: ret i64 [[OLD]] +int64_t atomic_fetch_sub_with_hint(int64_t *a, int64_t b) { + return __builtin_arm_atomic_fetch_sub_with_hint(a, b, __ATOMIC_RELEASE, 4); +} + +// CHECK-LABEL: @atomic_fetch_and_with_hint( +// CHECK: [[OLD:%.*]] = atomicrmw and ptr {{.*}}, i64 {{.*}} acquire, align 8, !mem.cache_hint ![[F1]] +// CHECK-NEXT: ret i64 [[OLD]] +int64_t atomic_fetch_and_with_hint(int64_t *a, int64_t b) { + return __builtin_arm_atomic_fetch_and_with_hint(a, b, __ATOMIC_ACQUIRE, 3); +} + +// CHECK-LABEL: @atomic_fetch_xor_with_hint( +// CHECK: [[OLD:%.*]] = atomicrmw xor ptr {{.*}}, i64 {{.*}} acq_rel, align 8, !mem.cache_hint ![[F1]] +// CHECK-NEXT: ret i64 [[OLD]] +int64_t atomic_fetch_xor_with_hint(int64_t *a, int64_t b) { + return __builtin_arm_atomic_fetch_xor_with_hint(a, b, __ATOMIC_ACQ_REL, 3); +} + +// CHECK-LABEL: @atomic_fetch_or_with_hint( +// CHECK: [[OLD:%.*]] = atomicrmw or ptr {{.*}}, i64 {{.*}} seq_cst, align 8, !mem.cache_hint ![[F3]] +// CHECK-NEXT: ret i64 [[OLD]] +int64_t atomic_fetch_or_with_hint(int64_t *a, int64_t b) { + return __builtin_arm_atomic_fetch_or_with_hint(a, b, __ATOMIC_SEQ_CST, 4); +} + +// CHECK-LABEL: @atomic_fetch_with_hint_consume( +// CHECK: [[OLD:%.*]] = atomicrmw add ptr {{.*}}, i64 {{.*}} acquire, align 8, !mem.cache_hint ![[F1]] +// CHECK-NEXT: ret i64 [[OLD]] +int64_t atomic_fetch_with_hint_consume(int64_t *a, int64_t b) { + return __builtin_arm_atomic_fetch_add_with_hint(a, b, __ATOMIC_CONSUME, 3); +} + +// CHECK-LABEL: @atomic_fetch_add_with_hint_ph( +// CHECK: [[OLD:%.*]] = atomicrmw add ptr {{.*}}, i64 {{.*}} monotonic, align 8, !mem.cache_hint ![[F3]] +// CHECK-NEXT: ret i64 [[OLD]] +int64_t atomic_fetch_add_with_hint_ph(int64_t *a, int64_t b) { + return __builtin_arm_atomic_fetch_add_with_hint(a, b, __ATOMIC_RELAXED, 4); +} + +// CHECK-LABEL: @atomic_fetch_with_invalid_hint( +// CHECK: atomicrmw add ptr {{.*}}, i64 {{.*}} monotonic, align 8{{$}} +// CHECK-NEXT: atomicrmw sub ptr {{.*}}, i64 {{.*}} monotonic, align 8{{$}} +// CHECK-NEXT: atomicrmw and ptr {{.*}}, i64 {{.*}} monotonic, align 8{{$}} +// CHECK-NEXT: [[OLD:%.*]] = atomicrmw add ptr {{.*}}, i64 {{.*}} monotonic, align 8{{$}} +// CHECK-NEXT: ret i64 [[OLD]] +int64_t atomic_fetch_with_invalid_hint(int64_t *a, int64_t b) { + __builtin_arm_atomic_fetch_add_with_hint(a, b, __ATOMIC_RELAXED, HINT_STSHH_KEEP); + __builtin_arm_atomic_fetch_sub_with_hint(a, b, __ATOMIC_RELAXED, HINT_STSHH_STRM); + __builtin_arm_atomic_fetch_and_with_hint(a, b, __ATOMIC_RELAXED, HINT_STCPH); + return __builtin_arm_atomic_fetch_add_with_hint(a, b, __ATOMIC_RELAXED, 5); } // CHECK: ![[M0]] = !{!"1:2:3:4:5"} @@ -255,3 +319,5 @@ void atomic_store_with_hint(int64_t *a, int64_t b) { // CHECK: ![[M8]] = !{!"aarch64.mem_hint", i32 3} // CHECK: ![[M9]] = !{i32 1, ![[M10:[0-9]+]]} // CHECK: ![[M10]] = !{!"aarch64.mem_hint", i32 4} +// CHECK: ![[F1]] = !{i32 0, ![[M8]]} +// CHECK: ![[F3]] = !{i32 0, ![[M10]]} diff --git a/clang/test/Sema/builtins-arm64.c b/clang/test/Sema/builtins-arm64.c index a79fb9177cec88..96774a02fca646 100644 --- a/clang/test/Sema/builtins-arm64.c +++ b/clang/test/Sema/builtins-arm64.c @@ -80,3 +80,33 @@ void test_atomic_store_hint(char *c_ptr, __int128 *inv_ptr, float *f_ptr, __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, "h"); // expected-error {{incompatible pointer to integer conversion passing 'char *' to parameter of type 'int'}} // expected-error@-1 {{invalid hint type argument to atomic hint operation ('int')}} } + +void test_atomic_fetch_hint(char *c_ptr, char c_data, const char *const_c_ptr, + float *f_ptr, float f_data, __int128 *wide_ptr, + __int128 wide_data, unsigned _BitInt(8) *bit_ptr, + unsigned _BitInt(8) bit_data, int variable, + int **ptr_ptr, _Atomic int *atomic_ptr) { + __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, 0); // expected-error {{too few arguments to function call, expected 4, have 3}} + __builtin_arm_atomic_fetch_sub_with_hint(c_ptr, c_data, 0, 3, 0); // expected-error {{too many arguments to function call, expected 4, have 5}} + + __builtin_arm_atomic_fetch_and_with_hint(0, c_data, 0, 3); // expected-error {{address argument to atomic hint builtin must be a pointer to a scalar integral type of 8, 16, 32, or 64 bits ('int' invalid)}} + __builtin_arm_atomic_fetch_xor_with_hint(c_ptr, f_data, 0, 3); // expected-error {{arguments are of diff erent types ('char' vs 'float')}} + __builtin_arm_atomic_fetch_or_with_hint(const_c_ptr, c_data, 0, 3); // expected-error {{address argument to atomic operation must be a pointer to non-const type ('const char' invalid)}} + __builtin_arm_atomic_fetch_add_with_hint(f_ptr, f_data, 0, 3); // expected-error {{address argument to atomic operation must be a pointer to integer ('float' invalid)}} + __builtin_arm_atomic_fetch_sub_with_hint(wide_ptr, wide_data, 0, 3); // expected-error {{address argument to atomic fetch with hint must be of size 8, 16, 32 or 64 bits}} + __builtin_arm_atomic_fetch_and_with_hint(bit_ptr, bit_data, 0, 3); // expected-error {{argument to atomic builtin of type '_BitInt' is not supported}} + __builtin_arm_atomic_fetch_xor_with_hint(ptr_ptr, 0, 0, 3); // expected-error {{address argument to atomic operation must be a pointer to integer ('int *' invalid)}} + __builtin_arm_atomic_fetch_or_with_hint(atomic_ptr, 0, 0, 3); // expected-error {{address argument to atomic operation must be a pointer to integer ('_Atomic(int)' invalid)}} + + __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, variable, 3); // expected-error {{invalid memory order argument to atomic hint operation ('int' invalid)}} + __builtin_arm_atomic_fetch_sub_with_hint(c_ptr, c_data, -1, 3); // expected-error {{invalid memory order argument to atomic hint operation (-1 invalid)}} + __builtin_arm_atomic_fetch_and_with_hint(c_ptr, c_data, 6, 3); // expected-error {{invalid memory order argument to atomic hint operation (6 invalid)}} + + __builtin_arm_atomic_fetch_xor_with_hint(c_ptr, c_data, 0, variable); // expected-error {{invalid hint type argument to atomic hint operation ('int')}} + __builtin_arm_atomic_fetch_or_with_hint(c_ptr, c_data, 0, "h"); // expected-error {{incompatible pointer to integer conversion passing 'char *' to parameter of type 'int'}} + // expected-error@-1 {{invalid hint type argument to atomic hint operation ('int')}} + __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, 0, 0); // expected-warning {{unrecognised hint type argument to atomic hint operation (0)}} + __builtin_arm_atomic_fetch_sub_with_hint(c_ptr, c_data, 0, 1); // expected-warning {{unrecognised hint type argument to atomic hint operation (1)}} + __builtin_arm_atomic_fetch_and_with_hint(c_ptr, c_data, 0, 2); // expected-warning {{unrecognised hint type argument to atomic hint operation (2)}} + __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, 0, 5); // expected-warning {{unrecognised hint type argument to atomic hint operation (5)}} +} diff --git a/llvm/include/llvm/Support/AArch64MemoryHints.h b/llvm/include/llvm/Support/AArch64MemoryHints.h index f47eb0c945d1b1..aefd1c10764623 100644 --- a/llvm/include/llvm/Support/AArch64MemoryHints.h +++ b/llvm/include/llvm/Support/AArch64MemoryHints.h @@ -19,11 +19,6 @@ enum class AArch64MemoryHint { SHUH_PH = 5, }; -template <typename Int> inline bool isValidAArch64MemoryHintValue(Int I) { - return (Int)AArch64MemoryHint::STSHH_KEEP <= I && - I <= (Int)AArch64MemoryHint::SHUH_PH; -} - template <typename Int> inline AArch64MemoryHint toAArch64MemoryHint(Int I) { switch (I) { case 0: diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index c3755d713f8aff..3320df7d951471 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -287,6 +287,9 @@ class AArch64AsmPrinter : public AsmPrinter { void emitAtomicHintPseudoExpansionRO(const MachineInstr *MI); void emitAtomicHintPseudoExpansionImm(const MachineInstr *MI); + // Emit expansion of atomic fetch with hint pseudo instructions + void emitAtomicFetchHintPseudoExpansion(const MachineInstr *MI); + void EmitToStreamer(MCStreamer &S, const MCInst &Inst); void EmitToStreamer(const MCInst &Inst) { EmitToStreamer(*OutStreamer, Inst); @@ -3402,6 +3405,20 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansionImm( EmitToStreamer(*OutStreamer, Store); } +void AArch64AsmPrinter::emitAtomicFetchHintPseudoExpansion( + const MachineInstr *MI) { + EmitToStreamer( + MCInstBuilder(AArch64::HINT).addImm(MI->getOperand(4).getImm())); + + MCInst Fetch; + Fetch.setOpcode(MI->getOperand(3).getImm()); + Fetch.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); + Fetch.addOperand(MCOperand::createReg(MI->getOperand(1).getReg())); + Fetch.addOperand(MCOperand::createReg(MI->getOperand(2).getReg())); + Fetch.setFlags(MI->getFlags()); + EmitToStreamer(*OutStreamer, Fetch); +} + // Simple pseudo-instructions have their lowering (with expansion to real // instructions) auto-generated. #include "AArch64GenMCPseudoLowering.inc" @@ -4125,6 +4142,10 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { case AArch64::ATOMIC_STORE_HINT_X: emitAtomicHintPseudoExpansion(MI); return; + case AArch64::ATOMIC_FETCH_HINT_W: + case AArch64::ATOMIC_FETCH_HINT_X: + emitAtomicFetchHintPseudoExpansion(MI); + return; case AArch64::ATOMIC_STORE_HINT_BroW: case AArch64::ATOMIC_STORE_HINT_HroW: case AArch64::ATOMIC_STORE_HINT_WroW: diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 5befbf84d3bb8e..7157122b1d9358 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -34062,11 +34062,14 @@ bool AArch64TargetLowering::fallBackToDAGISel(const Instruction &Inst) const { } // The !mem.cache_hint metadata is not supported by GISel and will be dropped. - // TODO: Remove this and handle atomic store hints in GISel once supported. + // TODO: Remove this and handle atomic hints in GISel once supported. if (auto *Store = dyn_cast<StoreInst>(&Inst)) { if (Store->isAtomic() && getMemCacheHintMetadata(*Store, 1)) return true; } + if (auto *RMW = dyn_cast<AtomicRMWInst>(&Inst)) + if (getMemCacheHintMetadata(*RMW)) + return true; return false; } diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td index 1fa094edfbb82c..6564971876d9ec 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td +++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td @@ -283,7 +283,7 @@ def : Pat<(relaxed_store<atomic_store_64> (STURDi FPR64Op:$val, GPR64sp:$Rn, simm9:$offset)>; //===---------------------------------- -// Atomic store with hint pseudos +// Atomic operations with hint pseudos //===---------------------------------- let Size = 8, isCodeGenOnly = 1, hasSideEffects = 1, mayStore = 1 in { @@ -438,6 +438,66 @@ let AddedComplexity = 1, GISelShouldIgnore = 1 in { defm : AtomicHintPatterns<0, 51, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH_PH); }]>; } +// Keep the hint and the LSE instruction together until assembly emission. +// The opcode operand lets all operations and orderings share these pseudos. +let Size = 8, isCodeGenOnly = 1, hasSideEffects = 1, mayLoad = 1, + mayStore = 1 in { + class BaseFetchHintPseudo<RegisterClass dsttype, RegisterClass srctype> + : Pseudo<(outs dsttype:$result), + (ins srctype:$data, GPR64sp:$addr, i32imm:$opcode, + i32imm:$hint), []>, Sched<[WriteAtomic]>; + + def ATOMIC_FETCH_HINT_W : BaseFetchHintPseudo<GPR32common, GPR32>; + def ATOMIC_FETCH_HINT_X : BaseFetchHintPseudo<GPR64common, GPR64>; +} + +class atomic_hint_fetch<PatFrag Base, code Pred> + : PatFrag<(ops node:$ptr, node:$val), + (Base node:$ptr, node:$val), Pred>; + +// Pass the LSE opcode as an immediate so all operations and orderings can use +// the same two fetch pseudos. +class AtomicFetchOpcode<string Inst> : SDNodeXForm<imm, + "return CurDAG->getTargetConstant(AArch64::" # Inst # ", SDLoc(N), MVT::i32);">; + +multiclass AtomicFetchHintPatternsForOpcode<string Inst, string Op, + RegisterClass RC, Instruction Pseudo> { + defvar Base = !cast<PatFrag>(Op); + defvar Opcode = AtomicFetchOpcode<Inst>; + def : Pat<(atomic_hint_fetch<Base, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH); }]> + GPR64sp:$addr, RC:$data), + (Pseudo RC:$data, GPR64sp:$addr, (Opcode (i32 0)), (i32 50))>; + def : Pat<(atomic_hint_fetch<Base, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH_PH); }]> + GPR64sp:$addr, RC:$data), + (Pseudo RC:$data, GPR64sp:$addr, (Opcode (i32 0)), (i32 51))>; +} + +multiclass AtomicFetchHintPatternsOrd<string Inst, string Suffix, string Op, + RegisterClass RC, Instruction Pseudo> { + defm : AtomicFetchHintPatternsForOpcode<Inst # Suffix, Op # "_monotonic", RC, Pseudo>; + defm : AtomicFetchHintPatternsForOpcode<Inst # "A" # Suffix, Op # "_acquire", RC, Pseudo>; + defm : AtomicFetchHintPatternsForOpcode<Inst # "L" # Suffix, Op # "_release", RC, Pseudo>; + defm : AtomicFetchHintPatternsForOpcode<Inst # "AL" # Suffix, Op # "_acq_rel", RC, Pseudo>; + defm : AtomicFetchHintPatternsForOpcode<Inst # "AL" # Suffix, Op # "_seq_cst", RC, Pseudo>; +} + +multiclass AtomicFetchHintPatterns<string Inst, string Op> { + defm : AtomicFetchHintPatternsOrd<Inst, "X", Op # "_i64", GPR64, ATOMIC_FETCH_HINT_X>; + defm : AtomicFetchHintPatternsOrd<Inst, "W", Op # "_i32", GPR32, ATOMIC_FETCH_HINT_W>; + defm : AtomicFetchHintPatternsOrd<Inst, "H", Op # "_i16", GPR32, ATOMIC_FETCH_HINT_W>; + defm : AtomicFetchHintPatternsOrd<Inst, "B", Op # "_i8", GPR32, ATOMIC_FETCH_HINT_W>; +} + +// SelectionDAG lowers SUB to ADD with a negated value and AND to CLR with a +// complemented value. Hinted atomics use SelectionDAG, so separate SUB/AND +// patterns are not needed here (unlike the GlobalISel patterns below). +let Predicates = [HasLSE], AddedComplexity = 1 in { + defm : AtomicFetchHintPatterns<"LDADD", "atomic_load_add">; + defm : AtomicFetchHintPatterns<"LDSET", "atomic_load_or">; + defm : AtomicFetchHintPatterns<"LDEOR", "atomic_load_xor">; + defm : AtomicFetchHintPatterns<"LDCLR", "atomic_load_clr">; +} + //===---------------------------------- // Low-level exclusive operations //===---------------------------------- diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-fetch-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-fetch-hint.ll new file mode 100644 index 00000000000000..aae55ab0c25cbf --- /dev/null +++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-fetch-hint.ll @@ -0,0 +1,241 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 +; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+lse -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+lse -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s + +; +; SHUH, Relaxed +; + +define i8 @test_atomic_fetch_add_shuh_relaxed_i8(ptr %ptr, i8 %val) nounwind { +; CHECK-LABEL: test_atomic_fetch_add_shuh_relaxed_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh +; CHECK-NEXT: ldaddb w1, w0, [x0] +; CHECK-NEXT: ret + %old = atomicrmw add ptr %ptr, i8 %val monotonic, align 1, !mem.cache_hint !0 + ret i8 %old +} + +define i16 @test_atomic_fetch_add_shuh_relaxed_i16(ptr %ptr, i16 %val) nounwind { +; CHECK-LABEL: test_atomic_fetch_add_shuh_relaxed_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh +; CHECK-NEXT: ldaddh w1, w0, [x0] +; CHECK-NEXT: ret + %old = atomicrmw add ptr %ptr, i16 %val monotonic, align 2, !mem.cache_hint !0 + ret i16 %old +} + +define i32 @test_atomic_fetch_add_shuh_relaxed_i32(ptr %ptr, i32 %val) nounwind { +; CHECK-LABEL: test_atomic_fetch_add_shuh_relaxed_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh +; CHECK-NEXT: ldadd w1, w0, [x0] +; CHECK-NEXT: ret + %old = atomicrmw add ptr %ptr, i32 %val monotonic, align 4, !mem.cache_hint !0 + ret i32 %old +} + +define i64 @test_atomic_fetch_add_shuh_relaxed_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: test_atomic_fetch_add_shuh_relaxed_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh +; CHECK-NEXT: ldadd x1, x0, [x0] +; CHECK-NEXT: ret +; CHECK- + %old = atomicrmw add ptr %ptr, i64 %val monotonic, align 8, !mem.cache_hint !0 + ret i64 %old +} + +; +; SHUH, AcquireRelease +; + +define i8 @test_atomic_fetch_and_shuh_acq_rel_i8(ptr %ptr, i8 %val) nounwind { +; CHECK-LABEL: test_atomic_fetch_and_shuh_acq_rel_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: mvn w8, w1 +; CHECK-NEXT: shuh +; CHECK-NEXT: ldclralb w8, w0, [x0] +; CHECK-NEXT: ret + %old = atomicrmw and ptr %ptr, i8 %val acq_rel, align 1, !mem.cache_hint !0 + ret i8 %old +} + +define i16 @test_atomic_fetch_and_shuh_acq_rel_i16(ptr %ptr, i16 %val) nounwind { +; CHECK-LABEL: test_atomic_fetch_and_shuh_acq_rel_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: mvn w8, w1 +; CHECK-NEXT: shuh +; CHECK-NEXT: ldclralh w8, w0, [x0] +; CHECK-NEXT: ret + %old = atomicrmw and ptr %ptr, i16 %val acq_rel, align 2, !mem.cache_hint !0 + ret i16 %old +} + +define i32 @test_atomic_fetch_and_shuh_acq_rel_i32(ptr %ptr, i32 %val) nounwind { +; CHECK-LABEL: test_atomic_fetch_and_shuh_acq_rel_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: mvn w8, w1 +; CHECK-NEXT: shuh +; CHECK-NEXT: ldclral w8, w0, [x0] +; CHECK-NEXT: ret + %old = atomicrmw and ptr %ptr, i32 %val acq_rel, align 4, !mem.cache_hint !0 + ret i32 %old +} + +define i64 @test_atomic_fetch_and_shuh_acq_rel_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: test_atomic_fetch_and_shuh_acq_rel_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: mvn x8, x1 +; CHECK-NEXT: shuh +; CHECK-NEXT: ldclral x8, x0, [x0] +; CHECK-NEXT: ret + + %old = atomicrmw and ptr %ptr, i64 %val acq_rel, align 8, !mem.cache_hint !0 + ret i64 %old +} + +; +; SHUH, SequentiallyConsistent +; + +; Sequentially consistent fetch-xor: retain acquire semantics when the +; result is unused. +define void @test_atomic_fetch_xor_shuh_seqcst_dead_i8(ptr %ptr, i8 %val) nounwind { +; CHECK-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh +; CHECK-NEXT: ldeoralb w1, w8, [x0] +; CHECK-NEXT: ret + %old = atomicrmw xor ptr %ptr, i8 %val seq_cst, align 1, !mem.cache_hint !0 + ret void +} + +define void @test_atomic_fetch_xor_shuh_seqcst_dead_i16(ptr %ptr, i16 %val) nounwind { +; CHECK-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh +; CHECK-NEXT: ldeoralh w1, w8, [x0] +; CHECK-NEXT: ret + %old = atomicrmw xor ptr %ptr, i16 %val seq_cst, align 2, !mem.cache_hint !0 + ret void +} + +define void @test_atomic_fetch_xor_shuh_seqcst_dead_i32(ptr %ptr, i32 %val) nounwind { +; CHECK-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh +; CHECK-NEXT: ldeoral w1, w8, [x0] +; CHECK-NEXT: ret + %old = atomicrmw xor ptr %ptr, i32 %val seq_cst, align 4, !mem.cache_hint !0 + ret void +} + +define void @test_atomic_fetch_xor_shuh_seqcst_dead_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh +; CHECK-NEXT: ldeoral x1, x8, [x0] +; CHECK-NEXT: ret + %old = atomicrmw xor ptr %ptr, i64 %val seq_cst, align 8, !mem.cache_hint !0 + ret void +} + +; +; SHUH PH, Acquire +; + +define i8 @test_atomic_fetch_sub_shuh_ph_acquire_i8(ptr %ptr, i8 %val) nounwind { +; CHECK-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: neg w8, w1 +; CHECK-NEXT: shuh ph +; CHECK-NEXT: ldaddab w8, w0, [x0] +; CHECK-NEXT: ret + %old = atomicrmw sub ptr %ptr, i8 %val acquire, align 1, !mem.cache_hint !2 + ret i8 %old +} + +define i16 @test_atomic_fetch_sub_shuh_ph_acquire_i16(ptr %ptr, i16 %val) nounwind { +; CHECK-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: neg w8, w1 +; CHECK-NEXT: shuh ph +; CHECK-NEXT: ldaddah w8, w0, [x0] +; CHECK-NEXT: ret + %old = atomicrmw sub ptr %ptr, i16 %val acquire, align 2, !mem.cache_hint !2 + ret i16 %old +} + +define i32 @test_atomic_fetch_sub_shuh_ph_acquire_i32(ptr %ptr, i32 %val) nounwind { +; CHECK-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: neg w8, w1 +; CHECK-NEXT: shuh ph +; CHECK-NEXT: ldadda w8, w0, [x0] +; CHECK-NEXT: ret + %old = atomicrmw sub ptr %ptr, i32 %val acquire, align 4, !mem.cache_hint !2 + ret i32 %old +} + +define i64 @test_atomic_fetch_sub_shuh_ph_acquire_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: neg x8, x1 +; CHECK-NEXT: shuh ph +; CHECK-NEXT: ldadda x8, x0, [x0] +; CHECK-NEXT: ret + + %old = atomicrmw sub ptr %ptr, i64 %val acquire, align 8, !mem.cache_hint !2 + ret i64 %old +} + +; +; SHUH PH, Release +; + +define i8 @test_atomic_fetch_or_shuh_ph_release_i8(ptr %ptr, i8 %val) nounwind { +; CHECK-LABEL: test_atomic_fetch_or_shuh_ph_release_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh ph +; CHECK-NEXT: ldsetlb w1, w0, [x0] +; CHECK-NEXT: ret + %old = atomicrmw or ptr %ptr, i8 %val release, align 1, !mem.cache_hint !2 + ret i8 %old +} + +define i16 @test_atomic_fetch_or_shuh_ph_release_i16(ptr %ptr, i16 %val) nounwind { +; CHECK-LABEL: test_atomic_fetch_or_shuh_ph_release_i16: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh ph +; CHECK-NEXT: ldsetlh w1, w0, [x0] +; CHECK-NEXT: ret + %old = atomicrmw or ptr %ptr, i16 %val release, align 2, !mem.cache_hint !2 + ret i16 %old +} + +define i32 @test_atomic_fetch_or_shuh_ph_release_i32(ptr %ptr, i32 %val) nounwind { +; CHECK-LABEL: test_atomic_fetch_or_shuh_ph_release_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh ph +; CHECK-NEXT: ldsetl w1, w0, [x0] +; CHECK-NEXT: ret + %old = atomicrmw or ptr %ptr, i32 %val release, align 4, !mem.cache_hint !2 + ret i32 %old +} + +define i64 @test_atomic_fetch_or_shuh_ph_release_i64(ptr %ptr, i64 %val) nounwind { +; CHECK-LABEL: test_atomic_fetch_or_shuh_ph_release_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: shuh ph +; CHECK-NEXT: ldsetl x1, x0, [x0] +; CHECK-NEXT: ret + %old = atomicrmw or ptr %ptr, i64 %val release, align 8, !mem.cache_hint !2 + ret i64 %old +} + +!0 = !{i32 0, !1} +!1 = !{!"aarch64.mem_hint", i32 3} +!2 = !{i32 0, !3} +!3 = !{!"aarch64.mem_hint", i32 4} diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll index 2528a4079b0320..06dc124c79088a 100644 --- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll +++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll @@ -457,7 +457,7 @@ define void @test_atomic_store_stcph_seqcst_i64(ptr %ptr, i64 %val) nounwind { define void @test_atomic_store_shuh_relaxed_i8(ptr %ptr, i8 %val) nounwind { ; CHECK-LABEL: test_atomic_store_shuh_relaxed_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: shuh{{$}} +; CHECK-NEXT: shuh ; CHECK-NEXT: strb w1, [x0] ; CHECK-NEXT: ret store atomic i8 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !6 @@ -531,7 +531,7 @@ define void @test_atomic_store_shuh_release_i32(ptr %ptr, i32 %val) nounwind { define void @test_atomic_store_shuh_release_i64(ptr %ptr, i64 %val) nounwind { ; CHECK-LABEL: test_atomic_store_shuh_release_i64: ; CHECK: // %bb.0: -; CHECK-NEXT: shuh{{$}} +; CHECK-NEXT: shuh ; CHECK-NEXT: stlr x1, [x0] ; CHECK-NEXT: ret store atomic i64 %val, ptr %ptr release, align 8, !mem.cache_hint !6 _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
