https://github.com/tommat01 created https://github.com/llvm/llvm-project/pull/208443
None >From 625256892ffea0953a810c8136cc76457365c3f3 Mon Sep 17 00:00:00 2001 From: Tomas Matheson <[email protected]> Date: Thu, 9 Jul 2026 13:10:03 +0100 Subject: [PATCH] [AArch64] ldiapp for 128-bit SC load with rcpc3 --- llvm/lib/CodeGen/AtomicExpandPass.cpp | 23 +++++++------- .../Target/AArch64/AArch64ISelLowering.cpp | 30 +++++++++++++++++-- llvm/lib/Target/AArch64/AArch64ISelLowering.h | 4 +++ .../Atomics/aarch64-atomic-load-rcpc3.ll | 24 +++++---------- .../Atomics/aarch64_be-atomic-load-rcpc3.ll | 24 +++++---------- 5 files changed, 57 insertions(+), 48 deletions(-) diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp index 6985a7a48147c..2d86592aac8d6 100644 --- a/llvm/lib/CodeGen/AtomicExpandPass.cpp +++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp @@ -94,8 +94,8 @@ class AtomicExpandImpl { bool bracketInstWithFences(Instruction *I, AtomicOrdering Order); bool tryInsertTrailingSeqCstFence(Instruction *AtomicI); template <typename AtomicInst> - bool tryInsertFencesForAtomic(AtomicInst *AtomicI, bool OrderingRequiresFence, - AtomicOrdering NewOrdering); + bool tryInsertFencesForAtomic(AtomicInst *AtomicI, + bool OrderingRequiresFence); IntegerType *getCorrespondingIntegerType(Type *T, const DataLayout &DL); LoadInst *convertAtomicLoadToIntegerType(LoadInst *LI); bool tryExpandAtomicLoad(LoadInst *LI); @@ -321,12 +321,11 @@ bool AtomicExpandImpl::tryInsertTrailingSeqCstFence(Instruction *AtomicI) { template <typename AtomicInst> bool AtomicExpandImpl::tryInsertFencesForAtomic(AtomicInst *AtomicI, - bool OrderingRequiresFence, - AtomicOrdering NewOrdering) { + bool OrderingRequiresFence) { bool ShouldInsertFences = TLI->shouldInsertFencesForAtomic(AtomicI); if (OrderingRequiresFence && ShouldInsertFences) { AtomicOrdering FenceOrdering = AtomicI->getOrdering(); - AtomicI->setOrdering(NewOrdering); + AtomicI->setOrdering(TLI->atomicOperationOrderAfterFenceSplit(AtomicI)); return bracketInstWithFences(AtomicI, FenceOrdering); } if (!ShouldInsertFences) @@ -351,8 +350,8 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) { MadeChange = true; } - MadeChange |= tryInsertFencesForAtomic( - LI, isAcquireOrStronger(LI->getOrdering()), AtomicOrdering::Monotonic); + MadeChange |= + tryInsertFencesForAtomic(LI, isAcquireOrStronger(LI->getOrdering())); MadeChange |= tryExpandAtomicLoad(LI); return MadeChange; @@ -374,8 +373,8 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) { MadeChange = true; } - MadeChange |= tryInsertFencesForAtomic( - SI, isReleaseOrStronger(SI->getOrdering()), AtomicOrdering::Monotonic); + MadeChange |= + tryInsertFencesForAtomic(SI, isReleaseOrStronger(SI->getOrdering())); MadeChange |= tryExpandAtomicStore(SI); return MadeChange; @@ -395,10 +394,8 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) { } MadeChange |= tryInsertFencesForAtomic( - RMWI, - isReleaseOrStronger(RMWI->getOrdering()) || - isAcquireOrStronger(RMWI->getOrdering()), - TLI->atomicOperationOrderAfterFenceSplit(RMWI)); + RMWI, isReleaseOrStronger(RMWI->getOrdering()) || + isAcquireOrStronger(RMWI->getOrdering())); // There are two different ways of expanding RMW instructions: // - into a load if it is idempotent diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 642afb8922af3..b7ab4d99bf87b 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -32024,7 +32024,7 @@ bool AArch64TargetLowering::isOpSuitableForRCPC3(const Instruction *I) const { if (auto LI = dyn_cast<LoadInst>(I)) return LI->getType()->getPrimitiveSizeInBits() == 128 && LI->getAlign() >= Align(16) && - LI->getOrdering() == AtomicOrdering::Acquire; + isAcquireOrStronger(LI->getOrdering()); if (auto SI = dyn_cast<StoreInst>(I)) return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() == 128 && @@ -32036,6 +32036,10 @@ bool AArch64TargetLowering::isOpSuitableForRCPC3(const Instruction *I) const { bool AArch64TargetLowering::shouldInsertFencesForAtomic( const Instruction *I) const { + if (auto *LI = dyn_cast<LoadInst>(I); + LI && LI->getOrdering() == AtomicOrdering::SequentiallyConsistent && + isOpSuitableForRCPC3(LI)) + return true; // need leading LDAR if (isOpSuitableForRCPC3(I)) return false; if (isOpSuitableForLSE128(I)) @@ -32074,11 +32078,21 @@ bool AArch64TargetLowering::shouldInsertTrailingSeqCstFenceForAtomicStore( return !Subtarget->hasLSE(); } +AtomicOrdering AArch64TargetLowering::atomicOperationOrderAfterFenceSplit( + const Instruction *I) const { + if (auto *LI = dyn_cast<LoadInst>(I); + LI && LI->getOrdering() == AtomicOrdering::SequentiallyConsistent && + isOpSuitableForRCPC3(LI)) + return AtomicOrdering::Acquire; + + return TargetLoweringBase::atomicOperationOrderAfterFenceSplit(I); +} + Instruction *AArch64TargetLowering::emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const { - // Keep seq_cst 128-bit LSE2 loads ordered against v8.0-style seq_cst LL/SC - // stores by emitting an (unused) LDAR before the LDP. + // Keep seq_cst 128-bit LSE2/RCPC3 loads ordered against v8.0-style seq_cst + // LL/SC stores by emitting an (unused) LDAR before the 128-bit load. if (auto *LI = dyn_cast<LoadInst>(Inst); LI && Ord == AtomicOrdering::SequentiallyConsistent && isOpSuitableForLDPSTP(LI)) { @@ -32092,6 +32106,16 @@ Instruction *AArch64TargetLowering::emitLeadingFence(IRBuilderBase &Builder, return TargetLoweringBase::emitLeadingFence(Builder, Inst, Ord); } +Instruction *AArch64TargetLowering::emitTrailingFence( + IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const { + if (auto *LI = dyn_cast<LoadInst>(Inst); + LI && Ord == AtomicOrdering::SequentiallyConsistent && + isOpSuitableForRCPC3(LI)) + return nullptr; + + return TargetLoweringBase::emitTrailingFence(Builder, Inst, Ord); +} + // Loads and stores less than 128-bits are already atomic; ones above that // are doomed anyway, so defer to the default libcall and blame the OS when // things go wrong. diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h index 69013d4010122..cc8cec8a83738 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h @@ -348,6 +348,10 @@ class AArch64TargetLowering : public TargetLowering { Instruction *emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override; + Instruction *emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, + AtomicOrdering Ord) const override; + AtomicOrdering + atomicOperationOrderAfterFenceSplit(const Instruction *I) const override; Value *emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr, AtomicOrdering Ord) const override; Value *emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr, diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc3.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc3.ll index 04f7572cf4332..4618e53a54f7a 100644 --- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc3.ll +++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc3.ll @@ -272,13 +272,11 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr) define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) { ; -O0-LABEL: load_atomic_i128_aligned_seq_cst: ; -O0: ldar x8, [x0] -; -O0: ldp x0, x1, [x0] -; -O0: dmb ish +; -O0: ldiapp x0, x1, [x0] ; ; -O1-LABEL: load_atomic_i128_aligned_seq_cst: ; -O1: ldar xzr, [x0] -; -O1: ldp x0, x1, [x0] -; -O1: dmb ish +; -O1: ldiapp x0, x1, [x0] %r = load atomic i128, ptr %ptr seq_cst, align 16 ret i128 %r } @@ -286,13 +284,11 @@ define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) { define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) { ; -O0-LABEL: load_atomic_i128_aligned_seq_cst_const: ; -O0: ldar x8, [x0] -; -O0: ldp x0, x1, [x0] -; -O0: dmb ish +; -O0: ldiapp x0, x1, [x0] ; ; -O1-LABEL: load_atomic_i128_aligned_seq_cst_const: ; -O1: ldar xzr, [x0] -; -O1: ldp x0, x1, [x0] -; -O1: dmb ish +; -O1: ldiapp x0, x1, [x0] %r = load atomic i128, ptr %ptr seq_cst, align 16 ret i128 %r } @@ -580,13 +576,11 @@ define dso_local i128 @load_atomic_i128_unaligned_seq_cst_const(ptr readonly %pt define dso_local fp128 @load_atomic_fp128_aligned_seq_cst(ptr %ptr) { ; -O0-LABEL: load_atomic_fp128_aligned_seq_cst: ; -O0: ldar x8, [x0] -; -O0: ldp x9, x8, [x0] -; -O0: dmb ish +; -O0: ldiapp x9, x8, [x0] ; ; -O1-LABEL: load_atomic_fp128_aligned_seq_cst: ; -O1: ldar xzr, [x0] -; -O1: ldp x8, x9, [x0] -; -O1: dmb ish +; -O1: ldiapp x8, x9, [x0] %r = load atomic fp128, ptr %ptr seq_cst, align 16 ret fp128 %r } @@ -594,13 +588,11 @@ define dso_local fp128 @load_atomic_fp128_aligned_seq_cst(ptr %ptr) { define dso_local fp128 @load_atomic_fp128_aligned_seq_cst_const(ptr readonly %ptr) { ; -O0-LABEL: load_atomic_fp128_aligned_seq_cst_const: ; -O0: ldar x8, [x0] -; -O0: ldp x9, x8, [x0] -; -O0: dmb ish +; -O0: ldiapp x9, x8, [x0] ; ; -O1-LABEL: load_atomic_fp128_aligned_seq_cst_const: ; -O1: ldar xzr, [x0] -; -O1: ldp x8, x9, [x0] -; -O1: dmb ish +; -O1: ldiapp x8, x9, [x0] %r = load atomic fp128, ptr %ptr seq_cst, align 16 ret fp128 %r } diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll index b25af543ff8fd..222a1a92741f5 100644 --- a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll +++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll @@ -272,13 +272,11 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr) define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) { ; -O0-LABEL: load_atomic_i128_aligned_seq_cst: ; -O0: ldar x8, [x0] -; -O0: ldp x0, x1, [x0] -; -O0: dmb ish +; -O0: ldiapp x0, x1, [x0] ; ; -O1-LABEL: load_atomic_i128_aligned_seq_cst: ; -O1: ldar xzr, [x0] -; -O1: ldp x0, x1, [x0] -; -O1: dmb ish +; -O1: ldiapp x0, x1, [x0] %r = load atomic i128, ptr %ptr seq_cst, align 16 ret i128 %r } @@ -286,13 +284,11 @@ define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) { define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) { ; -O0-LABEL: load_atomic_i128_aligned_seq_cst_const: ; -O0: ldar x8, [x0] -; -O0: ldp x0, x1, [x0] -; -O0: dmb ish +; -O0: ldiapp x0, x1, [x0] ; ; -O1-LABEL: load_atomic_i128_aligned_seq_cst_const: ; -O1: ldar xzr, [x0] -; -O1: ldp x0, x1, [x0] -; -O1: dmb ish +; -O1: ldiapp x0, x1, [x0] %r = load atomic i128, ptr %ptr seq_cst, align 16 ret i128 %r } @@ -580,13 +576,11 @@ define dso_local i128 @load_atomic_i128_unaligned_seq_cst_const(ptr readonly %pt define dso_local fp128 @load_atomic_fp128_aligned_seq_cst(ptr %ptr) { ; -O0-LABEL: load_atomic_fp128_aligned_seq_cst: ; -O0: ldar x8, [x0] -; -O0: ldp x8, x9, [x0] -; -O0: dmb ish +; -O0: ldiapp x8, x9, [x0] ; ; -O1-LABEL: load_atomic_fp128_aligned_seq_cst: ; -O1: ldar xzr, [x0] -; -O1: ldp x8, x9, [x0] -; -O1: dmb ish +; -O1: ldiapp x8, x9, [x0] %r = load atomic fp128, ptr %ptr seq_cst, align 16 ret fp128 %r } @@ -594,13 +588,11 @@ define dso_local fp128 @load_atomic_fp128_aligned_seq_cst(ptr %ptr) { define dso_local fp128 @load_atomic_fp128_aligned_seq_cst_const(ptr readonly %ptr) { ; -O0-LABEL: load_atomic_fp128_aligned_seq_cst_const: ; -O0: ldar x8, [x0] -; -O0: ldp x8, x9, [x0] -; -O0: dmb ish +; -O0: ldiapp x8, x9, [x0] ; ; -O1-LABEL: load_atomic_fp128_aligned_seq_cst_const: ; -O1: ldar xzr, [x0] -; -O1: ldp x8, x9, [x0] -; -O1: dmb ish +; -O1: ldiapp x8, x9, [x0] %r = load atomic fp128, ptr %ptr seq_cst, align 16 ret fp128 %r } _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
