https://github.com/kasuga-fj created https://github.com/llvm/llvm-project/pull/174391
None >From 02c7724b8e401171ac789ee27e8b4cbbe7788161 Mon Sep 17 00:00:00 2001 From: Ryotaro Kasuga <[email protected]> Date: Fri, 26 Dec 2025 07:04:55 +0000 Subject: [PATCH] [MachinePipelner] Add loop-carried dependencies for global barriers --- llvm/lib/CodeGen/MachinePipeliner.cpp | 50 +++++++++++++++++-- .../sms-loop-carried-fp-exceptions1.mir | 12 ++++- .../Hexagon/swp-loop-carried-order-dep6.mir | 40 +++++++++------ 3 files changed, 81 insertions(+), 21 deletions(-) diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp index 6b022783f4bb8..e5376b06f83a2 100644 --- a/llvm/lib/CodeGen/MachinePipeliner.cpp +++ b/llvm/lib/CodeGen/MachinePipeliner.cpp @@ -349,6 +349,10 @@ class LoopCarriedOrderDepsTracker { const SUnitWithMemInfo &Dst); void computeDependenciesAux(); + + void setLoopCarriedDep(const SUnit *Src, const SUnit *Dst) { + LoopCarried[Src->NodeNum].set(Dst->NodeNum); + } }; } // end anonymous namespace @@ -1137,7 +1141,7 @@ void LoopCarriedOrderDepsTracker::addDependenciesBetweenSUs( return; if (hasLoopCarriedMemDep(Src, Dst, *BAA, TII, TRI, DAG)) - LoopCarried[Src.SU->NodeNum].set(Dst.SU->NodeNum); + setLoopCarriedDep(Src.SU, Dst.SU); } void LoopCarriedOrderDepsTracker::addLoopCarriedDepenenciesForChunks( @@ -1160,11 +1164,16 @@ void LoopCarriedOrderDepsTracker::addLoopCarriedDepenenciesForChunks( void LoopCarriedOrderDepsTracker::computeDependenciesAux() { SmallVector<LoadStoreChunk, 2> Chunks(1); + SUnit *FirstBarrier = nullptr; + SUnit *LastBarrier = nullptr; for (const auto &TSU : TaggedSUnits) { InstrTag Tag = TSU.getTag(); SUnit *SU = TSU.getPointer(); switch (Tag) { case InstrTag::Barrier: + if (!FirstBarrier) + FirstBarrier = SU; + LastBarrier = SU; Chunks.emplace_back(); break; case InstrTag::LoadOrStore: @@ -1182,9 +1191,42 @@ void LoopCarriedOrderDepsTracker::computeDependenciesAux() { for (const LoadStoreChunk &Chunk : Chunks) addLoopCarriedDepenenciesForChunks(Chunk, Chunk); - // TODO: If there are multiple barrier instructions, dependencies from the - // last barrier instruction (or load/store below it) to the first barrier - // instruction (or load/store above it). + // There is no barrier instruction between load/store instructions in the same + // LoadStoreChunk. If there are one or more barrier instructions, the + // instructions sequence is as follows: + // + // Loads/Stores (Chunks.front()) + // Barrier (FirstBarrier) + // Loads/Stores + // Barrier + // ... + // Loads/Stores + // Barrier (LastBarrier) + // Loads/Stores (Chunks.back()) + // + // Since loads/stores must not be reordered across barrier instructions, and + // the order of barrier instructions must be preserved, add the following + // loop-carried dependences: + // + // - LastBarrier -> Loads/Stores in Chunks.front() + // - Loads/Stores in Chunks.back() -> FirstBarrier + // - LastBarrier -> FirstBarrier (if they are different) + // + if (FirstBarrier) { + assert(LastBarrier && "Both barriers should be set."); + for (const SUnitWithMemInfo &Dst : Chunks.front().Loads) + setLoopCarriedDep(LastBarrier, Dst.SU); + for (const SUnitWithMemInfo &Dst : Chunks.front().Stores) + setLoopCarriedDep(LastBarrier, Dst.SU); + + for (const SUnitWithMemInfo &Src : Chunks.back().Loads) + setLoopCarriedDep(Src.SU, FirstBarrier); + for (const SUnitWithMemInfo &Src : Chunks.back().Stores) + setLoopCarriedDep(Src.SU, FirstBarrier); + + if (FirstBarrier != LastBarrier) + setLoopCarriedDep(LastBarrier, FirstBarrier); + } } /// Add a chain edge between a load and store if the store can be an diff --git a/llvm/test/CodeGen/AArch64/sms-loop-carried-fp-exceptions1.mir b/llvm/test/CodeGen/AArch64/sms-loop-carried-fp-exceptions1.mir index bcc6a3ea9b285..214f7e245030a 100644 --- a/llvm/test/CodeGen/AArch64/sms-loop-carried-fp-exceptions1.mir +++ b/llvm/test/CodeGen/AArch64/sms-loop-carried-fp-exceptions1.mir @@ -5,16 +5,24 @@ # floating-point exception, and there is an instruction for barrier event. In # this case the order of them must not change. # +# SU(2): May raise FP exception +# SU(3): May raise FP exception +# SU(4): Store +# SU(5): Barrier +# SU(7): Barrier +# # FIXME: Currently the following dependencies are missed. # # Loop carried edges from SU(7) # Order # SU(2) # SU(3) -# SU(4) -# SU(5) # CHECK: ===== Loop Carried Edges Begin ===== +# CHECK-NEXT: Loop carried edges from SU(7) +# CHECK-NEXT: Order +# CHECK-NEXT: SU(4) +# CHECK-NEXT: SU(5) # CHECK-NEXT: ===== Loop Carried Edges End ===== --- | diff --git a/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep6.mir b/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep6.mir index 4281d15377141..8fe3b3d83aa94 100644 --- a/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep6.mir +++ b/llvm/test/CodeGen/Hexagon/swp-loop-carried-order-dep6.mir @@ -19,23 +19,33 @@ # } # ``` # -# FIXME: Currently the following dependencies are missed. -# Loop carried edges from SU(16) -# Order -# SU(6) -# SU(8) -# SU(10) -# SU(11) -# Loop carried edges from SU(17) -# Order -# SU(10) -# SU(11) -# Loop carried edges from SU(19) -# Order -# SU(10) -# SU(11) +# SU(6): Load +# SU(8): Store +# SU(10): Store +# SU(11): Barrier +# SU(16): Barrier +# SU(17): Load +# SU(19): Load +# +# As the order between load/store and barrier must be preserved, the following +# loop-carried dependnecies need to be added: +# - SU(16) -> SU(6), SU(8), SU(10) +# - SU(17), SU(19) -> SU(11) +# - SU(16) -> SU(11) (barrier to barrier) # CHECK: ===== Loop Carried Edges Begin ===== +# CHECK-NEXT: Loop carried edges from SU(16) +# CHECK-NEXT: Order +# CHECK-NEXT: SU(6) +# CHECK-NEXT: SU(8) +# CHECK-NEXT: SU(10) +# CHECK-NEXT: SU(11) +# CHECK-NEXT: Loop carried edges from SU(17) +# CHECK-NEXT: Order +# CHECK-NEXT: SU(11) +# CHECK-NEXT: Loop carried edges from SU(19) +# CHECK-NEXT: Order +# CHECK-NEXT: SU(11) # CHECK-NEXT: ===== Loop Carried Edges End ===== --- | _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
