https://github.com/bcahoon updated https://github.com/llvm/llvm-project/pull/225989
>From 5fc97e129c245db1577d8c4e1aee613d0f286e63 Mon Sep 17 00:00:00 2001 From: Michael Halkenhaeuser <[email protected]> Date: Wed, 23 Sep 2026 13:07:08 -0500 Subject: [PATCH 1/3] [AMDGPU] Measure MFMA read hazards at each producer Introduce more sophisticated traversal to avoid the following traps: - order-dependent traversal and discarding seen BBs despite shorter path - mis-matching distance and window of different producers Record the best distance per BB instead of a visited flag and sweep the arrivals in nondecreasing distance (bucket queue). This pairs producers with their actual distance to a consumer in one go. Fixed scenarios: - MFMA reading an MFMA result as srcA, srcB or srcC - VALU, memory or export instruction reading an MFMA result AI-assisted. --- .../lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 284 +++++--- llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h | 10 + .../CodeGen/AMDGPU/mai-hazards-gfx90a.mir | 623 ++++++++++++++++++ .../CodeGen/AMDGPU/mai-hazards-gfx942.mir | 78 +++ 4 files changed, 918 insertions(+), 77 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp index 4d25a723ae4a7..c7b44d128541d 100644 --- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp +++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp @@ -956,6 +956,119 @@ hasHazard(StateT InitialState, return false; } +// Number of trailing terminators of \p Pred not executed on the edge to \p MBB: +// control leaves at the first terminator branching to \p MBB, so the ones after +// it are dead on that edge. A fallthrough executes them all. +static unsigned countUnexecutedTerminators(const MachineBasicBlock &Pred, + const MachineBasicBlock *MBB) { + const SIInstrInfo &TII = + *Pred.getParent()->getSubtarget<GCNSubtarget>().getInstrInfo(); + unsigned Unexecuted = 0, Passed = 0; + for (auto I = Pred.instr_rbegin(), E = Pred.instr_rend(); + I != E && I->isTerminator(); ++I, ++Passed) { + // Only an explicit MBB destination identifies this edge; the last match is + // the earliest such branch in program order. + if (I->isBranch() && I->getNumOperands() > 0 && I->getOperand(0).isMBB() && + TII.getBranchDestBlock(*I) == MBB) + Unexecuted = Passed; + } + return Unexecuted; +} + +// The block walk behind GCNHazardRecognizer::getMaxWindowDeficit, which +// documents what it returns. Can only be run in a hazard recognizer mode. +static int getMaxWindowDeficit( + function_ref<std::optional<int>(const MachineInstr &)> WindowFor, + const MachineInstr *MI, int MaxWindow) { + int Deficit = 0; + + // Arrivals are swept in nondecreasing distance, so the first one at a block + // that is not stale is the shortest way into it. A queued Start points past + // the block's terminators, with the ones the edge executes already counted + // in Distance. + struct Arrival { + const MachineBasicBlock *MBB; + MachineBasicBlock::const_reverse_instr_iterator Start; + int Distance; + }; + SmallVector<SmallVector<Arrival, 2>, 4> Pending; + DenseMap<const MachineBasicBlock *, int> Best; + + auto Arrive = [&](const MachineBasicBlock *MBB, + MachineBasicBlock::const_reverse_instr_iterator Start, + int Distance) { + // Nothing this far back can ask for more than the answer already holds. + if (MaxWindow - Distance <= Deficit) + return; + + if (Pending.empty()) + Pending.resize(MaxWindow); + + auto [It, Inserted] = Best.try_emplace(MBB, Distance); + if (!Inserted) { + if (It->second <= Distance) + return; + It->second = Distance; + } + + assert(Distance < MaxWindow && "arrival outside the window"); + Pending[Distance].push_back({MBB, Start, Distance}); + }; + + // Taken by value: Arrive can grow the bucket this arrival is stored in. + auto Scan = [&](Arrival A) { + if (Best.lookup(A.MBB) < A.Distance) + return; + + int Distance = A.Distance; + for (auto I = A.Start, E = A.MBB->instr_rend(); I != E; ++I) { + if (I->isBundle()) + continue; + + if (std::optional<int> Window = WindowFor(*I)) { + assert(*Window >= 0 && *Window <= MaxWindow && "window out of bounds"); + Deficit = std::max(Deficit, *Window - Distance); + } + + if (I->isInlineAsm()) + continue; + + Distance += SIInstrInfo::getNumWaitStates(*I); + + if (Distance >= MaxWindow) + return; + } + + for (const MachineBasicBlock *Pred : A.MBB->predecessors()) { + auto Start = Pred->instr_rbegin(), End = Pred->instr_rend(); + for (unsigned N = countUnexecutedTerminators(*Pred, A.MBB); N; --N) + ++Start; + + int Executed = 0; + for (; Start != End && Start->isTerminator(); ++Start) { + assert(!WindowFor(*Start) && "terminators must not ask for a window"); + Executed += SIInstrInfo::getNumWaitStates(*Start); + } + + Arrive(Pred, Start, Distance + Executed); + } + }; + + // Not recorded in Best: a backedge may rescan this block from its end. + Scan({MI->getParent(), std::next(MI->getReverseIterator()), 0}); + + // Arrivals are never queued below the distance being swept, so one increasing + // sweep suffices; equal distances land in the bucket in hand, which the inner + // loop picks up by re-reading its size rather than iterating. Best keeps a + // zero-cost cycle from looping. + for (int Distance = 0; Distance < static_cast<int>(Pending.size()); + ++Distance) + for (unsigned I = 0; I != Pending[Distance].size(); ++I) + Scan(Pending[Distance][I]); + + return Deficit; +} + // Returns a minimum wait states since \p I walking all predecessors. // Only scans until \p IsExpired does not return true. // Can only be run in a hazard recognizer mode. @@ -1010,6 +1123,33 @@ getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard, Visited, GetNumWaitStates); } +int GCNHazardRecognizer::getMaxWindowDeficit(int MaxWindow, + WindowForFn WindowFor) const { + if (isHazardRecognizerMode()) + return ::getMaxWindowDeficit(WindowFor, CurrCycleInstr, MaxWindow); + + // EmittedInstrs is capped and can be shorter than the widest window, which + // costs scheduling quality only: the standalone pass still pads. + int Deficit = 0, Distance = 0; + for (MachineInstr *MI : EmittedInstrs) { + if (MI) { + if (std::optional<int> Window = WindowFor(*MI)) { + assert(*Window >= 0 && *Window <= MaxWindow && "window out of bounds"); + Deficit = std::max(Deficit, *Window - Distance); + } + + if (MI->isInlineAsm()) + continue; + } + + Distance += MI ? SIInstrInfo::getNumWaitStates(*MI) : 1; + + if (Distance >= MaxWindow) + break; + } + return Deficit; +} + int GCNHazardRecognizer::getWaitStatesSince( IsHazardFn IsHazard, int Limit, GetNumWaitStatesFn GetNumWaitStates) const { if (isHazardRecognizerMode()) { @@ -3410,35 +3550,21 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) const { if (!Use.isReg()) continue; Register Reg = Use.getReg(); - const MachineInstr *MI1; - - auto IsOverlappedMFMAFn = [Reg, &MI1, this](const MachineInstr &MI) { - if (!SIInstrInfo::isMFMA(MI)) - return false; - Register DstReg = MI.getOperand(0).getReg(); - MI1 = &MI; - return TRI.regsOverlap(DstReg, Reg); - }; WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates - getWaitStatesSinceDef(Reg, IsLegacyVALUNotDotFn, MaxWaitStates); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); - int NumWaitStates = - getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn, MaxWaitStates); - if (NumWaitStates == std::numeric_limits<int>::max()) - continue; - int OpNo = Use.getOperandNo(); - int NeedWaitStates = getMFMAReadWaitStates(*MI, *MI1, Reg, OpNo == SrcCIdx); - - assert(NeedWaitStates <= MaxWaitStates && - "hazard requirement exceeds the scan window"); - if (WaitStatesNeeded >= NeedWaitStates) - continue; - - WaitStatesNeededForUse = NeedWaitStates - NumWaitStates; - WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); + WaitStatesNeeded = std::max( + WaitStatesNeeded, + getMaxWindowDeficit( + MaxWaitStates, [&](const MachineInstr &P) -> std::optional<int> { + if (!SIInstrInfo::isMFMA(P) || + !TRI.regsOverlap(P.getOperand(0).getReg(), Reg)) + return std::nullopt; + return getMFMAReadWaitStates(*MI, P, Reg, OpNo == SrcCIdx); + })); if (WaitStatesNeeded == MaxWaitStates) break; @@ -3594,8 +3720,8 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const { bool IsMemOrExport = IsMem || SIInstrInfo::isEXP(*MI); bool IsVALU = SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true); - const MachineInstr *MFMA = nullptr; unsigned Reg; + const MachineInstr *MFMA = nullptr; auto IsMFMAWriteFn = [&Reg, &MFMA, this](const MachineInstr &MI) { if (!SIInstrInfo::isMFMA(MI) || !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg)) @@ -3683,63 +3809,67 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const { } } - MFMA = nullptr; - WaitStatesSinceDef = - getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates); - if (!MFMA) - continue; - - unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA); - int NumPasses = HazardDefLatency; - int NeedWaitStates = MaxWaitStates; - - if (SIInstrInfo::isDGEMM(MFMA->getOpcode())) { - switch (HazardDefLatency) { - case 4: - NeedWaitStates = IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates - : DMFMA4x4WriteVgprVALUReadWaitStates; - break; - case 8: - case 16: + auto MFMAWindow = [&](const MachineInstr &Producer) { + int NumPasses = TSchedModel.computeInstrLatency(&Producer); + int NeedWaitStates = MaxWaitStates; + + if (SIInstrInfo::isDGEMM(Producer.getOpcode())) { + switch (NumPasses) { + case 4: + NeedWaitStates = IsMemOrExport + ? DMFMA4x4WriteVgprMemExpReadWaitStates + : DMFMA4x4WriteVgprVALUReadWaitStates; + break; + case 8: + case 16: + NeedWaitStates = + IsMemOrExport + ? DMFMA16x16WriteVgprMemExpReadWaitStates + : (ST.hasGFX950Insts() + ? GFX950_DMFMA16x16WriteVgprVALUReadWaitStates + : DMFMA16x16WriteVgprVALUReadWaitStates); + break; + default: + llvm_unreachable("unexpected dgemm"); + } + } else if (ST.hasGFX940Insts()) { NeedWaitStates = - IsMemOrExport - ? DMFMA16x16WriteVgprMemExpReadWaitStates - : (ST.hasGFX950Insts() - ? GFX950_DMFMA16x16WriteVgprVALUReadWaitStates - : DMFMA16x16WriteVgprVALUReadWaitStates); - break; - default: - llvm_unreachable("unexpected dgemm"); - } - } else if (ST.hasGFX940Insts()) { - NeedWaitStates = - TII.isXDL(*MFMA) - ? GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates( - NumPasses, ST.hasGFX950Insts()) - : GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates( - NumPasses); - } else { - switch (HazardDefLatency) { - case 2: - NeedWaitStates = SMFMA4x4WriteVgprVALUMemExpReadWaitStates; - break; - case 8: - NeedWaitStates = SMFMA16x16WriteVgprVALUMemExpReadWaitStates; - break; - case 16: - NeedWaitStates = SMFMA32x32WriteVgprVALUMemExpReadWaitStates; - break; - default: - llvm_unreachable("unexpected number of passes for mfma"); + TII.isXDL(Producer) + ? GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates( + NumPasses, ST.hasGFX950Insts()) + : GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates( + NumPasses); + } else { + switch (NumPasses) { + case 2: + NeedWaitStates = SMFMA4x4WriteVgprVALUMemExpReadWaitStates; + break; + case 8: + NeedWaitStates = SMFMA16x16WriteVgprVALUMemExpReadWaitStates; + break; + case 16: + NeedWaitStates = SMFMA32x32WriteVgprVALUMemExpReadWaitStates; + break; + default: + llvm_unreachable("unexpected number of passes for mfma"); + } } - } - assert(NeedWaitStates <= MaxWaitStates && - "hazard requirement exceeds the scan window"); - int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; - WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); + assert(NeedWaitStates <= MaxWaitStates && + "hazard requirement exceeds the scan window"); + return NeedWaitStates; + }; - if (WaitStatesNeeded == MaxWaitStates) + WaitStatesNeeded = std::max( + WaitStatesNeeded, + getMaxWindowDeficit(MaxWaitStates, + [&](const MachineInstr &P) -> std::optional<int> { + if (!IsMFMAWriteFn(P)) + return std::nullopt; + return MFMAWindow(P); + })); + + if (WaitStatesNeeded >= MaxWaitStates) break; } } diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h index 60ebb73f99286..3910f676c9e98 100644 --- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h +++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h @@ -176,6 +176,16 @@ class GCNHazardRecognizer final : public ScheduleHazardRecognizer { // used on a newly inserted instruction before returning from PreEmitNoops. void runOnInstruction(MachineInstr *MI); + /// Wait states required after \p MI, or nullopt if \p MI is not waited for. + /// Must return nullopt for terminators. + typedef function_ref<std::optional<int>(const MachineInstr &)> WindowForFn; + + /// Returns the largest WindowFor(I) - distance(I) over the instructions + /// preceding the one being checked within \p MaxWindow, which must bound + /// every window WindowFor can return, and zero if it accepts none. Each + /// window is paired with the distance to the instruction that supplied it. + int getMaxWindowDeficit(int MaxWindow, WindowForFn WindowFor) const; + int getWaitStatesSince(IsHazardFn IsHazard, int Limit, GetNumWaitStatesFn GetNumWaitStates) const; int getWaitStatesSince(IsHazardFn IsHazard, int Limit) const; diff --git a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir index bf11056022b7d..f62457aa5f3d4 100644 --- a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir +++ b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir @@ -1453,3 +1453,626 @@ body: | $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X4F16_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_I32_16X16X4I8_e64 $vgpr0, $vgpr1, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec ... +# Window 11 with the nearest producer 1 wait state away needs 10; taking the +# detour's distance leaves it two short. +# GCN-LABEL: name: smfma16x16_write_vgpr_valu_read_diamond_long_edge_first +# GCN: bb.3: +# GCN-NEXT: S_NOP 9 +# GCN-NEXT: $vgpr4 = V_MOV_B32_e32 $vgpr2 +name: smfma16x16_write_vgpr_valu_read_diamond_long_edge_first +body: | + bb.0: + successors: %bb.2 + S_BRANCH %bb.2 + + bb.1: + successors: %bb.3 + $sgpr0 = S_MOV_B32 0 + S_BRANCH %bb.3 + + bb.2: + successors: %bb.1, %bb.3 + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr8_vgpr9, $vgpr10_vgpr11, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec + S_CBRANCH_SCC1 %bb.1, implicit $scc + + bb.3: + $vgpr4 = V_MOV_B32_e32 $vgpr2, implicit $exec + S_ENDPGM 0 +... +# GCN-LABEL: name: smfma16x16_write_vgpr_valu_read_diamond_short_edge_first +# GCN: bb.3: +# GCN-NEXT: S_NOP 9 +# GCN-NEXT: $vgpr4 = V_MOV_B32_e32 $vgpr2 +name: smfma16x16_write_vgpr_valu_read_diamond_short_edge_first +body: | + bb.0: + successors: %bb.1 + S_BRANCH %bb.1 + + bb.1: + successors: %bb.2, %bb.3 + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr8_vgpr9, $vgpr10_vgpr11, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec + S_CBRANCH_SCC1 %bb.3, implicit $scc + + bb.2: + successors: %bb.3 + $sgpr0 = S_MOV_B32 0 + S_BRANCH %bb.3 + + bb.3: + $vgpr4 = V_MOV_B32_e32 $vgpr2, implicit $exec + S_ENDPGM 0 +... + +# The same with a detour longer than the window: expiring on the long edge must +# not stop the direct edge from being measured. + +# GCN-LABEL: name: smfma16x16_valu_read_detour_longer_than_window +# GCN: bb.3: +# GCN-NEXT: S_NOP 9 +# GCN-NEXT: $vgpr4 = V_MOV_B32_e32 $vgpr2 +name: smfma16x16_valu_read_detour_longer_than_window +body: | + bb.0: + successors: %bb.2 + S_BRANCH %bb.2 + + bb.1: + successors: %bb.3 + $sgpr0 = S_MOV_B32 0 + $sgpr1 = S_MOV_B32 0 + $sgpr2 = S_MOV_B32 0 + $sgpr3 = S_MOV_B32 0 + $sgpr4 = S_MOV_B32 0 + $sgpr5 = S_MOV_B32 0 + $sgpr6 = S_MOV_B32 0 + $sgpr7 = S_MOV_B32 0 + $sgpr8 = S_MOV_B32 0 + $sgpr9 = S_MOV_B32 0 + $sgpr10 = S_MOV_B32 0 + $sgpr11 = S_MOV_B32 0 + S_BRANCH %bb.3 + + bb.2: + successors: %bb.1, %bb.3 + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr8_vgpr9, $vgpr10_vgpr11, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec + S_CBRANCH_SCC1 %bb.1, implicit $scc + + bb.3: + $vgpr4 = V_MOV_B32_e32 $vgpr2, implicit $exec + S_ENDPGM 0 +... +# An MFMA overlapping nothing must not set the window for one that does: the +# 32x32 writing v[0:3] asks for 16, the unrelated 4x4 for 2. +# GCN-LABEL: name: smfma32x32_srcc_read_unrelated_mfma_producer_block_first +# GCN: bb.3: +# GCN-NEXT: S_NOP 14 +# GCN-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32 +name: smfma32x32_srcc_read_unrelated_mfma_producer_block_first +body: | + bb.0: + successors: %bb.1, %bb.2 + S_CBRANCH_SCC1 %bb.2, implicit $scc + S_BRANCH %bb.1 + + bb.1: + successors: %bb.3 + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.3 + + bb.2: + successors: %bb.3 + $vgpr40_vgpr41_vgpr42_vgpr43 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr40_vgpr41_vgpr42_vgpr43, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.3 + + bb.3: + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec + S_ENDPGM 0 +... +# GCN-LABEL: name: smfma32x32_srcc_read_unrelated_mfma_decoy_block_first +# GCN: bb.3: +# GCN-NEXT: S_NOP 14 +# GCN-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32 +name: smfma32x32_srcc_read_unrelated_mfma_decoy_block_first +body: | + bb.0: + successors: %bb.1, %bb.2 + S_CBRANCH_SCC1 %bb.2, implicit $scc + S_BRANCH %bb.1 + + bb.1: + successors: %bb.3 + $vgpr40_vgpr41_vgpr42_vgpr43 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr40_vgpr41_vgpr42_vgpr43, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.3 + + bb.2: + successors: %bb.3 + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.3 + + bb.3: + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec + S_ENDPGM 0 +... +# srcC is exactly the producer's destination, so nothing is needed; an unrelated +# MFMA of the same shape must not turn that into 15. +# GCN-LABEL: name: smfma32x32_full_srcc_read_unrelated_mfma +# GCN: bb.3: +# GCN-NEXT: early-clobber {{.*}} = V_MFMA_F32_32X32X2F32 +name: smfma32x32_full_srcc_read_unrelated_mfma +body: | + bb.0: + successors: %bb.1, %bb.2 + S_CBRANCH_SCC1 %bb.2, implicit $scc + S_BRANCH %bb.1 + + bb.1: + successors: %bb.3 + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.3 + + bb.2: + successors: %bb.3 + $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 = V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.3 + + bb.3: + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + S_ENDPGM 0 +... +# Window and distance must come from one producer: the 32x32 needs 19 from 11 +# away, the 4x4 needs 5 from 1 away, so 8. Mispairing would ask 18. + +# GCN-LABEL: name: mfma_mixed_window_far_wide_near_narrow +# GCN: bb.3: +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: $vgpr16 = V_MOV_B32 +name: mfma_mixed_window_far_wide_near_narrow +body: | + bb.0: + successors: %bb.2, %bb.1 + S_CBRANCH_SCC1 %bb.2, implicit $scc + S_BRANCH %bb.1 + + bb.1: + successors: %bb.3 + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + $sgpr0 = S_MOV_B32 0 + $sgpr1 = S_MOV_B32 0 + $sgpr2 = S_MOV_B32 0 + $sgpr3 = S_MOV_B32 0 + $sgpr4 = S_MOV_B32 0 + $sgpr5 = S_MOV_B32 0 + $sgpr6 = S_MOV_B32 0 + $sgpr7 = S_MOV_B32 0 + $sgpr8 = S_MOV_B32 0 + $sgpr9 = S_MOV_B32 0 + S_BRANCH %bb.3 + + bb.2: + successors: %bb.3 + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.3 + + bb.3: + $vgpr16 = V_MOV_B32_e32 $vgpr0, implicit $exec + S_ENDPGM 0 +... +# The same around a loop: the 16x16 needs 11 from 1 away, the 4x4 needs 5 around +# the latch, so 10. Mispairing would ask 4. + +# GCN-LABEL: name: mfma_mixed_window_loop +# GCN: bb.2: +# GCN: S_NOP 9 +# GCN-NEXT: $vgpr16 = V_MOV_B32 +name: mfma_mixed_window_loop +body: | + bb.0: + successors: %bb.1 + S_BRANCH %bb.1 + + bb.1: + successors: %bb.2 + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_16X16X1F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.2 + + bb.2: + successors: %bb.3, %bb.4 + $vgpr16 = V_MOV_B32_e32 $vgpr0, implicit $exec + S_CBRANCH_SCC1 %bb.4, implicit $scc + S_BRANCH %bb.3 + + bb.3: + successors: %bb.2 + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.2 + + bb.4: + S_ENDPGM 0 +... +# A trailing branch to another successor is not executed on the edge into the +# consumer, so it must not count. Both need 10. + +# GCN-LABEL: name: smfma16x16_valu_read_two_terminators +# GCN: bb.2: +# GCN-NEXT: S_NOP 9 +# GCN-NEXT: $vgpr40 = V_MOV_B32 +name: smfma16x16_valu_read_two_terminators +body: | + bb.0: + successors: %bb.1, %bb.2 + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_16X16X1F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + S_CBRANCH_SCC1 %bb.2, implicit $scc + S_BRANCH %bb.1 + + bb.1: + successors: %bb.2 + S_BRANCH %bb.2 + + bb.2: + $vgpr40 = V_MOV_B32_e32 $vgpr0, implicit $exec + S_ENDPGM 0 +... +# Only the bb.3 edge skips bb.0's trailing S_BRANCH, so the edges arrive at +# different distances; along the cheaper one 3 of the 19 are left. + +# GCN-LABEL: name: smfma32x32_valu_read_unexecuted_branch_on_one_edge +# GCN: bb.4: +# GCN: $sgpr5 = S_MOV_B32 0 +# GCN-NEXT: S_NOP 2 +# GCN-NEXT: $vgpr16 = V_MOV_B32 +name: smfma32x32_valu_read_unexecuted_branch_on_one_edge +body: | + bb.0: + successors: %bb.1, %bb.3 + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + $sgpr0 = S_MOV_B32 0 + $sgpr1 = S_MOV_B32 0 + $sgpr2 = S_MOV_B32 0 + S_CBRANCH_SCC1 %bb.3, implicit $scc + S_BRANCH %bb.1 + bb.1: + successors: %bb.2, %bb.3 + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_16X16X1F32_vgprcd_e64 $vgpr20, $vgpr21, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + $sgpr0 = S_MOV_B32 0 + S_CBRANCH_SCC1 %bb.3, implicit $scc + S_BRANCH %bb.2 + bb.2: + successors: %bb.4 + $sgpr0 = S_MOV_B32 0 + S_BRANCH %bb.4 + bb.3: + successors: %bb.4 + $sgpr0 = S_MOV_B32 0 + $sgpr1 = S_MOV_B32 0 + $sgpr2 = S_MOV_B32 0 + $sgpr3 = S_MOV_B32 0 + $sgpr4 = S_MOV_B32 0 + S_BRANCH %bb.4 + bb.4: + $sgpr0 = S_MOV_B32 0 + $sgpr1 = S_MOV_B32 0 + $sgpr2 = S_MOV_B32 0 + $sgpr3 = S_MOV_B32 0 + $sgpr4 = S_MOV_B32 0 + $sgpr5 = S_MOV_B32 0 + $vgpr16 = V_MOV_B32_e32 $vgpr0, implicit $exec + S_ENDPGM 0 +... +# The same when the reader is an MFMA: srcA after a 32x32 needs 19, after a 4x4 +# needs 5, one branch away each, so 18. + +# GCN-LABEL: name: smfma_srca_read_mixed_shapes +# GCN: bb.3: +# GCN-NEXT: S_NOP 15 +# GCN-NEXT: S_NOP 1 +# GCN-NEXT: $agpr20_agpr21_agpr22_agpr23 = V_MFMA_F32_4X4X1F32 +name: smfma_srca_read_mixed_shapes +body: | + bb.0: + successors: %bb.1, %bb.2 + S_CBRANCH_SCC1 %bb.2, implicit $scc + S_BRANCH %bb.1 + + bb.1: + successors: %bb.3 + $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.3 + + bb.2: + successors: %bb.3 + $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.3 + + bb.3: + $agpr20_agpr21_agpr22_agpr23 = V_MFMA_F32_4X4X1F32_e64 $agpr1, $vgpr0, $agpr20_agpr21_agpr22_agpr23, 0, 0, 0, implicit $mode, implicit $exec + S_ENDPGM 0 +... +# A producer writing exactly the srcC register needs nothing and must not hide a +# farther partial overlap: the 16x16 needs 8 from 1 away. + +# GCN-LABEL: name: smfma_full_srcc_producer_does_not_hide_partial +# GCN: V_MFMA_F32_4X4X1F32 +# GCN-NEXT: S_NOP 6 +# GCN-NEXT: $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32 +name: smfma_full_srcc_producer_does_not_hide_partial +body: | + bb.0: + $agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17 = V_MFMA_F32_16X16X1F32_e64 $vgpr1, $vgpr0, $agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17, 0, 0, 0, implicit $mode, implicit $exec + $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr30_agpr31_agpr32_agpr33, 0, 0, 0, implicit $mode, implicit $exec + $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr4_agpr5_agpr6_agpr7, 0, 0, 0, implicit $mode, implicit $exec + S_ENDPGM 0 +... +# Control leaves at the first terminator that transfers to the successor, so a +# taken conditional branch is the only one executed: 1 of the 11. + +# GCN-LABEL: name: smfma16x16_valu_read_same_target_terminators +# GCN: bb.1: +# GCN-NEXT: S_NOP 9 +# GCN-NEXT: $vgpr4 = V_MOV_B32 +name: smfma16x16_valu_read_same_target_terminators +body: | + bb.0: + successors: %bb.1 + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr8_vgpr9, $vgpr10_vgpr11, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec + S_CBRANCH_SCC1 %bb.1, implicit $scc + S_BRANCH %bb.1 + + bb.1: + $vgpr4 = V_MOV_B32_e32 $vgpr2, implicit $exec + S_ENDPGM 0 +... +# Each successor executes a different number of the three terminators, so the +# same producer is 1 and 3 wait states away. + +# GCN-LABEL: name: smfma16x16_valu_read_first_conditional_target +# GCN: bb.2: +# GCN-NEXT: S_NOP 9 +# GCN-NEXT: $vgpr4 = V_MOV_B32 +name: smfma16x16_valu_read_first_conditional_target +body: | + bb.0: + successors: %bb.1, %bb.2, %bb.3 + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr8_vgpr9, $vgpr10_vgpr11, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec + S_CBRANCH_SCC1 %bb.2, implicit $scc + S_CBRANCH_EXECZ %bb.3, implicit $exec + S_BRANCH %bb.1 + + bb.1: + S_ENDPGM 0 + + bb.2: + $vgpr4 = V_MOV_B32_e32 $vgpr2, implicit $exec + S_ENDPGM 0 + + bb.3: + S_ENDPGM 0 +... +# GCN-LABEL: name: smfma16x16_valu_read_unconditional_target +# GCN: bb.1: +# GCN-NEXT: S_NOP 7 +# GCN-NEXT: $vgpr4 = V_MOV_B32 +name: smfma16x16_valu_read_unconditional_target +body: | + bb.0: + successors: %bb.1, %bb.2, %bb.3 + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr8_vgpr9, $vgpr10_vgpr11, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec + S_CBRANCH_SCC1 %bb.2, implicit $scc + S_CBRANCH_EXECZ %bb.3, implicit $exec + S_BRANCH %bb.1 + + bb.1: + $vgpr4 = V_MOV_B32_e32 $vgpr2, implicit $exec + S_ENDPGM 0 + + bb.2: + S_ENDPGM 0 + + bb.3: + S_ENDPGM 0 +... + +# The empty fallthrough contributes nothing: along bb.0 -> bb.1 -> bb.2 only the +# conditional branch executes, leaving 5 of the 11. + +# GCN-LABEL: name: smfma16x16_valu_read_across_empty_fallthrough +# GCN: bb.2: +# GCN-NEXT: $sgpr0 = S_MOV_B32 0 +# GCN: S_NOP 4 +# GCN-NEXT: $vgpr4 = V_MOV_B32 +name: smfma16x16_valu_read_across_empty_fallthrough +body: | + bb.0: + successors: %bb.1, %bb.3 + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr8_vgpr9, $vgpr10_vgpr11, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec + S_CBRANCH_SCC1 %bb.1, implicit $scc + S_BRANCH %bb.3 + + bb.1: + successors: %bb.2 + + bb.2: + $sgpr0 = S_MOV_B32 0 + $sgpr1 = S_MOV_B32 0 + $sgpr2 = S_MOV_B32 0 + $sgpr3 = S_MOV_B32 0 + $sgpr4 = S_MOV_B32 0 + $vgpr4 = V_MOV_B32_e32 $vgpr0, implicit $exec + S_ENDPGM 0 + + bb.3: + S_ENDPGM 0 +... + +# Blocks holding nothing but a branch: with no instruction to notice the window +# is spent, the bound has to be tested on the edge. 24 is past it. + +# GCN-LABEL: name: valu_read_across_long_branch_chain +# GCN: bb.24: +# GCN-NEXT: $vgpr8 = V_MOV_B32 +name: valu_read_across_long_branch_chain +body: | + bb.0: + successors: %bb.1 + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr1, $vgpr0, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.1 + + bb.1: + successors: %bb.2 + S_BRANCH %bb.2 + + bb.2: + successors: %bb.3 + S_BRANCH %bb.3 + + bb.3: + successors: %bb.4 + S_BRANCH %bb.4 + + bb.4: + successors: %bb.5 + S_BRANCH %bb.5 + + bb.5: + successors: %bb.6 + S_BRANCH %bb.6 + + bb.6: + successors: %bb.7 + S_BRANCH %bb.7 + + bb.7: + successors: %bb.8 + S_BRANCH %bb.8 + + bb.8: + successors: %bb.9 + S_BRANCH %bb.9 + + bb.9: + successors: %bb.10 + S_BRANCH %bb.10 + + bb.10: + successors: %bb.11 + S_BRANCH %bb.11 + + bb.11: + successors: %bb.12 + S_BRANCH %bb.12 + + bb.12: + successors: %bb.13 + S_BRANCH %bb.13 + + bb.13: + successors: %bb.14 + S_BRANCH %bb.14 + + bb.14: + successors: %bb.15 + S_BRANCH %bb.15 + + bb.15: + successors: %bb.16 + S_BRANCH %bb.16 + + bb.16: + successors: %bb.17 + S_BRANCH %bb.17 + + bb.17: + successors: %bb.18 + S_BRANCH %bb.18 + + bb.18: + successors: %bb.19 + S_BRANCH %bb.19 + + bb.19: + successors: %bb.20 + S_BRANCH %bb.20 + + bb.20: + successors: %bb.21 + S_BRANCH %bb.21 + + bb.21: + successors: %bb.22 + S_BRANCH %bb.22 + + bb.22: + successors: %bb.23 + S_BRANCH %bb.23 + + bb.23: + successors: %bb.24 + S_BRANCH %bb.24 + + bb.24: + $vgpr8 = V_MOV_B32_e32 $vgpr0, implicit $exec + S_ENDPGM 0 +... +# Every block in the empty chain is reached at the same distance, once by +# fallthrough and once from a branch below, so the traversal keeps queueing +# arrivals at the distance it is currently visiting. The chain costs nothing, +# leaving the whole 11 of the window. + +# GCN-LABEL: name: smfma16x16_valu_read_many_equal_distance_arrivals +# GCN: bb.7: +# GCN: S_NOP 10 +# GCN-NEXT: $vgpr4 = V_MOV_B32 +name: smfma16x16_valu_read_many_equal_distance_arrivals +body: | + bb.0: + successors: %bb.1 + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr8_vgpr9, $vgpr10_vgpr11, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec + + bb.1: + successors: %bb.2 + + bb.2: + successors: %bb.3 + + bb.3: + successors: %bb.4 + + bb.4: + successors: %bb.5 + + bb.5: + successors: %bb.6 + + bb.6: + successors: %bb.7 + + bb.7: + successors: %bb.8 + $vgpr4 = V_MOV_B32_e32 $vgpr2, implicit $exec + S_BRANCH %bb.8 + + bb.8: + successors: %bb.1, %bb.9 + S_CBRANCH_SCC1 %bb.1, implicit $scc + + bb.9: + successors: %bb.2, %bb.10 + S_CBRANCH_SCC1 %bb.2, implicit $scc + + bb.10: + successors: %bb.3, %bb.11 + S_CBRANCH_SCC1 %bb.3, implicit $scc + + bb.11: + successors: %bb.4, %bb.12 + S_CBRANCH_SCC1 %bb.4, implicit $scc + + bb.12: + successors: %bb.5, %bb.13 + S_CBRANCH_SCC1 %bb.5, implicit $scc + + bb.13: + successors: %bb.6 + S_CBRANCH_SCC1 %bb.6, implicit $scc + S_ENDPGM 0 +... diff --git a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir index 2f78efbdd256e..f97df59ee1b52 100644 --- a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir +++ b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir @@ -2715,3 +2715,81 @@ body: | $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X2F32_e64 $vgpr0, $vgpr1, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_16X16X1F32_e64 $vgpr0, $vgpr1, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec ... + +# Window and distance must come from one producer: srcA after a 16-pass XDL +# needs NumPasses + 3, one more on gfx950, after a 2-pass non-XDL NumPasses + 2. + +# GCN-LABEL: name: mfma_srca_read_mixed_xdl_and_smfma +# GCN: bb.3: +# GCN-NEXT: S_NOP 15 +# GFX942-NEXT: S_NOP 1 +# GFX950-NEXT: S_NOP 2 +# GCN-NEXT: $vgpr40_vgpr41_vgpr42_vgpr43 = V_MFMA_F32_4X4X1F32 +name: mfma_srca_read_mixed_xdl_and_smfma +body: | + bb.0: + successors: %bb.1, %bb.2 + S_CBRANCH_SCC1 %bb.2, implicit $scc + S_BRANCH %bb.1 + + bb.1: + successors: %bb.3 + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_MFMA_F32_32X32X4F16_vgprcd_e64 $vgpr34_vgpr35, $vgpr36_vgpr37, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.3 + + bb.2: + successors: %bb.3 + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr38, $vgpr39, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.3 + + bb.3: + $vgpr40_vgpr41_vgpr42_vgpr43 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr0, $vgpr39, $vgpr40_vgpr41_vgpr42_vgpr43, 0, 0, 0, implicit $mode, implicit $exec + S_ENDPGM 0 +... + +# The same for a srcC read, whose windows come from another table: 8-pass XDL +# needs NumPasses + 1, one more on gfx950, the 2-pass non-XDL one 2. + +# GCN-LABEL: name: mfma_srcc_read_mixed_xdl_and_smfma +# GCN: bb.3: +# GFX942-NEXT: S_NOP 7 +# GFX950-NEXT: S_NOP 8 +# GCN-NEXT: $agpr20_agpr21_agpr22_agpr23 = V_MFMA_F32_4X4X1F32 +name: mfma_srcc_read_mixed_xdl_and_smfma +body: | + bb.0: + successors: %bb.1, %bb.2 + S_CBRANCH_SCC1 %bb.2, implicit $scc + S_BRANCH %bb.1 + + bb.1: + successors: %bb.3 + $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_e64 $vgpr2_vgpr3, $vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.3 + + bb.2: + successors: %bb.3 + $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.3 + + bb.3: + $agpr20_agpr21_agpr22_agpr23 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + S_ENDPGM 0 +... +# A 4-pass producer writing exactly the srcC register asks nothing and must not +# end the search: the 8-pass XDL result 2 away needs 9, 10 on gfx950. + +# GCN-LABEL: name: mfma_full_srcc_producer_does_not_hide_partial +# GCN: V_MFMA_F32_16X16X16F16 +# GFX942-NEXT: S_NOP 7 +# GFX950-NEXT: S_NOP 8 +# GCN-NEXT: $agpr40_agpr41_agpr42_agpr43 = V_MFMA_F32_4X4X1F32 +name: mfma_full_srcc_producer_does_not_hide_partial +body: | + bb.0: + $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_e64 $vgpr2_vgpr3, $vgpr4_vgpr5, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_16X16X16F16_e64 $vgpr2_vgpr3, $vgpr4_vgpr5, $agpr30_agpr31_agpr32_agpr33, 0, 0, 0, implicit $mode, implicit $exec + $agpr40_agpr41_agpr42_agpr43 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr4_agpr5_agpr6_agpr7, 0, 0, 0, implicit $mode, implicit $exec + S_ENDPGM 0 +... + >From a122e864583bc42fa1e88d54f40eb012de6e69a7 Mon Sep 17 00:00:00 2001 From: Michael Halkenhaeuser <[email protected]> Date: Wed, 23 Sep 2026 12:52:16 -0500 Subject: [PATCH 2/3] [AMDGPU] Measure MFMA overwrite hazards at each instruction Apply previously established processing to: - VALU overwriting an MFMA result - VALU overwriting a register an MFMA took as srcC AI-assisted. --- .../lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 96 +++++++++---------- .../CodeGen/AMDGPU/mai-hazards-gfx90a.mir | 51 ++++++++++ .../CodeGen/AMDGPU/mai-hazards-gfx942.mir | 58 +++++++++++ 3 files changed, 156 insertions(+), 49 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp index c7b44d128541d..0c13255a68e72 100644 --- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp +++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp @@ -1017,6 +1017,7 @@ static int getMaxWindowDeficit( // Taken by value: Arrive can grow the bucket this arrival is stored in. auto Scan = [&](Arrival A) { + // A closer arrival has superseded this one. if (Best.lookup(A.MBB) < A.Distance) return; @@ -3721,12 +3722,10 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const { bool IsVALU = SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true); unsigned Reg; - const MachineInstr *MFMA = nullptr; - auto IsMFMAWriteFn = [&Reg, &MFMA, this](const MachineInstr &MI) { + auto IsMFMAWriteFn = [&Reg, this](const MachineInstr &MI) { if (!SIInstrInfo::isMFMA(MI) || !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg)) return false; - MFMA = &MI; return true; }; @@ -3912,14 +3911,11 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const { WaitStatesNeeded = std::max(WaitStatesNeeded, DotWriteDifferentVALUWrite - WaitStatesSinceDef); - MFMA = nullptr; - WaitStatesSinceDef = - getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates); - if (MFMA) { + auto MFMAWawWindow = [&](const MachineInstr &Producer) { int NeedWaitStates = MaxWaitStates; - int NumPasses = TSchedModel.computeInstrLatency(MFMA); + int NumPasses = TSchedModel.computeInstrLatency(&Producer); - if (SIInstrInfo::isDGEMM(MFMA->getOpcode())) { + if (SIInstrInfo::isDGEMM(Producer.getOpcode())) { switch (NumPasses) { case 4: NeedWaitStates = DMFMA4x4WriteVgprVALUWriteWaitStates; @@ -3933,7 +3929,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const { } } else if (ST.hasGFX940Insts()) { NeedWaitStates = - TII.isXDL(*MFMA) + TII.isXDL(Producer) ? GFX940_XDL_N_PassWriteVgprVALUWawWaitStates( NumPasses, ST.hasGFX950Insts()) : GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(NumPasses); @@ -3955,54 +3951,56 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const { assert(NeedWaitStates <= MaxWaitStates && "hazard requirement exceeds the scan window"); - int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; - WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); + return NeedWaitStates; + }; - if (WaitStatesNeeded == MaxWaitStates) - break; - } + WaitStatesNeeded = std::max( + WaitStatesNeeded, + getMaxWindowDeficit(MaxWaitStates, + [&](const MachineInstr &P) -> std::optional<int> { + if (!IsMFMAWriteFn(P)) + return std::nullopt; + return MFMAWawWindow(P); + })); + + if (WaitStatesNeeded >= MaxWaitStates) + break; - auto IsSMFMAReadAsCFn = [&Reg, &MFMA, this](const MachineInstr &MI) { - if (!SIInstrInfo::isMFMA(MI) || SIInstrInfo::isDGEMM(MI.getOpcode()) || - !MI.readsRegister(Reg, &TRI)) - return false; + // Wait states this write needs after an MFMA that read Reg as srcC. + auto SMFMAReadAsCWindow = + [&](const MachineInstr &Reader) -> std::optional<int> { + if (!SIInstrInfo::isMFMA(Reader) || + SIInstrInfo::isDGEMM(Reader.getOpcode()) || + !Reader.readsRegister(Reg, &TRI)) + return std::nullopt; - if (ST.hasGFX940Insts() && !TII.isXDL(MI)) - return false; + if (ST.hasGFX940Insts() && !TII.isXDL(Reader)) + return std::nullopt; const MachineOperand *SrcC = - TII.getNamedOperand(MI, AMDGPU::OpName::src2); - assert(SrcC); + TII.getNamedOperand(Reader, AMDGPU::OpName::src2); + assert(SrcC && "an MFMA must have a src2 operand"); if (!SrcC->isReg() || !TRI.regsOverlap(SrcC->getReg(), Reg)) - return false; + return std::nullopt; - MFMA = &MI; - return true; + switch (TSchedModel.computeInstrLatency(&Reader)) { + case 2: + return SMFMA4x4ReadVgprVALUWarWaitStates; + case 4: + assert(ST.hasGFX940Insts() && "4-pass XDL is gfx940 or later"); + return GFX940_XDL4PassReadVgprVALUWarWaitStates; + case 8: + return SMFMA16x16ReadVgprVALUWarWaitStates; + case 16: + [[fallthrough]]; + default: + return SMFMA32x32ReadVgprVALUWarWaitStates; + } }; - MFMA = nullptr; - int WaitStatesSinceUse = getWaitStatesSince(IsSMFMAReadAsCFn, - MaxWarWaitStates); - if (!MFMA) - continue; - - unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA); - int NeedWaitStates = MaxWaitStates; - switch (HazardDefLatency) { - case 2: NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates; - break; - case 4: assert(ST.hasGFX940Insts()); - NeedWaitStates = GFX940_XDL4PassReadVgprVALUWarWaitStates; - break; - case 8: NeedWaitStates = SMFMA16x16ReadVgprVALUWarWaitStates; - break; - case 16: [[fallthrough]]; - default: NeedWaitStates = SMFMA32x32ReadVgprVALUWarWaitStates; - break; - } - - int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceUse; - WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); + WaitStatesNeeded = + std::max(WaitStatesNeeded, + getMaxWindowDeficit(MaxWarWaitStates, SMFMAReadAsCWindow)); } return WaitStatesNeeded; diff --git a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir index f62457aa5f3d4..340b366fc8802 100644 --- a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir +++ b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir @@ -1872,6 +1872,57 @@ body: | S_ENDPGM 0 ... +# A VALU overwrite takes the window of the producer on its own path: 16 passes +# need 19, 2 passes need 5, one branch away each, so 18. + +# GCN-LABEL: name: smfma_valu_write_mixed_shapes +# GCN: bb.3: +# GCN-NEXT: S_NOP 15 +# GCN-NEXT: S_NOP 1 +# GCN-NEXT: $vgpr0 = V_MOV_B32 +name: smfma_valu_write_mixed_shapes +body: | + bb.0: + successors: %bb.1, %bb.2 + S_CBRANCH_SCC1 %bb.2, implicit $scc + S_BRANCH %bb.1 + bb.1: + successors: %bb.3 + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.3 + bb.2: + successors: %bb.3 + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr20, $vgpr21, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.3 + bb.3: + $vgpr0 = V_MOV_B32_e32 1, implicit $exec + S_ENDPGM 0 +... +# The same for a register read as srcC: 15 after a 32x32 read it, 7 after a +# 16x16, one branch away each, so 14. + +# GCN-LABEL: name: smfma_srcc_read_valu_write_mixed_shapes +# GCN: bb.3: +# GCN-NEXT: S_NOP 13 +# GCN-NEXT: $vgpr0 = V_MOV_B32 +name: smfma_srcc_read_valu_write_mixed_shapes +body: | + bb.0: + successors: %bb.1, %bb.2 + S_CBRANCH_SCC1 %bb.2, implicit $scc + S_BRANCH %bb.1 + bb.1: + successors: %bb.3 + $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55 = V_MFMA_F32_32X32X2F32_vgprcd_e64 $vgpr26, $vgpr27, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.3 + bb.2: + successors: %bb.3 + $vgpr60_vgpr61_vgpr62_vgpr63_vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75 = V_MFMA_F32_16X16X1F32_vgprcd_e64 $vgpr20, $vgpr21, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.3 + bb.3: + $vgpr0 = V_MOV_B32_e32 1, implicit $exec + S_ENDPGM 0 +... # The empty fallthrough contributes nothing: along bb.0 -> bb.1 -> bb.2 only the # conditional branch executes, leaving 5 of the 11. diff --git a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir index f97df59ee1b52..f3d0e857b3777 100644 --- a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir +++ b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir @@ -2793,3 +2793,61 @@ body: | S_ENDPGM 0 ... +# A VALU overwrite takes the window of the producer on its path: 16-pass XDL +# needs NumPasses + 3, one more on gfx950, against NumPasses + 2. + +# GCN-LABEL: name: waw_mixed_xdl_and_smfma +# GCN: bb.3: +# GCN-NEXT: S_NOP 15 +# GFX942-NEXT: S_NOP 1 +# GFX950-NEXT: S_NOP 2 +# GCN-NEXT: $vgpr0 = V_MOV_B32 +name: waw_mixed_xdl_and_smfma +body: | + bb.0: + successors: %bb.1, %bb.2 + S_CBRANCH_SCC1 %bb.2, implicit $scc + S_BRANCH %bb.1 + + bb.1: + successors: %bb.3 + $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = V_MFMA_F32_32X32X4F16_vgprcd_e64 $vgpr34_vgpr35, $vgpr36_vgpr37, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.3 + + bb.2: + successors: %bb.3 + $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_4X4X1F32_vgprcd_e64 $vgpr38, $vgpr39, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.3 + + bb.3: + $vgpr0 = V_MOV_B32_e32 1, implicit $exec + S_ENDPGM 0 +... +# The same where the register was read as srcC: 15 after a 16-pass reader, 3 +# after a 4-pass one, so 14. The 4-pass arm is gfx940 and later only. + +# GCN-LABEL: name: war_mixed_xdl_shapes +# GCN: bb.3: +# GCN-NEXT: S_NOP 13 +# GCN-NEXT: $vgpr0 = V_MOV_B32 +name: war_mixed_xdl_shapes +body: | + bb.0: + successors: %bb.1, %bb.2 + S_CBRANCH_SCC1 %bb.2, implicit $scc + S_BRANCH %bb.1 + + bb.1: + successors: %bb.3 + $vgpr64_vgpr65_vgpr66_vgpr67_vgpr68_vgpr69_vgpr70_vgpr71_vgpr72_vgpr73_vgpr74_vgpr75_vgpr76_vgpr77_vgpr78_vgpr79_vgpr80_vgpr81_vgpr82_vgpr83_vgpr84_vgpr85_vgpr86_vgpr87_vgpr88_vgpr89_vgpr90_vgpr91_vgpr92_vgpr93_vgpr94_vgpr95 = V_MFMA_F32_32X32X4F16_vgprcd_e64 $vgpr34_vgpr35, $vgpr36_vgpr37, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.3 + + bb.2: + successors: %bb.3 + $vgpr96_vgpr97_vgpr98_vgpr99 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr34_vgpr35, $vgpr36_vgpr37, $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, 0, implicit $mode, implicit $exec + S_BRANCH %bb.3 + + bb.3: + $vgpr0 = V_MOV_B32_e32 1, implicit $exec + S_ENDPGM 0 +... >From 58aca82e97049b40bc94bc95d178f5404c258599 Mon Sep 17 00:00:00 2001 From: Michael Halkenhaeuser <[email protected]> Date: Mon, 28 Sep 2026 08:30:25 -0500 Subject: [PATCH 3/3] fixup! [AMDGPU] Measure MFMA read hazards at each producer Reword the comments of the two tests where an exact srcC write must not hide an earlier partial one: name the producers, give the nearer one's window, and keep the farther one's window apart from the padding it leaves. On gfx940+ the exact write now has a window of its own, since two different MFMAs sharing an accumulator need wait states. AI-assisted. --- llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir | 4 ++-- llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir index 340b366fc8802..f83cb5b9d940d 100644 --- a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir +++ b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx90a.mir @@ -1788,8 +1788,8 @@ body: | $agpr20_agpr21_agpr22_agpr23 = V_MFMA_F32_4X4X1F32_e64 $agpr1, $vgpr0, $agpr20_agpr21_agpr22_agpr23, 0, 0, 0, implicit $mode, implicit $exec S_ENDPGM 0 ... -# A producer writing exactly the srcC register needs nothing and must not hide a -# farther partial overlap: the 16x16 needs 8 from 1 away. +# The first 4x4 writes exactly the srcC register, with no window, but must not +# hide the 16x16 one instruction back: its window of 8 leaves 7 to pad. # GCN-LABEL: name: smfma_full_srcc_producer_does_not_hide_partial # GCN: V_MFMA_F32_4X4X1F32 diff --git a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir index f3d0e857b3777..cc272b5cc98ea 100644 --- a/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir +++ b/llvm/test/CodeGen/AMDGPU/mai-hazards-gfx942.mir @@ -2776,8 +2776,8 @@ body: | $agpr20_agpr21_agpr22_agpr23 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec S_ENDPGM 0 ... -# A 4-pass producer writing exactly the srcC register asks nothing and must not -# end the search: the 8-pass XDL result 2 away needs 9, 10 on gfx950. +# The 16x16 writes exactly the srcC register, with a window of 5 (6 on gfx950), +# but must not hide the 32x32 with a window of 9 (10) and leaves 8 (9) to pad. # GCN-LABEL: name: mfma_full_srcc_producer_does_not_hide_partial # GCN: V_MFMA_F32_16X16X16F16 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
