https://github.com/jrbyrnes updated https://github.com/llvm/llvm-project/pull/205626
>From c7221f445229a30f0ab02d1e5f4bc70242884341 Mon Sep 17 00:00:00 2001 From: Jeffrey Byrnes <[email protected]> Date: Tue, 23 Jun 2026 10:17:48 -0700 Subject: [PATCH 1/2] [AMDGPU] Add DSLatencyMode flag + attr to control LDS latency Change-Id: Ia5fee7983adffdb837ba2e876a64d805263b60c3 --- .../Target/AMDGPU/AMDGPUBarrierLatency.cpp | 5 +- .../AMDGPU/AMDGPUCoExecSchedStrategy.cpp | 3 +- .../lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 23 ++-- llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 3 +- llvm/lib/Target/AMDGPU/GCNSubtarget.cpp | 9 ++ llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 61 ++++++++- llvm/lib/Target/AMDGPU/SIInstrInfo.h | 20 +++ llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp | 4 +- .../coexec-sched-flavor-classification.mir | 2 +- llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll | 8 +- .../CodeGen/AMDGPU/ds-latency-mode-attr.mir | 61 +++++++++ .../AMDGPU/ds-latency-mode-branch-cost.mir | 53 ++++++++ .../ds-latency-mode-default-scheduler.mir | 119 ++++++++++++++++++ .../CodeGen/AMDGPU/ds-latency-mode-flag.mir | 29 +++++ 14 files changed, 373 insertions(+), 27 deletions(-) create mode 100644 llvm/test/CodeGen/AMDGPU/ds-latency-mode-attr.mir create mode 100644 llvm/test/CodeGen/AMDGPU/ds-latency-mode-branch-cost.mir create mode 100644 llvm/test/CodeGen/AMDGPU/ds-latency-mode-default-scheduler.mir create mode 100644 llvm/test/CodeGen/AMDGPU/ds-latency-mode-flag.mir diff --git a/llvm/lib/Target/AMDGPU/AMDGPUBarrierLatency.cpp b/llvm/lib/Target/AMDGPU/AMDGPUBarrierLatency.cpp index 18b4727a8605b..d5f6b73753b1b 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUBarrierLatency.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUBarrierLatency.cpp @@ -94,7 +94,6 @@ void BarrierLatency::apply(ScheduleDAGInstrs *DAG) { const unsigned BarrierSignalWaitLatency = BarrierSignalWaitLatencyOpt; SmallVector<SUnit *, 8> RegionTDM; SmallVector<SUnit *, 8> RegionAsync; - const TargetSchedModel *SchedModel = DAG->getSchedModel(); for (SUnit &SU : DAG->SUnits) { const MachineInstr *MI = SU.getInstr(); @@ -117,9 +116,9 @@ void BarrierLatency::apply(ScheduleDAGInstrs *DAG) { if (!MI->mayLoad() || MI->mayStore()) continue; + unsigned InstrLatency = TII->getInstrLatency(*MI); addLatencyToEdge(PredDep, SU, - SchedModel ? SchedModel->computeInstrLatency(MI, false) - : FenceLatency); + InstrLatency ? InstrLatency : FenceLatency); } } else if (Op == AMDGPU::S_BARRIER_WAIT) { for (SDep &PredDep : SU.Preds) { diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp index d29ee5084cc6c..0daa1e1d88693 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp @@ -183,7 +183,7 @@ unsigned CandidateHeuristics::getHWUICyclesForInst(SUnit *SU) { MachineInstr *MI = SU->getInstr(); if (MI->mayLoadOrStore()) - return SchedModel->computeInstrLatency(MI); + return SII->getInstrLatency(*MI); unsigned ReleaseAtCycle = 0; const MCSchedClassDesc *SC = DAG->getSchedClass(SU); @@ -714,6 +714,7 @@ ScheduleDAGInstrs * llvm::createGCNCoExecMachineScheduler(MachineSchedContext *C) { LLVM_DEBUG(dbgs() << "AMDGPU coexec preRA scheduler selected for " << C->MF->getName() << '\n'); + ScheduleDAGMILive *DAG = new GCNScheduleDAGMILive( C, std::make_unique<AMDGPUCoExecSchedStrategy>(C)); DAG->addMutation(createIGroupLPDAGMutation(AMDGPU::SchedulingPhase::Initial)); diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp index 722ddb5d0c4dd..7dc285a11ad1f 100644 --- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp +++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp @@ -2188,7 +2188,7 @@ static unsigned getWMMAHazardInstInCategory(const MachineInstr &MI, bool IsLowestRateWMMA = ST.hasGFX125xLowestRateWMMA(); unsigned Category = 0; - unsigned Latency = SchedModel.computeInstrLatency(&MI); + unsigned Latency = TII->getInstrLatency(MI); switch (Latency) { case 8: Category = IsSWMMAC ? 2 : 0; @@ -2665,8 +2665,7 @@ int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) const { Register DstReg = MI.getOperand(0).getReg(); if (DstReg == Reg) return false; - HazardDefLatency = - std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI)); + HazardDefLatency = std::max(HazardDefLatency, TII.getInstrLatency(MI)); return TRI.regsOverlap(DstReg, Reg); }; @@ -2742,8 +2741,7 @@ int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) const { if (!SIInstrInfo::isMFMA(MI)) return false; Register Reg = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg(); - HazardDefLatency = - std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI)); + HazardDefLatency = std::max(HazardDefLatency, TII.getInstrLatency(MI)); return TRI.regsOverlap(Reg, DstReg); }; @@ -2904,8 +2902,7 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) const { (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64)) NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates; - else if (ST.hasGFX940Insts() && - TSchedModel.computeInstrLatency(MI1) == 2) + else if (ST.hasGFX940Insts() && TII.getInstrLatency(*MI1) == 2) NeedWaitStates = GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates; } else { switch (Opc1) { @@ -2925,7 +2922,7 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) const { NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates; break; default: - int NumPasses = TSchedModel.computeInstrLatency(MI1); + int NumPasses = TII.getInstrLatency(*MI1); if (ST.hasGFX940Insts()) { if (TII.isXDL(*MI) && !TII.isXDL(*MI1)) break; @@ -2982,7 +2979,7 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) const { NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates; break; default: - int NumPasses = TSchedModel.computeInstrLatency(MI1); + int NumPasses = TII.getInstrLatency(*MI1); if (ST.hasGFX940Insts()) { NeedWaitStates = @@ -3262,7 +3259,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const { if (!MFMA) continue; - unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA); + unsigned HazardDefLatency = TII.getInstrLatency(*MFMA); int NumPasses = HazardDefLatency; int NeedWaitStates = MaxWaitStates; @@ -3357,7 +3354,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const { getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates); if (MFMA) { int NeedWaitStates = MaxWaitStates; - int NumPasses = TSchedModel.computeInstrLatency(MFMA); + int NumPasses = TII.getInstrLatency(*MFMA); if (SIInstrInfo::isDGEMM(MFMA->getOpcode())) { switch (NumPasses) { @@ -3424,7 +3421,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const { if (!MFMA) continue; - unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA); + unsigned HazardDefLatency = TII.getInstrLatency(*MFMA); int NeedWaitStates = MaxWaitStates; switch (HazardDefLatency) { case 2: NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates; @@ -3463,7 +3460,7 @@ bool GCNHazardRecognizer::ShouldPreferAnother(SUnit *SU) const { if (IsMFMAFn(*MI)) { int W = getWaitStatesSince(IsMFMAFn, 16); if (MAI) - return W < (int)TSchedModel.computeInstrLatency(MAI); + return W < (int)TII.getInstrLatency(*MAI); } return false; diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp index a4f854beaeebe..a59f6c643fca7 100644 --- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp +++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp @@ -1999,10 +1999,11 @@ GCNSchedStage::computeSUnitReadyCycle(const SUnit &SU, unsigned CurrCycle, DenseMap<unsigned, unsigned> &ReadyCycles, const TargetSchedModel &SM) { unsigned ReadyCycle = CurrCycle; + const SIInstrInfo *SII = static_cast<const SIInstrInfo *>(DAG.TII); for (auto &D : SU.Preds) { if (D.isAssignedRegDep()) { MachineInstr *DefMI = D.getSUnit()->getInstr(); - unsigned Latency = SM.computeInstrLatency(DefMI); + unsigned Latency = SII->getInstrLatency(*DefMI); unsigned DefReady = ReadyCycles[DAG.getSUnit(DefMI)->NodeNum]; ReadyCycle = std::max(ReadyCycle, DefReady + Latency); } diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp b/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp index 37efb3a51cb9d..c2267780a84fc 100644 --- a/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp +++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp @@ -786,6 +786,15 @@ void GCNSubtarget::adjustSchedDependency( return; // This is not a data dependency anymore. } + // DS load/store latency is variable depending on LDS contention. + if (InstrInfo.isDS(*DefI) && + InstrInfo.getDSLatencyMultiplier(*DefI->getMF()) != 1) { + // For LDS instructions, we have overrides to change default latencies. + unsigned Latency = InstrInfo.getInstrLatency(*DefI); + Dep.setLatency(Latency); + return; + } + if (DefI->isBundle()) { const SIRegisterInfo *TRI = getRegisterInfo(); auto Reg = Dep.getReg(); diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index 43cdaa34cf3e3..c4e0c0d467541 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -15,6 +15,7 @@ #include "AMDGPU.h" #include "AMDGPUInstrInfo.h" #include "AMDGPULaneMaskUtils.h" +#include "AMDGPUTargetMachine.h" #include "GCNHazardRecognizer.h" #include "GCNSubtarget.h" #include "SIMachineFunctionInfo.h" @@ -63,6 +64,17 @@ static cl::opt<bool> Fix16BitCopies( cl::init(true), cl::ReallyHidden); +static cl::opt<SIInstrInfo::DSLatencyMode> DSLatency( + "amdgpu-ds-latency-mode", cl::desc("LDS latency mode (LDS contention)"), + cl::values( + clEnumValN(SIInstrInfo::DSLatencyMode::Fast, "fast", + "Use default/pinned latency (no contention)"), + clEnumValN(SIInstrInfo::DSLatencyMode::Loaded, "loaded", + "Use loaded latency (moderate contention, 3x latency)"), + clEnumValN(SIInstrInfo::DSLatencyMode::Overloaded, "overloaded", + "Use overloaded latency (high contention, 5x latency)")), + cl::init(SIInstrInfo::DSLatencyMode::Fast), cl::Hidden); + SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST) : AMDGPUGenInstrInfo(ST, RI, AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN), @@ -10834,12 +10846,24 @@ unsigned SIInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, unsigned Lat = 0, Count = 0; for (++I; I != E && I->isBundledWithPred(); ++I) { ++Count; - Lat = std::max(Lat, SchedModel.computeInstrLatency(&*I)); + Lat = std::max(Lat, getInstrLatency(*I)); } return Lat + Count - 1; } - return SchedModel.computeInstrLatency(&MI); + return getInstrLatency(MI); +} + +unsigned SIInstrInfo::getInstrLatency(const MachineInstr &MI) const { + if (SchedModel.hasInstrSchedModel()) { + unsigned Latency = SchedModel.computeInstrLatency(&MI); + if (isDS(MI)) { + Latency *= getDSLatencyMultiplier(*MI.getMF()); + } + return Latency; + } + + return 0; } const MachineOperand & @@ -11528,3 +11552,36 @@ bool SIInstrInfo::isXDL(const MachineInstr &MI) const { return AMDGPU::getMAIIsGFX940XDL(Opcode); } + +unsigned SIInstrInfo::getDSLatencyMultiplier(const MachineFunction &MF) { + const Function &F = MF.getFunction(); + + // Priority selection goes to the attribute + Attribute A = F.getFnAttribute("amdgpu-ds-latency-mode"); + if (A.isValid()) { + StringRef Val = A.getValueAsString(); + if (Val == "fast") + return 1; + if (Val == "loaded") + return 3; + if (Val == "overloaded") + return 5; + } + + // If using coexec scheduler, default to "loaded" mode unless overridden + // by the command line option. + if (DSLatency.getNumOccurrences() == 0 && + AMDGPU::getSchedStrategy(F) == "coexec") + return 3; + + switch (DSLatency) { + case DSLatencyMode::Fast: + return 1; // Use default scheduling model latency + case DSLatencyMode::Loaded: + return 3; + case DSLatencyMode::Overloaded: + return 5; + } + + return 1; +} diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h index 1d67c8664ff44..2359a7622b106 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h @@ -1197,6 +1197,21 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo { return Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT; } + /// DS latency modes. The latency of DS load/store instructions + /// is variable depending on LDS contention. + enum class DSLatencyMode { + Fast, ///< Use default/pinned latency (no contention) + Loaded, ///< Use loaded latency (moderate contention, 3x latency) + Overloaded ///< Use overloaded latency (high contention, 5x latency) + }; + + /// \p returns the DS instruction latency multiplier based on the selected + /// DSLatencyMode. \p returns 1 if the default + /// scheduling model latency should be used (fast mode). + /// Checks the function attribute first, then if using coexec scheduler + /// defaults to "loaded", then falls back to the global command line option. + static unsigned getDSLatencyMultiplier(const MachineFunction &MF); + static unsigned getNonSoftWaitcntOpcode(unsigned Opcode) { switch (Opcode) { case AMDGPU::S_WAITCNT_soft: @@ -1738,10 +1753,15 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo { LiveIntervals *LIS = nullptr, VirtRegMap *VRM = nullptr) const override; + // Silence a hidden overloaded virtual function warning. + using TargetInstrInfo::getInstrLatency; + unsigned getInstrLatency(const InstrItineraryData *ItinData, const MachineInstr &MI, unsigned *PredCost = nullptr) const override; + unsigned getInstrLatency(const MachineInstr &MI) const; + const MachineOperand &getCalleeOperand(const MachineInstr &MI) const override; ValueUniformity getValueUniformity(const MachineInstr &MI) const final; diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp index a496c9a4daa71..19097118df3f0 100644 --- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp +++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp @@ -421,14 +421,14 @@ class BranchWeightCostModel { BranchProb = Head.getSuccProbability(FromIt); if (BranchProb.isUnknown()) BranchProb = BranchProbability::getZero(); - BranchTakenCost = SchedModel.computeInstrLatency(&Branch); + BranchTakenCost = TII.getInstrLatency(Branch); } bool isProfitable(const MachineInstr &MI) { if (TII.isWaitcnt(MI.getOpcode())) return false; - ThenCyclesCost += SchedModel.computeInstrLatency(&MI); + ThenCyclesCost += TII.getInstrLatency(MI); // Consider `P = N/D` to be the probability of execz being false (skipping // the then-block) The transformation is profitable if always executing the diff --git a/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir b/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir index 82dd1d8748675..497558cab6e42 100644 --- a/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir +++ b/llvm/test/CodeGen/AMDGPU/coexec-sched-flavor-classification.mir @@ -5,7 +5,7 @@ # CHECK-DAG: VALU(1c): 4 cycles, 4 instrs # CHECK-DAG: TRANS: 2 cycles, 2 instrs # CHECK-DAG: VMEM: 3200 cycles, 10 instrs -# CHECK-DAG: DS: 80 cycles, 4 instrs +# CHECK-DAG: DS: 240 cycles, 4 instrs # CHECK-DAG: DMA: 640 cycles, 2 instrs --- | diff --git a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll index b225d8f18805a..61bfd9f0db77e 100644 --- a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll +++ b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll @@ -311,8 +311,6 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace ; COEXEC-NEXT: ds_load_tr16_b128 v[112:115], v124 offset:640 ; COEXEC-NEXT: ds_load_tr16_b128 v[116:119], v124 offset:704 ; COEXEC-NEXT: ds_load_tr16_b128 v[120:123], v124 offset:896 -; COEXEC-NEXT: s_wait_dscnt 0x13 -; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31] ; COEXEC-NEXT: ds_load_tr16_b128 v[124:127], v124 offset:960 ; COEXEC-NEXT: ds_load_tr16_b128 v[128:131], v156 offset:128 ; COEXEC-NEXT: ds_load_tr16_b128 v[132:135], v156 offset:192 @@ -320,10 +318,12 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace ; COEXEC-NEXT: ds_load_tr16_b128 v[140:143], v156 offset:448 ; COEXEC-NEXT: ds_load_tr16_b128 v[144:147], v156 offset:640 ; COEXEC-NEXT: ds_load_tr16_b128 v[148:151], v156 offset:704 -; COEXEC-NEXT: s_wait_dscnt 0x16 -; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23] ; COEXEC-NEXT: ds_load_tr16_b128 v[152:155], v156 offset:896 ; COEXEC-NEXT: ds_load_tr16_b128 v[156:159], v156 offset:960 +; COEXEC-NEXT: s_wait_dscnt 0x1c +; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[24:31], v[32:39], v[40:47], v[24:31] +; COEXEC-NEXT: s_wait_dscnt 0x18 +; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[48:55], v[56:63], v[16:23] ; COEXEC-NEXT: s_wait_dscnt 0x14 ; COEXEC-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[64:71], v[72:79], v[8:15] ; COEXEC-NEXT: s_wait_dscnt 0x10 diff --git a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-attr.mir b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-attr.mir new file mode 100644 index 0000000000000..59bc0c0554c9c --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-attr.mir @@ -0,0 +1,61 @@ +# REQUIRES: asserts +# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -debug-only=machine-scheduler %s -filetype=null 2>&1 | FileCheck %s + + +# CHECK: Region: test_ds_latency_fast +# CHECK: DS: 80 cycles, 4 instrs + +# CHECK: Region: test_ds_latency_loaded +# CHECK: DS: 240 cycles, 4 instrs + +# CHECK: Region: test_ds_latency_overloaded +# CHECK: DS: 400 cycles, 4 instrs + +--- | + define void @test_ds_latency_fast() "amdgpu-waves-per-eu"="1,1" "amdgpu-ds-latency-mode"="fast" { ret void } + define void @test_ds_latency_loaded() "amdgpu-waves-per-eu"="1,1" "amdgpu-ds-latency-mode"="loaded" { ret void } + define void @test_ds_latency_overloaded() "amdgpu-waves-per-eu"="1,1" "amdgpu-ds-latency-mode"="overloaded" { ret void } + +... + +--- +name: test_ds_latency_fast +tracksRegLiveness: true +body: | + bb.0: + %0:vgpr_32 = IMPLICIT_DEF + %1:vreg_64_align2 = IMPLICIT_DEF + %2:vgpr_32 = DS_READ_B32 %0, 0, 0, implicit $m0, implicit $exec + %3:vreg_64_align2 = DS_READ_B64 %0, 0, 0, implicit $m0, implicit $exec + DS_WRITE_B32 %0, %2, 0, 0, implicit $m0, implicit $exec + DS_WRITE_B64 %0, %1, 0, 0, implicit $m0, implicit $exec + S_ENDPGM 0, implicit %2, implicit %3 +... + +--- +name: test_ds_latency_loaded +tracksRegLiveness: true +body: | + bb.0: + %0:vgpr_32 = IMPLICIT_DEF + %1:vreg_64_align2 = IMPLICIT_DEF + %2:vgpr_32 = DS_READ_B32 %0, 0, 0, implicit $m0, implicit $exec + %3:vreg_64_align2 = DS_READ_B64 %0, 0, 0, implicit $m0, implicit $exec + DS_WRITE_B32 %0, %2, 0, 0, implicit $m0, implicit $exec + DS_WRITE_B64 %0, %1, 0, 0, implicit $m0, implicit $exec + S_ENDPGM 0, implicit %2, implicit %3 +... + +--- +name: test_ds_latency_overloaded +tracksRegLiveness: true +body: | + bb.0: + %0:vgpr_32 = IMPLICIT_DEF + %1:vreg_64_align2 = IMPLICIT_DEF + %2:vgpr_32 = DS_READ_B32 %0, 0, 0, implicit $m0, implicit $exec + %3:vreg_64_align2 = DS_READ_B64 %0, 0, 0, implicit $m0, implicit $exec + DS_WRITE_B32 %0, %2, 0, 0, implicit $m0, implicit $exec + DS_WRITE_B64 %0, %1, 0, 0, implicit $m0, implicit $exec + S_ENDPGM 0, implicit %2, implicit %3 +... diff --git a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-branch-cost.mir b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-branch-cost.mir new file mode 100644 index 0000000000000..95bf9df77965a --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-branch-cost.mir @@ -0,0 +1,53 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6 +# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass si-pre-emit-peephole -amdgpu-ds-latency-mode=fast %s -o - | FileCheck -check-prefix=FAST %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass si-pre-emit-peephole -amdgpu-ds-latency-mode=overloaded %s -o - | FileCheck -check-prefix=OVERLOADED %s + +# Test that ds-latency-mode affects the branch-over-removal cost model in +# SIPreEmitPeephole. With higher DS latency, the cost of the then-block +# is higher, making branch removal less profitable. + +--- +name: skip_execz_ds_multi +body: | + ; FAST-LABEL: name: skip_execz_ds_multi + ; FAST: bb.0: + ; FAST-NEXT: successors: %bb.1(0x78000000) + ; FAST-NEXT: {{ $}} + ; FAST-NEXT: bb.1: + ; FAST-NEXT: successors: %bb.2(0x80000000) + ; FAST-NEXT: {{ $}} + ; FAST-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec + ; FAST-NEXT: DS_WRITE_B32 $vgpr0, $vgpr0, 0, 0, implicit $m0, implicit $exec + ; FAST-NEXT: DS_WRITE_B32 $vgpr0, $vgpr0, 4, 0, implicit $m0, implicit $exec + ; FAST-NEXT: {{ $}} + ; FAST-NEXT: bb.2: + ; FAST-NEXT: S_ENDPGM 0 + ; + ; OVERLOADED-LABEL: name: skip_execz_ds_multi + ; OVERLOADED: bb.0: + ; OVERLOADED-NEXT: successors: %bb.1(0x78000000), %bb.2(0x08000000) + ; OVERLOADED-NEXT: {{ $}} + ; OVERLOADED-NEXT: S_CBRANCH_EXECZ %bb.2, implicit $exec + ; OVERLOADED-NEXT: {{ $}} + ; OVERLOADED-NEXT: bb.1: + ; OVERLOADED-NEXT: successors: %bb.2(0x80000000) + ; OVERLOADED-NEXT: {{ $}} + ; OVERLOADED-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec + ; OVERLOADED-NEXT: DS_WRITE_B32 $vgpr0, $vgpr0, 0, 0, implicit $m0, implicit $exec + ; OVERLOADED-NEXT: DS_WRITE_B32 $vgpr0, $vgpr0, 4, 0, implicit $m0, implicit $exec + ; OVERLOADED-NEXT: {{ $}} + ; OVERLOADED-NEXT: bb.2: + ; OVERLOADED-NEXT: S_ENDPGM 0 + bb.0: + successors: %bb.1(0x78000000), %bb.2(0x08000000) + S_CBRANCH_EXECZ %bb.2, implicit $exec + + bb.1: + successors: %bb.2 + $vgpr0 = V_MOV_B32_e32 0, implicit $exec + DS_WRITE_B32 $vgpr0, $vgpr0, 0, 0, implicit $m0, implicit $exec + DS_WRITE_B32 $vgpr0, $vgpr0, 4, 0, implicit $m0, implicit $exec + + bb.2: + S_ENDPGM 0 +... diff --git a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-default-scheduler.mir b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-default-scheduler.mir new file mode 100644 index 0000000000000..41695b863b00a --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-default-scheduler.mir @@ -0,0 +1,119 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6 +# REQUIRES: asserts +# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass=machine-scheduler -amdgpu-ds-latency-mode=fast %s -o - | FileCheck -check-prefix=FAST %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass=machine-scheduler -amdgpu-ds-latency-mode=overloaded %s -o - | FileCheck -check-prefix=OVERLOADED %s + +--- +name: test_ds_latency_default_sched +tracksRegLiveness: true +body: | + bb.0: + ; FAST-LABEL: name: test_ds_latency_default_sched + ; FAST: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF + ; FAST-NEXT: $m0 = S_MOV_B32 -1 + ; FAST-NEXT: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[DEF]], 0, 0, implicit $m0, implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: [[V_ADD_U32_e32_4:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DS_READ_B32_]], [[DS_READ_B32_]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_5:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_6:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_7:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_8:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_9:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_10:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_11:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_12:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_13:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_14:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_15:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_16:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_17:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_18:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_19:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_20:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_21:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_22:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_23:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_24:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_25:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_26:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_27:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_28:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_29:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: dead [[V_ADD_U32_e32_30:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; FAST-NEXT: S_ENDPGM 0, implicit [[V_ADD_U32_e32_4]] + ; + ; OVERLOADED-LABEL: name: test_ds_latency_default_sched + ; OVERLOADED: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF + ; OVERLOADED-NEXT: $m0 = S_MOV_B32 -1 + ; OVERLOADED-NEXT: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[DEF]], 0, 0, implicit $m0, implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_4:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_5:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_6:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_7:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_8:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_9:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_10:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_11:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_12:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_13:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_14:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_15:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_16:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_17:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_18:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_19:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_20:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_21:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_22:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_23:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: [[V_ADD_U32_e32_24:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DS_READ_B32_]], [[DS_READ_B32_]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_25:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_26:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_27:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_28:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_29:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: dead [[V_ADD_U32_e32_30:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF]], [[DEF]], implicit $exec + ; OVERLOADED-NEXT: S_ENDPGM 0, implicit [[V_ADD_U32_e32_24]] + %0:vgpr_32 = IMPLICIT_DEF + $m0 = S_MOV_B32 -1 + %1:vgpr_32 = DS_READ_B32 %0, 0, 0, implicit $m0, implicit $exec + %2:vgpr_32 = V_ADD_U32_e32 %1, %1, implicit $exec + %3:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %4:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %5:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %6:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %7:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %8:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %9:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %10:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %11:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %12:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %13:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %14:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %15:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %16:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %17:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %18:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %19:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %20:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %21:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %22:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %23:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %24:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %25:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %26:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %27:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %28:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %29:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %30:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %31:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + %32:vgpr_32 = V_ADD_U32_e32 %0, %0, implicit $exec + S_ENDPGM 0, implicit %2 +... diff --git a/llvm/test/CodeGen/AMDGPU/ds-latency-mode-flag.mir b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-flag.mir new file mode 100644 index 0000000000000..527390abcc8f5 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/ds-latency-mode-flag.mir @@ -0,0 +1,29 @@ +# REQUIRES: asserts +# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -debug-only=machine-scheduler -amdgpu-ds-latency-mode=fast %s -filetype=null 2>&1 | FileCheck -check-prefix=FAST %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -debug-only=machine-scheduler -amdgpu-ds-latency-mode=loaded %s -filetype=null 2>&1 | FileCheck -check-prefix=LOADED %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -debug-only=machine-scheduler -amdgpu-ds-latency-mode=overloaded %s -filetype=null 2>&1 | FileCheck -check-prefix=OVERLOADED %s + +# Test that the -amdgpu-ds-latency-mode command line flag affects DS cycle +# accounting in the coexec scheduler. +# DS instructions have base latency of 20 cycles each (4 instructions = 80 cycles). +# - fast: 4 * 20 * 1 = 80 cycles +# - loaded: 4 * 20 * 3 = 240 cycles +# - overloaded: 4 * 20 * 5 = 400 cycles + +# FAST: DS: 80 cycles, 4 instrs +# LOADED: DS: 240 cycles, 4 instrs +# OVERLOADED: DS: 400 cycles, 4 instrs + +--- +name: test_ds_latency_flag +tracksRegLiveness: true +body: | + bb.0: + %0:vgpr_32 = IMPLICIT_DEF + %1:vreg_64_align2 = IMPLICIT_DEF + %2:vgpr_32 = DS_READ_B32 %0, 0, 0, implicit $m0, implicit $exec + %3:vreg_64_align2 = DS_READ_B64 %0, 0, 0, implicit $m0, implicit $exec + DS_WRITE_B32 %0, %2, 0, 0, implicit $m0, implicit $exec + DS_WRITE_B64 %0, %1, 0, 0, implicit $m0, implicit $exec + S_ENDPGM 0, implicit %2, implicit %3 +... >From 3f913cfe6ee32823b1eddfffbfe904382468561c Mon Sep 17 00:00:00 2001 From: Jeffrey Byrnes <[email protected]> Date: Tue, 7 Jul 2026 18:03:38 -0700 Subject: [PATCH 2/2] More documentation Change-Id: I65ff1a1b605796d21a52766dcf04c64bc64aaa4e --- llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 2 +- llvm/lib/Target/AMDGPU/SIInstrInfo.h | 27 ++++++++++++++++++++------ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index c4e0c0d467541..18101bb70abb5 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -68,7 +68,7 @@ static cl::opt<SIInstrInfo::DSLatencyMode> DSLatency( "amdgpu-ds-latency-mode", cl::desc("LDS latency mode (LDS contention)"), cl::values( clEnumValN(SIInstrInfo::DSLatencyMode::Fast, "fast", - "Use default/pinned latency (no contention)"), + "Use default/pinged latency (no contention)"), clEnumValN(SIInstrInfo::DSLatencyMode::Loaded, "loaded", "Use loaded latency (moderate contention, 3x latency)"), clEnumValN(SIInstrInfo::DSLatencyMode::Overloaded, "overloaded", diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h index 2359a7622b106..a513ecf8652a7 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h @@ -1197,17 +1197,29 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo { return Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT; } - /// DS latency modes. The latency of DS load/store instructions - /// is variable depending on LDS contention. + /// DS latency modes. The latency of an individual DS load/store instruction + /// is variable. Some of the major sources that cause this variation are hard + /// to model at compile time. For example, if we have multiple LDS + /// instructions in flight, one of these may take longer than the other due to + /// a bank conflict. Given that these bank conflicts can occur across waves, + /// it is hard to accurately model this. This is compounded given the presence + /// of the LDS FIFO -- if an earlier LDS instruction has, for example, a bank + /// conflict, this will impact the latency of the current LDS instruction. + /// + /// Given these complexities, we offer different DSLatencyModes for kernels to + /// express the average latency for LDS instructions in the kernel. We expect + /// that for kernels with many closely packed LDS isntructions, the average + /// latency for LDS instructions will be relatively high. Thus we should use + /// DSLatencyMode::Loaded or DSLatencyMode::Overloaded. enum class DSLatencyMode { - Fast, ///< Use default/pinned latency (no contention) - Loaded, ///< Use loaded latency (moderate contention, 3x latency) - Overloaded ///< Use overloaded latency (high contention, 5x latency) + Fast, // Use default/pinged latency (no contention) + Loaded, // Use loaded latency (moderate contention, 3x latency) + Overloaded // Use overloaded latency (high contention, 5x latency) }; /// \p returns the DS instruction latency multiplier based on the selected /// DSLatencyMode. \p returns 1 if the default - /// scheduling model latency should be used (fast mode). + /// scheduling model latency should be used (pinged mode). /// Checks the function attribute first, then if using coexec scheduler /// defaults to "loaded", then falls back to the global command line option. static unsigned getDSLatencyMultiplier(const MachineFunction &MF); @@ -1760,6 +1772,9 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo { const MachineInstr &MI, unsigned *PredCost = nullptr) const override; + /// \returns the latency of a given instruction \p MI. This implements custom + /// overrides for certain cases (e.g. when using dfifferent DSLatencyModes to + /// express increased LDS resource contention). unsigned getInstrLatency(const MachineInstr &MI) const; const MachineOperand &getCalleeOperand(const MachineInstr &MI) const override; _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
