https://github.com/mssefat updated https://github.com/llvm/llvm-project/pull/218074
>From 04a57042c2034b195a86614b938bd3126c13f578 Mon Sep 17 00:00:00 2001 From: mssefat <[email protected]> Date: Wed, 19 Aug 2026 20:02:07 -0500 Subject: [PATCH 1/2] [AMDGPU] Apply occupancy-aware register allocation anti-hints --- llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 131 +++++- llvm/lib/Target/AMDGPU/SIRegisterInfo.h | 16 + .../AMDGPU/llvm.amdgcn.mfma.anti-hints.mir | 373 ++++++++++++++++++ 3 files changed, 519 insertions(+), 1 deletion(-) create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp index 971dedd0878f1..97fca7ca20a26 100644 --- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp @@ -11,18 +11,23 @@ // //===----------------------------------------------------------------------===// +#include "SIRegisterInfo.h" #include "AMDGPU.h" #include "AMDGPURegisterBankInfo.h" #include "GCNSubtarget.h" #include "MCTargetDesc/AMDGPUInstPrinter.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "SIMachineFunctionInfo.h" -#include "SIRegisterInfo.h" #include "llvm/CodeGen/LiveIntervals.h" #include "llvm/CodeGen/LiveRegUnits.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/RegisterScavenging.h" +#include "llvm/MC/MCRegister.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" + +#define DEBUG_TYPE "amdgpu-si-register-info" using namespace llvm; @@ -4173,6 +4178,130 @@ bool SIRegisterInfo::getRegAllocationHints(Register VirtReg, } } +bool SIRegisterInfo::shouldApplyAntiHints( + const MachineFunction &MF, unsigned NumAllocatedVGPRs, + unsigned &MaxVGPRsForCurrentOccupancy) const { + + const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); + unsigned DynamicVGPRBlockSize = MFI->getDynamicVGPRBlockSize(); + unsigned TargetOccupancy = MFI->getOccupancy(); + unsigned CurrentOccupancy = + ST.getOccupancyWithNumVGPRs(NumAllocatedVGPRs, DynamicVGPRBlockSize); + MaxVGPRsForCurrentOccupancy = + ST.getMaxNumVGPRs(CurrentOccupancy, DynamicVGPRBlockSize); + + LLVM_DEBUG(dbgs() << "anti-hints: " << NumAllocatedVGPRs + << " VGPRs allocated, target occupancy " << TargetOccupancy + << ", current occupancy " << CurrentOccupancy << '\n'); + + // If we are already at lowest occupancy, then there is no need to protect + // against occupancy regression. + if (CurrentOccupancy == 1) + return true; + + // Set max VGPRs for target and current occupancy to early bail out if we are + // close to the limit. + unsigned MaxVGPRsCutOffForTargetOccupancy = + (ST.getMaxNumVGPRs(TargetOccupancy, DynamicVGPRBlockSize) * 80) / 100; + unsigned MaxVGPRsCutOffForCurrentOccupancy = + (MaxVGPRsForCurrentOccupancy * 95) / 100; + + if (NumAllocatedVGPRs >= MaxVGPRsCutOffForTargetOccupancy) { + LLVM_DEBUG(dbgs() << "anti-hints: not applied, at or above the " + << MaxVGPRsCutOffForTargetOccupancy + << " VGPR cutoff for target occupancy\n"); + return false; + } + + if (NumAllocatedVGPRs >= MaxVGPRsCutOffForCurrentOccupancy) { + LLVM_DEBUG(dbgs() << "anti-hints: not applied, at or above the " + << MaxVGPRsCutOffForCurrentOccupancy + << " VGPR cutoff for current occupancy\n"); + return false; + } + + return true; +} + +void SIRegisterInfo::applyRegAllocationAntiHints( + Register VirtReg, ArrayRef<MCPhysReg> &Order, + SmallVectorImpl<MCPhysReg> &OrderStorage, + SmallVectorImpl<MCPhysReg> &AntiHints, const MachineFunction &MF, + const VirtRegMap *VRM, const LiveRegMatrix *Matrix) const { + + // Early exit to default order if we have no anti-hints or no VRM. + if (AntiHints.empty() || !VRM) + return; + + // Get total number of allocated VGPRs to determine the current occupancy. + unsigned NumVGPRs = 0; + unsigned NumAGPRs = 0; + if (Matrix) { + for (MCPhysReg Reg : AMDGPU::VGPR_32RegClass) + if (Matrix->isPhysRegUsed(Reg)) + NumVGPRs = std::max(NumVGPRs, getHWRegIndex(Reg) + 1); + for (MCPhysReg Reg : AMDGPU::AGPR_32RegClass) + if (Matrix->isPhysRegUsed(Reg)) + NumAGPRs = std::max(NumAGPRs, getHWRegIndex(Reg) + 1); + } + unsigned NumAllocatedVGPRs = + AMDGPU::getTotalNumVGPRs(ST.hasGFX90AInsts(), NumAGPRs, NumVGPRs); + + // Early exit if we should not apply anti-hints. + unsigned MaxVGPRsForCurrentOccupancy = 0; + if (!shouldApplyAntiHints(MF, NumAllocatedVGPRs, MaxVGPRsForCurrentOccupancy)) + return; + + // Returns true if Reg fits within the current occupancy VGPR budget. + auto IsWithinBudget = [&](MCPhysReg Reg) -> bool { + const TargetRegisterClass *RC = getPhysRegBaseClass(Reg); + + // No VGPR or AGPR usage. + if (!RC || + (!isVGPRClass(RC) && !isAGPRClass(RC) && !isVectorSuperClass(RC))) + return true; + + unsigned NumRegs = divideCeil(getRegSizeInBits(*RC), 32); + unsigned Highest = getHWRegIndex(Reg) + NumRegs - 1; + return Highest < MaxVGPRsForCurrentOccupancy; + }; + + // Copy order. + OrderStorage.clear(); + OrderStorage.assign(Order.begin(), Order.end()); + + // Helper to check if a register overlaps with any anti-hint. + auto isAntiHinted = [&](MCPhysReg Reg) { + return llvm::any_of(AntiHints, [&](MCPhysReg AntiHint) { + return regsOverlap(Reg, AntiHint); + }); + }; + + // Find the cutoff point for the current occupancy VGPR budget. + auto *BeyondBudgetStart = llvm::find_if( + OrderStorage, [&](MCPhysReg Reg) { return !IsWithinBudget(Reg); }); + + // Only shuffle within the current occupancy VGPR budget. + auto *PartitionPoint = + std::stable_partition(OrderStorage.begin(), BeyondBudgetStart, + [&](MCPhysReg Reg) { return !isAntiHinted(Reg); }); + + Order = OrderStorage; + LLVM_DEBUG({ + size_t NonAntiHintedCount = + std::distance(OrderStorage.begin(), PartitionPoint); + size_t AntiHintedCount = std::distance(PartitionPoint, BeyondBudgetStart); + size_t BeyondBudgetCount = + std::distance(BeyondBudgetStart, OrderStorage.end()); + dbgs() << "Added " << NonAntiHintedCount + << " non-anti-hinted registers first\n" + << "Added " << AntiHintedCount + << " anti-hinted registers at the end\n" + << "Beyond current occupancy budget, left: " << BeyondBudgetCount + << '\n'; + }); +} + MCRegister SIRegisterInfo::getReturnAddressReg(const MachineFunction &MF) const { // Not a callee saved register. return AMDGPU::SGPR30_SGPR31; diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h index e464c9334ffea..61fbf90d84488 100644 --- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h +++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h @@ -15,6 +15,11 @@ #define LLVM_LIB_TARGET_AMDGPU_SIREGISTERINFO_H #include "llvm/ADT/BitVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/LiveRegMatrix.h" +#include "llvm/CodeGen/Register.h" +#include "llvm/CodeGen/VirtRegMap.h" +#include "llvm/MC/MCRegister.h" #define GET_REGINFO_HEADER #include "AMDGPUGenRegisterInfo.inc" @@ -364,6 +369,17 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo { const MachineFunction &MF, const VirtRegMap *VRM, const LiveRegMatrix *Matrix) const override; + bool shouldApplyAntiHints(const MachineFunction &MF, + unsigned NumAllocatedVGPRs, + unsigned &MaxVGPRsForCurrentOccupancy) const; + + void applyRegAllocationAntiHints( + Register VirtReg, ArrayRef<MCPhysReg> &Order, + SmallVectorImpl<MCPhysReg> &OrderStorage, + SmallVectorImpl<MCPhysReg> &AntiHints, const MachineFunction &MF, + const VirtRegMap *VRM = nullptr, + const LiveRegMatrix *Matrix = nullptr) const override; + const int *getRegUnitPressureSets(MCRegUnit RegUnit) const override; MCRegister getReturnAddressReg(const MachineFunction &MF) const; diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir new file mode 100644 index 0000000000000..5d698f533acf2 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.anti-hints.mir @@ -0,0 +1,373 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6 +# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=greedy,machineverifier,virtregrewriter,post-RA-hazard-rec %s -o - | FileCheck %s + +--- | + define amdgpu_kernel void @anti_hints_present() #0 { ret void } + define amdgpu_kernel void @anti_hints_absent() #0 { ret void } + define amdgpu_kernel void @anti_hints_suppressed_by_target_budget() #0 { ret void } + define amdgpu_kernel void @anti_hints_suppressed_by_current_budget() #0 { ret void } + attributes #0 = { nounwind "amdgpu-agpr-alloc"="0" "frame-pointer"="none"} +... + +--- +name: anti_hints_present +tracksRegLiveness: true +machineFunctionInfo: + isEntryFunction: true + occupancy: 8 +registers: + - { id: 0, class: vreg_128_align2 } + - { id: 1, class: vgpr_32 } + - { id: 2, class: vgpr_32 } + - { id: 3, class: sgpr_128 } + - { id: 4, class: vgpr_32 } + - { id: 5, class: vgpr_32 } + - { id: 6, class: vreg_128_align2 } + - { id: 7, class: vreg_512_align2 } + - { id: 8, class: vreg_512_align2 } + - { id: 9, class: vgpr_32, anti-hints: [ '%7' ] } + - { id: 10, class: vgpr_32, anti-hints: [ '%7' ] } + - { id: 11, class: vreg_128_align2, anti-hints: [ '%7' ] } + - { id: 12, class: vgpr_32, anti-hints: [ '%7' ] } + - { id: 13, class: vreg_512_align2 } + - { id: 14, class: vgpr_32, anti-hints: [ '%8' ] } + - { id: 15, class: vreg_128_align2, anti-hints: [ '%8' ] } + - { id: 16, class: vgpr_32, anti-hints: [ '%8' ] } + - { id: 17, class: vreg_512_align2 } + - { id: 18, class: vreg_128_align2, anti-hints: [ '%8', '%13' ] } + - { id: 19, class: vreg_128_align2, anti-hints: [ '%13' ] } +liveins: + - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' } + - { reg: '$vgpr4' } + - { reg: '$vgpr5' } +body: | + bb.0: + liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $vgpr5 + ; CHECK-LABEL: name: anti_hints_present + ; CHECK: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec + ; CHECK-NEXT: renamable $vgpr46 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec + ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8) + ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF + ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: renamable $vgpr47 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec + ; CHECK-NEXT: renamable $vgpr48 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec + ; CHECK-NEXT: renamable $vgpr42_vgpr43_vgpr44_vgpr45 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3) + ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr42_vgpr43_vgpr44_vgpr45, 4096, 0, implicit $exec :: (store (s128), addrspace 3) + ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr46, implicit $exec + ; CHECK-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec + ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3) + ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr38_vgpr39_vgpr40_vgpr41, 6144, 0, implicit $exec :: (store (s128), addrspace 3) + ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec + ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3) + ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr38_vgpr39_vgpr40_vgpr41, 8192, 0, implicit $exec :: (store (s128), addrspace 3) + ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr42, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8) + ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr46, implicit killed renamable $vgpr47, implicit killed renamable $vgpr48, implicit killed renamable $vgpr43, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr38_vgpr39_vgpr40_vgpr41 + %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3 + %1:vgpr_32 = COPY $vgpr4 + %2:vgpr_32 = COPY $vgpr5 + %3:sgpr_128 = IMPLICIT_DEF + %4:vgpr_32 = V_ADD_U32_e32 4096, %2, implicit $exec + %5:vgpr_32 = V_LSHLREV_B32_e32 2, %2, implicit $exec + %6:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8) + %7:vreg_512_align2 = IMPLICIT_DEF + %8:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %6.sub0_sub1, %7, 0, 0, 0, implicit $mode, implicit $exec + %9:vgpr_32 = V_LSHLREV_B32_e32 2, %4, implicit $exec + %10:vgpr_32 = V_LSHLREV_B32_e32 4, %2, implicit $exec + %11:vreg_128_align2 = DS_READ_B128_gfx9 %1, 0, 0, implicit $exec :: (load (s128), addrspace 3) + DS_WRITE_B128_gfx9 %1, %11, 4096, 0, implicit $exec :: (store (s128), addrspace 3) + %12:vgpr_32 = V_ADD_U32_e32 %2, %5, implicit $exec + %13:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub2_sub3, %6.sub2_sub3, %8, 0, 0, 0, implicit $mode, implicit $exec + %14:vgpr_32 = V_LSHLREV_B32_e32 2, %12, implicit $exec + %15:vreg_128_align2 = DS_READ_B128_gfx9 %1, 2048, 0, implicit $exec :: (load (s128), addrspace 3) + DS_WRITE_B128_gfx9 %1, %15, 6144, 0, implicit $exec :: (store (s128), addrspace 3) + %16:vgpr_32 = V_ADD_U32_e32 8192, %12, implicit $exec + %17:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %13, 0, 0, 0, implicit $mode, implicit $exec + %18:vreg_128_align2 = DS_READ_B128_gfx9 %1, 4096, 0, implicit $exec :: (load (s128), addrspace 3) + DS_WRITE_B128_gfx9 %1, %18, 8192, 0, implicit $exec :: (store (s128), addrspace 3) + %19:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %16, %3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8) + S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %5, implicit %9, implicit %10, implicit %14, implicit %17, implicit %19 +... + +--- +name: anti_hints_absent +tracksRegLiveness: true +machineFunctionInfo: + isEntryFunction: true + occupancy: 8 +registers: + - { id: 0, class: vreg_128_align2 } + - { id: 1, class: vgpr_32 } + - { id: 2, class: vgpr_32 } + - { id: 3, class: sgpr_128 } + - { id: 4, class: vgpr_32 } + - { id: 5, class: vgpr_32 } + - { id: 6, class: vreg_128_align2 } + - { id: 7, class: vreg_512_align2 } + - { id: 8, class: vreg_512_align2 } + - { id: 9, class: vgpr_32 } + - { id: 10, class: vgpr_32 } + - { id: 11, class: vreg_128_align2 } + - { id: 12, class: vgpr_32 } + - { id: 13, class: vreg_512_align2 } + - { id: 14, class: vgpr_32 } + - { id: 15, class: vreg_128_align2 } + - { id: 16, class: vgpr_32 } + - { id: 17, class: vreg_512_align2 } + - { id: 18, class: vreg_128_align2 } + - { id: 19, class: vreg_128_align2 } +liveins: + - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' } + - { reg: '$vgpr4' } + - { reg: '$vgpr5' } +body: | + bb.0: + liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $vgpr5 + ; CHECK-LABEL: name: anti_hints_absent + ; CHECK: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec + ; CHECK-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec + ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8) + ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF + ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec + ; CHECK-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec + ; CHECK-NEXT: S_NOP 4 + ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3) + ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3) + ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec + ; CHECK-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec + ; CHECK-NEXT: S_NOP 5 + ; CHECK-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3) + ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3) + ; CHECK-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec + ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: S_NOP 6 + ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3) + ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3) + ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8) + ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25 + %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3 + %1:vgpr_32 = COPY $vgpr4 + %2:vgpr_32 = COPY $vgpr5 + %3:sgpr_128 = IMPLICIT_DEF + %4:vgpr_32 = V_ADD_U32_e32 4096, %2, implicit $exec + %5:vgpr_32 = V_LSHLREV_B32_e32 2, %2, implicit $exec + %6:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8) + %7:vreg_512_align2 = IMPLICIT_DEF + %8:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %6.sub0_sub1, %7, 0, 0, 0, implicit $mode, implicit $exec + %9:vgpr_32 = V_LSHLREV_B32_e32 2, %4, implicit $exec + %10:vgpr_32 = V_LSHLREV_B32_e32 4, %2, implicit $exec + %11:vreg_128_align2 = DS_READ_B128_gfx9 %1, 0, 0, implicit $exec :: (load (s128), addrspace 3) + DS_WRITE_B128_gfx9 %1, %11, 4096, 0, implicit $exec :: (store (s128), addrspace 3) + %12:vgpr_32 = V_ADD_U32_e32 %2, %5, implicit $exec + %13:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub2_sub3, %6.sub2_sub3, %8, 0, 0, 0, implicit $mode, implicit $exec + %14:vgpr_32 = V_LSHLREV_B32_e32 2, %12, implicit $exec + %15:vreg_128_align2 = DS_READ_B128_gfx9 %1, 2048, 0, implicit $exec :: (load (s128), addrspace 3) + DS_WRITE_B128_gfx9 %1, %15, 6144, 0, implicit $exec :: (store (s128), addrspace 3) + %16:vgpr_32 = V_ADD_U32_e32 8192, %12, implicit $exec + %17:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %13, 0, 0, 0, implicit $mode, implicit $exec + %18:vreg_128_align2 = DS_READ_B128_gfx9 %1, 4096, 0, implicit $exec :: (load (s128), addrspace 3) + DS_WRITE_B128_gfx9 %1, %18, 8192, 0, implicit $exec :: (store (s128), addrspace 3) + %19:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %16, %3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8) + S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %5, implicit %9, implicit %10, implicit %14, implicit %17, implicit %19 +... + +--- +name: anti_hints_suppressed_by_target_budget +tracksRegLiveness: true +machineFunctionInfo: + isEntryFunction: true + occupancy: 8 +registers: + - { id: 0, class: vreg_128_align2 } + - { id: 1, class: vgpr_32 } + - { id: 2, class: vgpr_32 } + - { id: 3, class: sgpr_128 } + - { id: 4, class: vgpr_32 } + - { id: 5, class: vgpr_32 } + - { id: 6, class: vreg_128_align2 } + - { id: 7, class: vreg_512_align2 } + - { id: 8, class: vreg_512_align2 } + - { id: 9, class: vgpr_32, anti-hints: [ '%7' ] } + - { id: 10, class: vgpr_32, anti-hints: [ '%7' ] } + - { id: 11, class: vreg_128_align2, anti-hints: [ '%7' ] } + - { id: 12, class: vgpr_32, anti-hints: [ '%7' ] } + - { id: 13, class: vreg_512_align2 } + - { id: 14, class: vgpr_32, anti-hints: [ '%8' ] } + - { id: 15, class: vreg_128_align2, anti-hints: [ '%8' ] } + - { id: 16, class: vgpr_32, anti-hints: [ '%8' ] } + - { id: 17, class: vreg_512_align2 } + - { id: 18, class: vreg_128_align2, anti-hints: [ '%8', '%13' ] } + - { id: 19, class: vreg_128_align2, anti-hints: [ '%13' ] } + - { id: 20, class: vreg_128_align2 } + - { id: 21, class: vreg_128_align2 } + - { id: 22, class: vreg_128_align2 } +liveins: + - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' } + - { reg: '$vgpr4' } + - { reg: '$vgpr5' } + - { reg: '$vgpr52_vgpr53_vgpr54_vgpr55' } + - { reg: '$vgpr56_vgpr57_vgpr58_vgpr59' } + - { reg: '$vgpr60_vgpr61_vgpr62_vgpr63' } +body: | + bb.0: + liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $vgpr5, $vgpr52_vgpr53_vgpr54_vgpr55, $vgpr56_vgpr57_vgpr58_vgpr59, $vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-LABEL: name: anti_hints_suppressed_by_target_budget + ; CHECK: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr52_vgpr53_vgpr54_vgpr55, $vgpr56_vgpr57_vgpr58_vgpr59, $vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec + ; CHECK-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec + ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8) + ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF + ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec + ; CHECK-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec + ; CHECK-NEXT: S_NOP 4 + ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3) + ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3) + ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec + ; CHECK-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec + ; CHECK-NEXT: S_NOP 5 + ; CHECK-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3) + ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3) + ; CHECK-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec + ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: S_NOP 6 + ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3) + ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3) + ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8) + ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, implicit killed renamable $vgpr52_vgpr53_vgpr54_vgpr55, implicit killed renamable $vgpr56_vgpr57_vgpr58_vgpr59, implicit killed renamable $vgpr60_vgpr61_vgpr62_vgpr63 + %20:vreg_128_align2 = COPY $vgpr52_vgpr53_vgpr54_vgpr55 + %21:vreg_128_align2 = COPY $vgpr56_vgpr57_vgpr58_vgpr59 + %22:vreg_128_align2 = COPY $vgpr60_vgpr61_vgpr62_vgpr63 + %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3 + %1:vgpr_32 = COPY $vgpr4 + %2:vgpr_32 = COPY $vgpr5 + %3:sgpr_128 = IMPLICIT_DEF + %4:vgpr_32 = V_ADD_U32_e32 4096, %2, implicit $exec + %5:vgpr_32 = V_LSHLREV_B32_e32 2, %2, implicit $exec + %6:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8) + %7:vreg_512_align2 = IMPLICIT_DEF + %8:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %6.sub0_sub1, %7, 0, 0, 0, implicit $mode, implicit $exec + %9:vgpr_32 = V_LSHLREV_B32_e32 2, %4, implicit $exec + %10:vgpr_32 = V_LSHLREV_B32_e32 4, %2, implicit $exec + %11:vreg_128_align2 = DS_READ_B128_gfx9 %1, 0, 0, implicit $exec :: (load (s128), addrspace 3) + DS_WRITE_B128_gfx9 %1, %11, 4096, 0, implicit $exec :: (store (s128), addrspace 3) + %12:vgpr_32 = V_ADD_U32_e32 %2, %5, implicit $exec + %13:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub2_sub3, %6.sub2_sub3, %8, 0, 0, 0, implicit $mode, implicit $exec + %14:vgpr_32 = V_LSHLREV_B32_e32 2, %12, implicit $exec + %15:vreg_128_align2 = DS_READ_B128_gfx9 %1, 2048, 0, implicit $exec :: (load (s128), addrspace 3) + DS_WRITE_B128_gfx9 %1, %15, 6144, 0, implicit $exec :: (store (s128), addrspace 3) + %16:vgpr_32 = V_ADD_U32_e32 8192, %12, implicit $exec + %17:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %13, 0, 0, 0, implicit $mode, implicit $exec + %18:vreg_128_align2 = DS_READ_B128_gfx9 %1, 4096, 0, implicit $exec :: (load (s128), addrspace 3) + DS_WRITE_B128_gfx9 %1, %18, 8192, 0, implicit $exec :: (store (s128), addrspace 3) + %19:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %16, %3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8) + S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %5, implicit %9, implicit %10, implicit %14, implicit %17, implicit %19, implicit %20, implicit %21, implicit %22 +... + +--- +name: anti_hints_suppressed_by_current_budget +tracksRegLiveness: true +machineFunctionInfo: + isEntryFunction: true + occupancy: 4 +registers: + - { id: 0, class: vreg_128_align2 } + - { id: 1, class: vgpr_32 } + - { id: 2, class: vgpr_32 } + - { id: 3, class: sgpr_128 } + - { id: 4, class: vgpr_32 } + - { id: 5, class: vgpr_32 } + - { id: 6, class: vreg_128_align2 } + - { id: 7, class: vreg_512_align2 } + - { id: 8, class: vreg_512_align2 } + - { id: 9, class: vgpr_32, anti-hints: [ '%7' ] } + - { id: 10, class: vgpr_32, anti-hints: [ '%7' ] } + - { id: 11, class: vreg_128_align2, anti-hints: [ '%7' ] } + - { id: 12, class: vgpr_32, anti-hints: [ '%7' ] } + - { id: 13, class: vreg_512_align2 } + - { id: 14, class: vgpr_32, anti-hints: [ '%8' ] } + - { id: 15, class: vreg_128_align2, anti-hints: [ '%8' ] } + - { id: 16, class: vgpr_32, anti-hints: [ '%8' ] } + - { id: 17, class: vreg_512_align2 } + - { id: 18, class: vreg_128_align2, anti-hints: [ '%8', '%13' ] } + - { id: 19, class: vreg_128_align2, anti-hints: [ '%13' ] } + - { id: 20, class: vreg_128_align2 } + - { id: 21, class: vreg_128_align2 } + - { id: 22, class: vreg_128_align2 } +liveins: + - { reg: '$vgpr0_vgpr1_vgpr2_vgpr3' } + - { reg: '$vgpr4' } + - { reg: '$vgpr5' } + - { reg: '$vgpr52_vgpr53_vgpr54_vgpr55' } + - { reg: '$vgpr56_vgpr57_vgpr58_vgpr59' } + - { reg: '$vgpr60_vgpr61_vgpr62_vgpr63' } +body: | + bb.0: + liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4, $vgpr5, $vgpr52_vgpr53_vgpr54_vgpr55, $vgpr56_vgpr57_vgpr58_vgpr59, $vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-LABEL: name: anti_hints_suppressed_by_current_budget + ; CHECK: liveins: $vgpr4, $vgpr5, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr52_vgpr53_vgpr54_vgpr55, $vgpr56_vgpr57_vgpr58_vgpr59, $vgpr60_vgpr61_vgpr62_vgpr63 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF + ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 4096, $vgpr5, implicit $exec + ; CHECK-NEXT: renamable $vgpr43 = V_LSHLREV_B32_e32 2, $vgpr5, implicit $exec + ; CHECK-NEXT: renamable $vgpr38_vgpr39_vgpr40_vgpr41 = BUFFER_LOAD_DWORDX4_OFFEN renamable $vgpr42, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8) + ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = IMPLICIT_DEF + ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr38_vgpr39, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: renamable $vgpr38 = V_LSHLREV_B32_e32 2, killed $vgpr42, implicit $exec + ; CHECK-NEXT: renamable $vgpr39 = V_LSHLREV_B32_e32 4, $vgpr5, implicit $exec + ; CHECK-NEXT: S_NOP 4 + ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 0, 0, implicit $exec :: (load (s128), addrspace 3) + ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 4096, 0, implicit $exec :: (store (s128), addrspace 3) + ; CHECK-NEXT: renamable $vgpr42 = V_ADD_U32_e32 $vgpr5, $vgpr43, implicit $exec + ; CHECK-NEXT: early-clobber renamable $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, killed $vgpr40_vgpr41, killed $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: renamable $vgpr40 = V_LSHLREV_B32_e32 2, $vgpr42, implicit $exec + ; CHECK-NEXT: S_NOP 5 + ; CHECK-NEXT: renamable $vgpr6_vgpr7_vgpr8_vgpr9 = DS_READ_B128_gfx9 renamable $vgpr4, 2048, 0, implicit $exec :: (load (s128), addrspace 3) + ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, 6144, 0, implicit $exec :: (store (s128), addrspace 3) + ; CHECK-NEXT: renamable $vgpr41 = V_ADD_U32_e32 8192, killed $vgpr42, implicit $exec + ; CHECK-NEXT: early-clobber renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr2_vgpr3, killed $vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37, 0, 0, 0, implicit $mode, implicit $exec + ; CHECK-NEXT: S_NOP 6 + ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = DS_READ_B128_gfx9 renamable $vgpr4, 4096, 0, implicit $exec :: (load (s128), addrspace 3) + ; CHECK-NEXT: DS_WRITE_B128_gfx9 renamable $vgpr4, killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, 8192, 0, implicit $exec :: (store (s128), addrspace 3) + ; CHECK-NEXT: renamable $vgpr22_vgpr23_vgpr24_vgpr25 = BUFFER_LOAD_DWORDX4_OFFEN killed renamable $vgpr41, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8) + ; CHECK-NEXT: S_ENDPGM 0, implicit killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed renamable $vgpr4, implicit killed renamable $vgpr5, implicit killed renamable $vgpr43, implicit killed renamable $vgpr38, implicit killed renamable $vgpr39, implicit killed renamable $vgpr40, implicit killed renamable $vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21, implicit killed renamable $vgpr22_vgpr23_vgpr24_vgpr25, implicit killed renamable $vgpr52_vgpr53_vgpr54_vgpr55, implicit killed renamable $vgpr56_vgpr57_vgpr58_vgpr59, implicit killed renamable $vgpr60_vgpr61_vgpr62_vgpr63 + %20:vreg_128_align2 = COPY $vgpr52_vgpr53_vgpr54_vgpr55 + %21:vreg_128_align2 = COPY $vgpr56_vgpr57_vgpr58_vgpr59 + %22:vreg_128_align2 = COPY $vgpr60_vgpr61_vgpr62_vgpr63 + %0:vreg_128_align2 = COPY $vgpr0_vgpr1_vgpr2_vgpr3 + %1:vgpr_32 = COPY $vgpr4 + %2:vgpr_32 = COPY $vgpr5 + %3:sgpr_128 = IMPLICIT_DEF + %4:vgpr_32 = V_ADD_U32_e32 4096, %2, implicit $exec + %5:vgpr_32 = V_LSHLREV_B32_e32 2, %2, implicit $exec + %6:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %4, %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8) + %7:vreg_512_align2 = IMPLICIT_DEF + %8:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %6.sub0_sub1, %7, 0, 0, 0, implicit $mode, implicit $exec + %9:vgpr_32 = V_LSHLREV_B32_e32 2, %4, implicit $exec + %10:vgpr_32 = V_LSHLREV_B32_e32 4, %2, implicit $exec + %11:vreg_128_align2 = DS_READ_B128_gfx9 %1, 0, 0, implicit $exec :: (load (s128), addrspace 3) + DS_WRITE_B128_gfx9 %1, %11, 4096, 0, implicit $exec :: (store (s128), addrspace 3) + %12:vgpr_32 = V_ADD_U32_e32 %2, %5, implicit $exec + %13:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub2_sub3, %6.sub2_sub3, %8, 0, 0, 0, implicit $mode, implicit $exec + %14:vgpr_32 = V_LSHLREV_B32_e32 2, %12, implicit $exec + %15:vreg_128_align2 = DS_READ_B128_gfx9 %1, 2048, 0, implicit $exec :: (load (s128), addrspace 3) + DS_WRITE_B128_gfx9 %1, %15, 6144, 0, implicit $exec :: (store (s128), addrspace 3) + %16:vgpr_32 = V_ADD_U32_e32 8192, %12, implicit $exec + %17:vreg_512_align2 = contract V_MFMA_F32_32X32X8F16_vgprcd_e64 %0.sub0_sub1, %0.sub2_sub3, %13, 0, 0, 0, implicit $mode, implicit $exec + %18:vreg_128_align2 = DS_READ_B128_gfx9 %1, 4096, 0, implicit $exec :: (load (s128), addrspace 3) + DS_WRITE_B128_gfx9 %1, %18, 8192, 0, implicit $exec :: (store (s128), addrspace 3) + %19:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN %16, %3, 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8) + S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %5, implicit %9, implicit %10, implicit %14, implicit %17, implicit %19, implicit %20, implicit %21, implicit %22 +... >From 5a1ffce4766e26eb752daaf0e1edade9fdeddf85 Mon Sep 17 00:00:00 2001 From: mssefat <[email protected]> Date: Sun, 30 Aug 2026 21:27:17 -0500 Subject: [PATCH 2/2] Addressed review: Removed OrderStorage --- llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 25 +++++++++++------------ llvm/lib/Target/AMDGPU/SIRegisterInfo.h | 4 ++-- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp index 97fca7ca20a26..b83922edabdd0 100644 --- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp @@ -4224,8 +4224,8 @@ bool SIRegisterInfo::shouldApplyAntiHints( } void SIRegisterInfo::applyRegAllocationAntiHints( - Register VirtReg, ArrayRef<MCPhysReg> &Order, - SmallVectorImpl<MCPhysReg> &OrderStorage, + Register VirtReg, ArrayRef<MCPhysReg> Order, + SmallVectorImpl<MCPhysReg> &HintsAndCustomOrder, unsigned NumHints, SmallVectorImpl<MCPhysReg> &AntiHints, const MachineFunction &MF, const VirtRegMap *VRM, const LiveRegMatrix *Matrix) const { @@ -4266,9 +4266,8 @@ void SIRegisterInfo::applyRegAllocationAntiHints( return Highest < MaxVGPRsForCurrentOccupancy; }; - // Copy order. - OrderStorage.clear(); - OrderStorage.assign(Order.begin(), Order.end()); + HintsAndCustomOrder.truncate(NumHints); + HintsAndCustomOrder.append(Order.begin(), Order.end()); // Helper to check if a register overlaps with any anti-hint. auto isAntiHinted = [&](MCPhysReg Reg) { @@ -4278,21 +4277,21 @@ void SIRegisterInfo::applyRegAllocationAntiHints( }; // Find the cutoff point for the current occupancy VGPR budget. - auto *BeyondBudgetStart = llvm::find_if( - OrderStorage, [&](MCPhysReg Reg) { return !IsWithinBudget(Reg); }); + auto *BeyondBudgetStart = + llvm::find_if(llvm::drop_begin(HintsAndCustomOrder, NumHints), + [&](MCPhysReg Reg) { return !IsWithinBudget(Reg); }); // Only shuffle within the current occupancy VGPR budget. - auto *PartitionPoint = - std::stable_partition(OrderStorage.begin(), BeyondBudgetStart, - [&](MCPhysReg Reg) { return !isAntiHinted(Reg); }); + auto *PartitionPoint = std::stable_partition( + HintsAndCustomOrder.begin() + NumHints, BeyondBudgetStart, + [&](MCPhysReg Reg) { return !isAntiHinted(Reg); }); - Order = OrderStorage; LLVM_DEBUG({ size_t NonAntiHintedCount = - std::distance(OrderStorage.begin(), PartitionPoint); + std::distance(HintsAndCustomOrder.begin() + NumHints, PartitionPoint); size_t AntiHintedCount = std::distance(PartitionPoint, BeyondBudgetStart); size_t BeyondBudgetCount = - std::distance(BeyondBudgetStart, OrderStorage.end()); + std::distance(BeyondBudgetStart, HintsAndCustomOrder.end()); dbgs() << "Added " << NonAntiHintedCount << " non-anti-hinted registers first\n" << "Added " << AntiHintedCount diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h index 61fbf90d84488..7765849ab804b 100644 --- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h +++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h @@ -374,8 +374,8 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo { unsigned &MaxVGPRsForCurrentOccupancy) const; void applyRegAllocationAntiHints( - Register VirtReg, ArrayRef<MCPhysReg> &Order, - SmallVectorImpl<MCPhysReg> &OrderStorage, + Register VirtReg, ArrayRef<MCPhysReg> Order, + SmallVectorImpl<MCPhysReg> &HintsAndCustomOrder, unsigned NumHints, SmallVectorImpl<MCPhysReg> &AntiHints, const MachineFunction &MF, const VirtRegMap *VRM = nullptr, const LiveRegMatrix *Matrix = nullptr) const override; _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
