https://github.com/saxlungs updated https://github.com/llvm/llvm-project/pull/224080
>From 12ad2b9973685b99caffa0822b50549ff15d299f Mon Sep 17 00:00:00 2001 From: Domenic Nutile <[email protected]> Date: Wed, 16 Sep 2026 12:12:46 -0400 Subject: [PATCH] [AMDGPU] Don't spill an SGPR while SCC is live in frame index lowering When SCC is live into a scalar frame index user, the scaling path avoids SALU ops that write SCC by computing the address in a VGPR and reading it back with V_READFIRSTLANE_B32. If the destination of that readfirstlane is scavenged with spilling allowed, an AMDGPU SGPR spill writes inactive lanes, so it flips EXEC with S_NOT_B64 and clobbers SCC. Instead, scavenge that register with AllowSpill=false. --- llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 39 ++++-- ...to-vmem-scc-clobber-reserved-exec-copy.mir | 126 ++++++++++++++++++ 2 files changed, 155 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp index 7cad384c98a8d..b6b1bccb0820d 100644 --- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp @@ -3421,11 +3421,32 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, // Scaling FrameReg in place is the last resort when there is nothing to // scavenge. It has to be undone after MI, which is only possible while MI // does not use FrameReg for anything besides the frame index. - bool CanUseFrameRegAsScratch = IsSALU && !LiveSCC && FrameReg && - !MI->readsRegister(FrameReg, this) && - !MI->modifiesRegister(FrameReg, this); + bool CanUseFrameRegAsSGPRScratch = IsSALU && FrameReg && + !MI->readsRegister(FrameReg, this) && + !MI->modifiesRegister(FrameReg, this); + // ResultReg is a VGPR while SCC is live, so FrameReg cannot stand in for + // it there. + bool CanUseFrameRegAsScratch = CanUseFrameRegAsSGPRScratch && !LiveSCC; bool RestoreFrameReg = false; + + // Scavenge the scalar temporary the V_READFIRSTLANE_B32 result lands in. + // Spilling an SGPR here would flip EXEC with S_NOT, and that clobbers the + // SCC this path exists to preserve, so fall back to FrameReg instead. + auto ScavengeSGPRForReadfirstlane = + [&](MachineBasicBlock::iterator To) -> Register { + if (Register Reg = RS->scavengeRegisterBackwards( + AMDGPU::SReg_32_XM0RegClass, To, false, 0, + /*AllowSpill=*/false)) + return Reg; + if (CanUseFrameRegAsSGPRScratch) { + RestoreFrameReg = true; + return FrameReg; + } + return RS->scavengeRegisterBackwards(AMDGPU::SReg_32_XM0RegClass, To, + false, 0); + }; + Register ResultReg; if (IsCopy) { ResultReg = MI->getOperand(0).getReg(); @@ -3479,8 +3500,7 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, assert(ResultReg.isPhysical()); NewDest = ResultReg; } else { - NewDest = RS->scavengeRegisterBackwards(AMDGPU::SReg_32_XM0RegClass, - Shift, false, 0); + NewDest = ScavengeSGPRForReadfirstlane(Shift); } BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), NewDest) .addReg(TmpResultReg); @@ -3600,9 +3620,7 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, if (IsCopy) { NewDest = ResultReg; } else { - NewDest = RS->scavengeRegisterBackwards( - AMDGPU::SReg_32_XM0RegClass, *Add, false, 0, - /*AllowSpill=*/true); + NewDest = ScavengeSGPRForReadfirstlane(*Add); } BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), @@ -3641,8 +3659,9 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, if (Offset) { int64_t ScaledOffset = -Offset * ST.getWavefrontSize(); - bool SCCLiveAfterMI = MI->definesRegister(AMDGPU::SCC, this) && - !MI->registerDefIsDead(AMDGPU::SCC, this); + bool SCCLiveAfterMI = MI->definesRegister(AMDGPU::SCC, this) + ? !MI->registerDefIsDead(AMDGPU::SCC, this) + : LiveSCC; if (!SCCLiveAfterMI) { BuildMI(*MBB, InsPt, DL, TII->get(AMDGPU::S_ADD_I32), FrameReg) .addReg(FrameReg) diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spill-to-vmem-scc-clobber-reserved-exec-copy.mir b/llvm/test/CodeGen/AMDGPU/sgpr-spill-to-vmem-scc-clobber-reserved-exec-copy.mir index 5ea0cbfbebab7..20995bc210351 100644 --- a/llvm/test/CodeGen/AMDGPU/sgpr-spill-to-vmem-scc-clobber-reserved-exec-copy.mir +++ b/llvm/test/CodeGen/AMDGPU/sgpr-spill-to-vmem-scc-clobber-reserved-exec-copy.mir @@ -166,3 +166,129 @@ body: | S_ENDPGM 0, implicit $vcc ... + +--- +# SCC is live *into* the scalar frame index user: S_CMP_EQ_U32 produces it and +# S_CSELECT_B32 reads it. With no SGPR free to hold the V_READFIRSTLANE_B32 +# result, scavenging one would emergency-spill, and an SGPR spill flips EXEC +# with S_NOT_B64, clobbering the very SCC this path preserves. Scale FrameReg in +# place instead and restore it after MI with S_MUL_I32, which does not write SCC. +name: scc_live_into_scalar_frame_index_no_free_sgpr +tracksRegLiveness: true +stack: + - { id: 0, size: 64, alignment: 4 } + - { id: 1, size: 64, alignment: 4 } +machineFunctionInfo: + isEntryFunction: false + scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3' + frameOffsetReg: '$sgpr33' + stackPtrOffsetReg: '$sgpr32' + sgprForEXECCopy: '$sgpr100_sgpr101' + occupancy: 2 +body: | + ; CHECK-LABEL: name: scc_live_into_scalar_frame_index_no_free_sgpr + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: frame-setup CFI_INSTRUCTION llvm_def_aspace_cfa $sgpr32, 0, 6 + ; CHECK-NEXT: frame-setup CFI_INSTRUCTION llvm_register_pair $pc_reg, $sgpr30, 32, $sgpr31, 32 + ; CHECK-NEXT: S_BRANCH %bb.1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: successors: %bb.2(0x80000000) + ; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31_sgpr32_sgpr33_sgpr34_sgpr35, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83, $sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, $sgpr96_sgpr97_sgpr98_sgpr99, $vcc + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_CMP_EQ_U32 renamable $sgpr96, 0, implicit-def $scc + ; CHECK-NEXT: $vgpr0 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec + ; CHECK-NEXT: $sgpr32 = V_READFIRSTLANE_B32 $vgpr0, implicit $exec + ; CHECK-NEXT: renamable $vcc_lo = S_CSELECT_B32 $sgpr32, renamable $sgpr97, implicit $scc + ; CHECK-NEXT: $sgpr32 = S_MUL_I32 $sgpr32, 64 + ; CHECK-NEXT: S_NOP 0, implicit killed $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, implicit killed $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31_sgpr32_sgpr33_sgpr34_sgpr35, implicit killed $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, implicit killed $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, implicit killed $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83, implicit killed $sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, implicit killed $sgpr96_sgpr97_sgpr98_sgpr99 + ; CHECK-NEXT: S_BRANCH %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: liveins: $vcc + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_ENDPGM 0, implicit $vcc + bb.0: + S_BRANCH %bb.1 + + bb.1: + liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31_sgpr32_sgpr33_sgpr34_sgpr35, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83, $sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, $sgpr96_sgpr97_sgpr98_sgpr99, $vcc + + S_CMP_EQ_U32 renamable $sgpr96, 0, implicit-def $scc + renamable $vcc_lo = S_CSELECT_B32 %stack.0, renamable $sgpr97, implicit $scc + S_NOP 0, implicit killed $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, implicit killed $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31_sgpr32_sgpr33_sgpr34_sgpr35, implicit killed $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, implicit killed $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, implicit killed $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83, implicit killed $sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, implicit killed $sgpr96_sgpr97_sgpr98_sgpr99 + S_BRANCH %bb.2 + + bb.2: + liveins: $vcc + + S_ENDPGM 0, implicit $vcc +... + +--- +# As above, but with a non-zero offset and with SCC live *across* the frame index +# user: S_CBRANCH_SCC1 still needs it after S_CSELECT_B32 has read it. Restoring +# FrameReg therefore has to fold the offset back without losing SCC, so S_MUL_I32 +# is followed by the S_ADDC_U32/S_BITCMP1_B32/S_BITSET0_B32 sequence that +# smuggles SCC through bit 0, which the scaling has just cleared. +name: scc_live_into_scalar_frame_index_no_free_sgpr_nonzero_offset +tracksRegLiveness: true +stack: + - { id: 0, size: 64, alignment: 4 } + - { id: 1, size: 64, alignment: 4 } +machineFunctionInfo: + isEntryFunction: false + scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3' + frameOffsetReg: '$sgpr33' + stackPtrOffsetReg: '$sgpr32' + sgprForEXECCopy: '$sgpr100_sgpr101' + occupancy: 2 +body: | + ; CHECK-LABEL: name: scc_live_into_scalar_frame_index_no_free_sgpr_nonzero_offset + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: frame-setup CFI_INSTRUCTION llvm_def_aspace_cfa $sgpr32, 0, 6 + ; CHECK-NEXT: frame-setup CFI_INSTRUCTION llvm_register_pair $pc_reg, $sgpr30, 32, $sgpr31, 32 + ; CHECK-NEXT: S_BRANCH %bb.1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: successors: %bb.2(0x80000000) + ; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31_sgpr32_sgpr33_sgpr34_sgpr35, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83, $sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, $sgpr96_sgpr97_sgpr98_sgpr99, $vcc + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_CMP_EQ_U32 renamable $sgpr96, 0, implicit-def $scc + ; CHECK-NEXT: $vgpr0 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec + ; CHECK-NEXT: $vgpr0 = V_ADD_U32_e32 64, killed $vgpr0, implicit $exec + ; CHECK-NEXT: $sgpr32 = V_READFIRSTLANE_B32 $vgpr0, implicit $exec + ; CHECK-NEXT: renamable $vcc_lo = S_CSELECT_B32 $sgpr32, renamable $sgpr97, implicit $scc + ; CHECK-NEXT: $sgpr32 = S_MUL_I32 $sgpr32, 64 + ; CHECK-NEXT: $sgpr32 = S_ADDC_U32 $sgpr32, -4096, implicit-def $scc, implicit $scc + ; CHECK-NEXT: S_BITCMP1_B32 $sgpr32, 0, implicit-def $scc + ; CHECK-NEXT: $sgpr32 = S_BITSET0_B32 0, $sgpr32 + ; CHECK-NEXT: S_NOP 0, implicit killed $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, implicit killed $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31_sgpr32_sgpr33_sgpr34_sgpr35, implicit killed $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, implicit killed $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, implicit killed $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83, implicit killed $sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, implicit killed $sgpr96_sgpr97_sgpr98_sgpr99 + ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.2, implicit $scc + ; CHECK-NEXT: S_BRANCH %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: liveins: $vcc + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_ENDPGM 0, implicit $vcc + bb.0: + S_BRANCH %bb.1 + + bb.1: + liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31_sgpr32_sgpr33_sgpr34_sgpr35, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83, $sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, $sgpr96_sgpr97_sgpr98_sgpr99, $vcc + + S_CMP_EQ_U32 renamable $sgpr96, 0, implicit-def $scc + renamable $vcc_lo = S_CSELECT_B32 %stack.1, renamable $sgpr97, implicit $scc + S_NOP 0, implicit killed $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, implicit killed $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31_sgpr32_sgpr33_sgpr34_sgpr35, implicit killed $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, implicit killed $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, implicit killed $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83, implicit killed $sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, implicit killed $sgpr96_sgpr97_sgpr98_sgpr99 + S_CBRANCH_SCC1 %bb.2, implicit $scc + S_BRANCH %bb.2 + + bb.2: + liveins: $vcc + + S_ENDPGM 0, implicit $vcc +... _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
