https://github.com/saxlungs updated https://github.com/llvm/llvm-project/pull/224080
>From f4edcdabd222b92798e2adf1732ccf226c6c48b0 Mon Sep 17 00:00:00 2001 From: Domenic Nutile <[email protected]> Date: Wed, 16 Sep 2026 12:12:46 -0400 Subject: [PATCH 1/3] [AMDGPU] Don't spill an SGPR while SCC is live in frame index lowering When SCC is live into a scalar frame index user, the scaling path avoids SALU ops that write SCC by computing the address in a VGPR and reading it back with V_READFIRSTLANE_B32. If the destination of that readfirstlane is scavenged with spilling allowed, an AMDGPU SGPR spill writes inactive lanes, so it flips EXEC with S_NOT_B64 and clobbers SCC. Instead, scavenge that register with AllowSpill=false. --- llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 39 ++++-- ...to-vmem-scc-clobber-reserved-exec-copy.mir | 126 +++++++++++------- 2 files changed, 110 insertions(+), 55 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp index 7cad384c98a8d..b6b1bccb0820d 100644 --- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp @@ -3421,11 +3421,32 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, // Scaling FrameReg in place is the last resort when there is nothing to // scavenge. It has to be undone after MI, which is only possible while MI // does not use FrameReg for anything besides the frame index. - bool CanUseFrameRegAsScratch = IsSALU && !LiveSCC && FrameReg && - !MI->readsRegister(FrameReg, this) && - !MI->modifiesRegister(FrameReg, this); + bool CanUseFrameRegAsSGPRScratch = IsSALU && FrameReg && + !MI->readsRegister(FrameReg, this) && + !MI->modifiesRegister(FrameReg, this); + // ResultReg is a VGPR while SCC is live, so FrameReg cannot stand in for + // it there. + bool CanUseFrameRegAsScratch = CanUseFrameRegAsSGPRScratch && !LiveSCC; bool RestoreFrameReg = false; + + // Scavenge the scalar temporary the V_READFIRSTLANE_B32 result lands in. + // Spilling an SGPR here would flip EXEC with S_NOT, and that clobbers the + // SCC this path exists to preserve, so fall back to FrameReg instead. + auto ScavengeSGPRForReadfirstlane = + [&](MachineBasicBlock::iterator To) -> Register { + if (Register Reg = RS->scavengeRegisterBackwards( + AMDGPU::SReg_32_XM0RegClass, To, false, 0, + /*AllowSpill=*/false)) + return Reg; + if (CanUseFrameRegAsSGPRScratch) { + RestoreFrameReg = true; + return FrameReg; + } + return RS->scavengeRegisterBackwards(AMDGPU::SReg_32_XM0RegClass, To, + false, 0); + }; + Register ResultReg; if (IsCopy) { ResultReg = MI->getOperand(0).getReg(); @@ -3479,8 +3500,7 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, assert(ResultReg.isPhysical()); NewDest = ResultReg; } else { - NewDest = RS->scavengeRegisterBackwards(AMDGPU::SReg_32_XM0RegClass, - Shift, false, 0); + NewDest = ScavengeSGPRForReadfirstlane(Shift); } BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), NewDest) .addReg(TmpResultReg); @@ -3600,9 +3620,7 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, if (IsCopy) { NewDest = ResultReg; } else { - NewDest = RS->scavengeRegisterBackwards( - AMDGPU::SReg_32_XM0RegClass, *Add, false, 0, - /*AllowSpill=*/true); + NewDest = ScavengeSGPRForReadfirstlane(*Add); } BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), @@ -3641,8 +3659,9 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, if (Offset) { int64_t ScaledOffset = -Offset * ST.getWavefrontSize(); - bool SCCLiveAfterMI = MI->definesRegister(AMDGPU::SCC, this) && - !MI->registerDefIsDead(AMDGPU::SCC, this); + bool SCCLiveAfterMI = MI->definesRegister(AMDGPU::SCC, this) + ? !MI->registerDefIsDead(AMDGPU::SCC, this) + : LiveSCC; if (!SCCLiveAfterMI) { BuildMI(*MBB, InsPt, DL, TII->get(AMDGPU::S_ADD_I32), FrameReg) .addReg(FrameReg) diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spill-to-vmem-scc-clobber-reserved-exec-copy.mir b/llvm/test/CodeGen/AMDGPU/sgpr-spill-to-vmem-scc-clobber-reserved-exec-copy.mir index 68c0c63e14fac..36a532d6c63e0 100644 --- a/llvm/test/CodeGen/AMDGPU/sgpr-spill-to-vmem-scc-clobber-reserved-exec-copy.mir +++ b/llvm/test/CodeGen/AMDGPU/sgpr-spill-to-vmem-scc-clobber-reserved-exec-copy.mir @@ -170,9 +170,9 @@ body: | --- # SCC is live *into* the scalar frame index user: S_CMP_EQ_U32 produces it and # S_CSELECT_B32 reads it. With no SGPR free to hold the V_READFIRSTLANE_B32 -# result, scavenging one emergency-spills, and an SGPR spill flips EXEC with -# S_NOT_B64. Those EXEC flips land between the S_CMP_EQ_U32 and the -# S_CSELECT_B32 and clobber the SCC in between, which is wrong. +# result, scavenging one would emergency-spill, and an SGPR spill flips EXEC +# with S_NOT_B64, clobbering the very SCC this path preserves. Scale FrameReg in +# place instead and restore it after MI with S_MUL_I32, which does not write SCC. name: scc_live_into_scalar_frame_index_no_free_sgpr tracksRegLiveness: true stack: @@ -199,27 +199,10 @@ body: | ; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31_sgpr32_sgpr33_sgpr34_sgpr35, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83, $sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, $sgpr96_sgpr97_sgpr98_sgpr99, $vcc ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: S_CMP_EQ_U32 renamable $sgpr96, 0, implicit-def $scc - ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc, implicit-def $vgpr1 - ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.2, addrspace 5) - ; CHECK-NEXT: $vgpr1 = SI_SPILL_S32_TO_VGPR killed $sgpr4, 0, undef $vgpr1 - ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.2, addrspace 5) - ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc - ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.2, addrspace 5) - ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc - ; CHECK-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" load (s32) from %stack.2, addrspace 5) - ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc, implicit killed $vgpr1 ; CHECK-NEXT: $vgpr0 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec - ; CHECK-NEXT: $sgpr4 = V_READFIRSTLANE_B32 $vgpr0, implicit $exec - ; CHECK-NEXT: renamable $vcc_lo = S_CSELECT_B32 killed $sgpr4, renamable $sgpr97, implicit $scc - ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc, implicit-def $vgpr0 - ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.2, addrspace 5) - ; CHECK-NEXT: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" load (s32) from %stack.2, addrspace 5) - ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc - ; CHECK-NEXT: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" load (s32) from %stack.2, addrspace 5) - ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc - ; CHECK-NEXT: $sgpr4 = SI_RESTORE_S32_FROM_VGPR killed $vgpr0, 0 - ; CHECK-NEXT: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" load (s32) from %stack.2, addrspace 5) - ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc, implicit killed $vgpr0 + ; CHECK-NEXT: $sgpr32 = V_READFIRSTLANE_B32 $vgpr0, implicit $exec + ; CHECK-NEXT: renamable $vcc_lo = S_CSELECT_B32 $sgpr32, renamable $sgpr97, implicit $scc + ; CHECK-NEXT: $sgpr32 = S_MUL_I32 $sgpr32, 64 ; CHECK-NEXT: S_NOP 0, implicit killed $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, implicit killed $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31_sgpr32_sgpr33_sgpr34_sgpr35, implicit killed $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, implicit killed $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, implicit killed $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83, implicit killed $sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, implicit killed $sgpr96_sgpr97_sgpr98_sgpr99 ; CHECK-NEXT: S_BRANCH %bb.2 ; CHECK-NEXT: {{ $}} @@ -244,10 +227,77 @@ body: | S_ENDPGM 0, implicit $vcc ... +--- +# As above, but with a non-zero offset and with SCC live *across* the frame index +# user: S_CBRANCH_SCC1 still needs it after S_CSELECT_B32 has read it. Restoring +# FrameReg therefore has to fold the offset back without losing SCC, so S_MUL_I32 +# is followed by the S_ADDC_U32/S_BITCMP1_B32/S_BITSET0_B32 sequence that +# smuggles SCC through bit 0, which the scaling has just cleared. +name: scc_live_into_scalar_frame_index_no_free_sgpr_nonzero_offset +tracksRegLiveness: true +stack: + - { id: 0, size: 64, alignment: 4 } + - { id: 1, size: 64, alignment: 4 } +machineFunctionInfo: + isEntryFunction: false + scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3' + frameOffsetReg: '$sgpr33' + stackPtrOffsetReg: '$sgpr32' + sgprForEXECCopy: '$sgpr100_sgpr101' + occupancy: 2 +body: | + ; CHECK-LABEL: name: scc_live_into_scalar_frame_index_no_free_sgpr_nonzero_offset + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: frame-setup CFI_INSTRUCTION llvm_def_aspace_cfa $sgpr32, 0, 6 + ; CHECK-NEXT: frame-setup CFI_INSTRUCTION llvm_register_pair $pc_reg, $sgpr30, 32, $sgpr31, 32 + ; CHECK-NEXT: S_BRANCH %bb.1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: successors: %bb.2(0x80000000) + ; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31_sgpr32_sgpr33_sgpr34_sgpr35, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83, $sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, $sgpr96_sgpr97_sgpr98_sgpr99, $vcc + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_CMP_EQ_U32 renamable $sgpr96, 0, implicit-def $scc + ; CHECK-NEXT: $vgpr0 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec + ; CHECK-NEXT: $vgpr0 = V_ADD_U32_e32 64, killed $vgpr0, implicit $exec + ; CHECK-NEXT: $sgpr32 = V_READFIRSTLANE_B32 $vgpr0, implicit $exec + ; CHECK-NEXT: renamable $vcc_lo = S_CSELECT_B32 $sgpr32, renamable $sgpr97, implicit $scc + ; CHECK-NEXT: $sgpr32 = S_MUL_I32 $sgpr32, 64 + ; CHECK-NEXT: $sgpr32 = S_ADDC_U32 $sgpr32, -4096, implicit-def $scc, implicit $scc + ; CHECK-NEXT: S_BITCMP1_B32 $sgpr32, 0, implicit-def $scc + ; CHECK-NEXT: $sgpr32 = S_BITSET0_B32 0, $sgpr32 + ; CHECK-NEXT: S_NOP 0, implicit killed $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, implicit killed $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31_sgpr32_sgpr33_sgpr34_sgpr35, implicit killed $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, implicit killed $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, implicit killed $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83, implicit killed $sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, implicit killed $sgpr96_sgpr97_sgpr98_sgpr99 + ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.2, implicit $scc + ; CHECK-NEXT: S_BRANCH %bb.2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: liveins: $vcc + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: S_ENDPGM 0, implicit $vcc + bb.0: + S_BRANCH %bb.1 + + bb.1: + liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31_sgpr32_sgpr33_sgpr34_sgpr35, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83, $sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, $sgpr96_sgpr97_sgpr98_sgpr99, $vcc + + S_CMP_EQ_U32 renamable $sgpr96, 0, implicit-def $scc + renamable $vcc_lo = S_CSELECT_B32 %stack.1, renamable $sgpr97, implicit $scc + S_NOP 0, implicit killed $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, implicit killed $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31_sgpr32_sgpr33_sgpr34_sgpr35, implicit killed $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, implicit killed $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, implicit killed $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83, implicit killed $sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, implicit killed $sgpr96_sgpr97_sgpr98_sgpr99 + S_CBRANCH_SCC1 %bb.2, implicit $scc + S_BRANCH %bb.2 + + bb.2: + liveins: $vcc + + S_ENDPGM 0, implicit $vcc +... + --- # As above, but with a non-zero offset and with SCC dead after the frame index -# user: S_CSELECT_B32 is its last read. The emergency spill clobbers SCC before -# that read here too. +# user: S_CSELECT_B32 is its last read. FrameReg is still restored with the +# S_ADDC_U32/S_BITCMP1_B32/S_BITSET0_B32 sequence that preserves SCC, even +# though nothing after MI reads it. name: scc_live_into_dead_after_scalar_frame_index_no_free_sgpr_nonzero_offset tracksRegLiveness: true stack: @@ -275,27 +325,13 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: S_CMP_EQ_U32 renamable $sgpr96, 0, implicit-def $scc ; CHECK-NEXT: $vgpr0 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec - ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc, implicit-def $vgpr1 - ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.2, addrspace 5) - ; CHECK-NEXT: $vgpr1 = SI_SPILL_S32_TO_VGPR killed $sgpr4, 0, undef $vgpr1 - ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.2, addrspace 5) - ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc - ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.2, addrspace 5) - ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc - ; CHECK-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" load (s32) from %stack.2, addrspace 5) - ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc, implicit killed $vgpr1 ; CHECK-NEXT: $vgpr0 = V_ADD_U32_e32 64, killed $vgpr0, implicit $exec - ; CHECK-NEXT: $sgpr4 = V_READFIRSTLANE_B32 $vgpr0, implicit $exec - ; CHECK-NEXT: renamable $vcc_lo = S_CSELECT_B32 killed $sgpr4, renamable $sgpr97, implicit $scc - ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc, implicit-def $vgpr0 - ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.2, addrspace 5) - ; CHECK-NEXT: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" load (s32) from %stack.2, addrspace 5) - ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc - ; CHECK-NEXT: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" load (s32) from %stack.2, addrspace 5) - ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc - ; CHECK-NEXT: $sgpr4 = SI_RESTORE_S32_FROM_VGPR killed $vgpr0, 0 - ; CHECK-NEXT: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" load (s32) from %stack.2, addrspace 5) - ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc, implicit killed $vgpr0 + ; CHECK-NEXT: $sgpr32 = V_READFIRSTLANE_B32 $vgpr0, implicit $exec + ; CHECK-NEXT: renamable $vcc_lo = S_CSELECT_B32 $sgpr32, renamable $sgpr97, implicit $scc + ; CHECK-NEXT: $sgpr32 = S_MUL_I32 $sgpr32, 64 + ; CHECK-NEXT: $sgpr32 = S_ADDC_U32 $sgpr32, -4096, implicit-def $scc, implicit $scc + ; CHECK-NEXT: S_BITCMP1_B32 $sgpr32, 0, implicit-def $scc + ; CHECK-NEXT: $sgpr32 = S_BITSET0_B32 0, $sgpr32 ; CHECK-NEXT: S_NOP 0, implicit killed $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, implicit killed $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31_sgpr32_sgpr33_sgpr34_sgpr35, implicit killed $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, implicit killed $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, implicit killed $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83, implicit killed $sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, implicit killed $sgpr96_sgpr97_sgpr98_sgpr99 ; CHECK-NEXT: S_BRANCH %bb.2 ; CHECK-NEXT: {{ $}} >From 895a2d3573ba6c8a1124a14a6d0a0a339d6ff0c8 Mon Sep 17 00:00:00 2001 From: Domenic Nutile <[email protected]> Date: Mon, 21 Sep 2026 13:31:31 -0400 Subject: [PATCH 2/3] [AMDGPU] Use the scavenger to test whether SCC is live after MI The register scavenger is stepped backwards to the liveness state immediately after MI, so RS->isRegUsed(SCC) already answers "is SCC live after MI" directly. Replace the hand-rolled test with that query. --- llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 6 +++--- ...r-spill-to-vmem-scc-clobber-reserved-exec-copy.mir | 11 +++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp index b6b1bccb0820d..87e18dba86d4b 100644 --- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp @@ -3409,6 +3409,9 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, // In an entry function/kernel the offset is already swizzled. bool IsSALU = isSGPRClass(TII->getRegClass(MI->getDesc(), FIOperandNum)); bool LiveSCC = isSCCLiveInto(*RS, *MI); + // The scavenger is positioned at the liveness state immediately after MI, + // so we need only check if SCC is used. + bool SCCLiveAfterMI = RS->isRegUsed(AMDGPU::SCC); const TargetRegisterClass *RC = IsSALU && !LiveSCC ? &AMDGPU::SReg_32RegClass : &AMDGPU::VGPR_32RegClass; @@ -3659,9 +3662,6 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, if (Offset) { int64_t ScaledOffset = -Offset * ST.getWavefrontSize(); - bool SCCLiveAfterMI = MI->definesRegister(AMDGPU::SCC, this) - ? !MI->registerDefIsDead(AMDGPU::SCC, this) - : LiveSCC; if (!SCCLiveAfterMI) { BuildMI(*MBB, InsPt, DL, TII->get(AMDGPU::S_ADD_I32), FrameReg) .addReg(FrameReg) diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spill-to-vmem-scc-clobber-reserved-exec-copy.mir b/llvm/test/CodeGen/AMDGPU/sgpr-spill-to-vmem-scc-clobber-reserved-exec-copy.mir index 36a532d6c63e0..c499f00a57361 100644 --- a/llvm/test/CodeGen/AMDGPU/sgpr-spill-to-vmem-scc-clobber-reserved-exec-copy.mir +++ b/llvm/test/CodeGen/AMDGPU/sgpr-spill-to-vmem-scc-clobber-reserved-exec-copy.mir @@ -295,9 +295,10 @@ body: | --- # As above, but with a non-zero offset and with SCC dead after the frame index -# user: S_CSELECT_B32 is its last read. FrameReg is still restored with the -# S_ADDC_U32/S_BITCMP1_B32/S_BITSET0_B32 sequence that preserves SCC, even -# though nothing after MI reads it. +# user: S_CSELECT_B32 is its last read. Restoring FrameReg can therefore fold +# the offset back with a plain S_ADD_I32 and clobber SCC freely, instead of the +# S_ADDC_U32/S_BITCMP1_B32/S_BITSET0_B32 sequence needed when SCC has a later +# use. name: scc_live_into_dead_after_scalar_frame_index_no_free_sgpr_nonzero_offset tracksRegLiveness: true stack: @@ -329,9 +330,7 @@ body: | ; CHECK-NEXT: $sgpr32 = V_READFIRSTLANE_B32 $vgpr0, implicit $exec ; CHECK-NEXT: renamable $vcc_lo = S_CSELECT_B32 $sgpr32, renamable $sgpr97, implicit $scc ; CHECK-NEXT: $sgpr32 = S_MUL_I32 $sgpr32, 64 - ; CHECK-NEXT: $sgpr32 = S_ADDC_U32 $sgpr32, -4096, implicit-def $scc, implicit $scc - ; CHECK-NEXT: S_BITCMP1_B32 $sgpr32, 0, implicit-def $scc - ; CHECK-NEXT: $sgpr32 = S_BITSET0_B32 0, $sgpr32 + ; CHECK-NEXT: $sgpr32 = S_ADD_I32 $sgpr32, -4096, implicit-def $scc ; CHECK-NEXT: S_NOP 0, implicit killed $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, implicit killed $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31_sgpr32_sgpr33_sgpr34_sgpr35, implicit killed $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, implicit killed $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, implicit killed $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83, implicit killed $sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, implicit killed $sgpr96_sgpr97_sgpr98_sgpr99 ; CHECK-NEXT: S_BRANCH %bb.2 ; CHECK-NEXT: {{ $}} >From 8fcba4442db8445f1aaa7f785f353a7260bab668 Mon Sep 17 00:00:00 2001 From: Domenic Nutile <[email protected]> Date: Mon, 21 Sep 2026 15:18:15 -0400 Subject: [PATCH 3/3] [AMDGPU] Update based on review feedback Replace the lambda with the check inlined at both sites, and report a fatal error when neither a free SGPR nor FrameReg is available, rather than silently falling back to the spilling scavenge. --- llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 43 +++++++++++++---------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp index 87e18dba86d4b..ec324d3667032 100644 --- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp @@ -3433,23 +3433,6 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, bool RestoreFrameReg = false; - // Scavenge the scalar temporary the V_READFIRSTLANE_B32 result lands in. - // Spilling an SGPR here would flip EXEC with S_NOT, and that clobbers the - // SCC this path exists to preserve, so fall back to FrameReg instead. - auto ScavengeSGPRForReadfirstlane = - [&](MachineBasicBlock::iterator To) -> Register { - if (Register Reg = RS->scavengeRegisterBackwards( - AMDGPU::SReg_32_XM0RegClass, To, false, 0, - /*AllowSpill=*/false)) - return Reg; - if (CanUseFrameRegAsSGPRScratch) { - RestoreFrameReg = true; - return FrameReg; - } - return RS->scavengeRegisterBackwards(AMDGPU::SReg_32_XM0RegClass, To, - false, 0); - }; - Register ResultReg; if (IsCopy) { ResultReg = MI->getOperand(0).getReg(); @@ -3503,7 +3486,19 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, assert(ResultReg.isPhysical()); NewDest = ResultReg; } else { - NewDest = ScavengeSGPRForReadfirstlane(Shift); + // Spilling an SGPR here would flip EXEC with S_NOT, and that + // clobbers the SCC this path exists to preserve, so scale FrameReg + // in place instead. + NewDest = RS->scavengeRegisterBackwards(AMDGPU::SReg_32_XM0RegClass, + Shift, false, 0, + /*AllowSpill=*/false); + if (!NewDest) { + if (!CanUseFrameRegAsSGPRScratch) + report_fatal_error("Cannot scavenge register while SCC is live " + "in FI elimination!"); + NewDest = FrameReg; + RestoreFrameReg = true; + } } BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), NewDest) .addReg(TmpResultReg); @@ -3623,7 +3618,17 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, if (IsCopy) { NewDest = ResultReg; } else { - NewDest = ScavengeSGPRForReadfirstlane(*Add); + // As above, an SGPR spill here would clobber the live SCC. + NewDest = RS->scavengeRegisterBackwards( + AMDGPU::SReg_32_XM0RegClass, *Add, false, 0, + /*AllowSpill=*/false); + if (!NewDest) { + if (!CanUseFrameRegAsSGPRScratch) + report_fatal_error("Cannot scavenge register while SCC is " + "live in FI elimination!"); + NewDest = FrameReg; + RestoreFrameReg = true; + } } BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
