https://github.com/arsenm created https://github.com/llvm/llvm-project/pull/208423
Currently the AMDGPU implementation of foldImmediate handles a small set of special cases that require rewriting the instruction opcode (COPY -> mov, and fma-like to fmac-like). Most general immediate folding is handled in SIFoldOperands. Teach PeepholeOpt to fold all inline immediates into target instructions which should always be an improvement. This is to help alleviate some phase ordering problems in future changes. I've also never liked how SIFoldOperands is written and it could use a rewrite, and this will alleviate some of its responsibilties. It will always be necessary since some operand folds require additional context that foldImmediate won't have (namely, we have to make contextually aware tradeoffs for which operands are worth folding to respect constant bus restrictions, considering other users of the values). Co-authored-by: Claude (Opus 4.8) <[email protected]> >From 58eaa33aeed68a10a45a82719dcba23605225628 Mon Sep 17 00:00:00 2001 From: Matt Arsenault <[email protected]> Date: Wed, 8 Jul 2026 13:45:32 +0200 Subject: [PATCH] AMDGPU: Fold inline immediates in peephole-opt Currently the AMDGPU implementation of foldImmediate handles a small set of special cases that require rewriting the instruction opcode (COPY -> mov, and fma-like to fmac-like). Most general immediate folding is handled in SIFoldOperands. Teach PeepholeOpt to fold all inline immediates into target instructions which should always be an improvement. This is to help alleviate some phase ordering problems in future changes. I've also never liked how SIFoldOperands is written and it could use a rewrite, and this will alleviate some of its responsibilties. It will always be necessary since some operand folds require additional context that foldImmediate won't have (namely, we have to make contextually aware tradeoffs for which operands are worth folding to respect constant bus restrictions, considering other users of the values). Co-authored-by: Claude (Opus 4.8) <[email protected]> --- llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 68 ++++++++++-- llvm/test/CodeGen/AMDGPU/optimize-compare.mir | 8 +- .../test/CodeGen/AMDGPU/peephole-fold-imm.mir | 101 ++++++++++++++++-- llvm/test/CodeGen/AMDGPU/sdiv64.ll | 10 +- 4 files changed, 159 insertions(+), 28 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index 90ab5fd545c11..598c4a3801f97 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -3569,6 +3569,28 @@ static unsigned getNewFMAMKInst(const GCNSubtarget &ST, unsigned Opc) { } } +// Return true if \p Opc is a mad/fma family which can be converted to to a +// madmk/madak form with a folded literal constant. +static bool isMADFMA(unsigned Opc) { + switch (Opc) { + case AMDGPU::V_MAD_F32_e64: + case AMDGPU::V_MAC_F32_e64: + case AMDGPU::V_MAD_F16_e64: + case AMDGPU::V_MAC_F16_e64: + case AMDGPU::V_FMA_F32_e64: + case AMDGPU::V_FMAC_F32_e64: + case AMDGPU::V_FMA_F16_e64: + case AMDGPU::V_FMAC_F16_e64: + case AMDGPU::V_FMAC_F16_t16_e64: + case AMDGPU::V_FMAC_F16_fake16_e64: + case AMDGPU::V_FMA_F64_e64: + case AMDGPU::V_FMAC_F64_e64: + return true; + default: + return false; + } +} + bool SIInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg, MachineRegisterInfo *MRI) const { int64_t Imm; @@ -3694,16 +3716,10 @@ bool SIInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, return true; } - if (HasMultipleUses) - return false; + if (isMADFMA(Opc)) { + if (HasMultipleUses) + return false; - if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 || - Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 || - Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 || - Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 || - Opc == AMDGPU::V_FMAC_F16_t16_e64 || - Opc == AMDGPU::V_FMAC_F16_fake16_e64 || Opc == AMDGPU::V_FMA_F64_e64 || - Opc == AMDGPU::V_FMAC_F64_e64) { // Don't fold if we are using source or output modifiers. The new VOP2 // instructions don't have them. if (hasAnyModifiersSet(UseMI)) @@ -3899,9 +3915,41 @@ bool SIInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, return true; } + + return false; } - return false; + // Early exit for generic instructions which will never fold an immediate. + if (!isTargetSpecificOpcode(UseMI.getOpcode())) + return false; + + // Directly fold inline immediates into the uses. These should be free-ish + // regardless of the uses. + bool FoldedInlineImm = false; + + for (MachineOperand &UseMO : UseMI.explicit_uses()) { + if (!UseMO.isReg() || UseMO.getReg() != Reg) + continue; + + unsigned UseOpIdx = UseMO.getOperandNo(); + + int64_t ImmVal = Imm; + if (unsigned UseSubReg = UseMO.getSubReg()) { + std::optional<int64_t> SubImm = extractSubregFromImm(Imm, UseSubReg); + if (!SubImm) + continue; + ImmVal = *SubImm; + } + + if (!isInlineConstant(UseMI, UseOpIdx, ImmVal)) + continue; + + UseMO.ChangeToImmediate(ImmVal); + FoldedInlineImm = true; + } + + // TODO: Consider applying tryConstantFoldOp here + return FoldedInlineImm; } static bool diff --git a/llvm/test/CodeGen/AMDGPU/optimize-compare.mir b/llvm/test/CodeGen/AMDGPU/optimize-compare.mir index ef9f4498d7b33..671a1d89711e5 100644 --- a/llvm/test/CodeGen/AMDGPU/optimize-compare.mir +++ b/llvm/test/CodeGen/AMDGPU/optimize-compare.mir @@ -2476,7 +2476,7 @@ body: | ; GCN-NEXT: {{ $}} ; GCN-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0 ; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1 - ; GCN-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], [[S_MOV_B32_]], implicit-def $scc + ; GCN-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], 1, implicit-def $scc ; GCN-NEXT: S_ENDPGM 0 %0:sreg_32 = COPY $sgpr0 %1:sreg_32 = S_MOV_B32 1 @@ -2512,7 +2512,7 @@ body: | ; GCN-NEXT: {{ $}} ; GCN-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0 ; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1 - ; GCN-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[S_MOV_B32_]], [[COPY]], implicit-def $scc + ; GCN-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 1, [[COPY]], implicit-def $scc ; GCN-NEXT: S_ENDPGM 0 %0:sreg_32 = COPY $sgpr0 %1:sreg_32 = S_MOV_B32 1 @@ -2549,7 +2549,7 @@ body: | ; GCN-NEXT: {{ $}} ; GCN-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0 ; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 3 - ; GCN-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], [[S_MOV_B32_]], implicit-def $scc + ; GCN-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], 3, implicit-def $scc ; GCN-NEXT: S_CMP_EQ_U32 killed [[S_ADD_U32_]], 0, implicit-def $scc ; GCN-NEXT: S_ENDPGM 0 %0:sreg_32 = COPY $sgpr0 @@ -2587,7 +2587,7 @@ body: | ; GCN-NEXT: {{ $}} ; GCN-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0 ; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4 - ; GCN-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[S_MOV_B32_]], [[COPY]], implicit-def $scc + ; GCN-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 4, [[COPY]], implicit-def $scc ; GCN-NEXT: S_CMP_EQ_U32 killed [[S_ADD_U32_]], 0, implicit-def $scc ; GCN-NEXT: S_ENDPGM 0 %0:sreg_32 = COPY $sgpr0 diff --git a/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir b/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir index e4fe3b951cf98..273f532a59359 100644 --- a/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir +++ b/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir @@ -437,9 +437,9 @@ name: fold_v_mov_b64_pseudo_64_to_unaligned body: | bb.0: ; GCN-LABEL: name: fold_v_mov_b64_pseudo_64_to_unaligned - ; GCN: [[V_MOV_B64_PSEUDO:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 1311768467750121200, implicit $exec - ; GCN-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 1311768467750121200, implicit $exec - ; GCN-NEXT: SI_RETURN_TO_EPILOG implicit [[V_MOV_B]] + ; GCN: [[V_MOV_B:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 1311768467750121200, implicit $exec + ; GCN-NEXT: [[V_MOV_B1:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 1311768467750121200, implicit $exec + ; GCN-NEXT: SI_RETURN_TO_EPILOG implicit [[V_MOV_B1]] %0:vreg_64_align2 = V_MOV_B64_PSEUDO 1311768467750121200, implicit $exec %1:vreg_64 = COPY killed %0 SI_RETURN_TO_EPILOG implicit %1 @@ -872,13 +872,13 @@ name: fold_imm_copy_kill_bug tracksRegLiveness: true body: | bb.0: + ; GCN-LABEL: name: fold_imm_copy_kill_bug - ; GCN: %0:vgpr_32 = V_MOV_B32_e32 0, implicit $exec - ; GCN-NEXT: %1:vgpr_32 = V_MOV_B32_e32 1, implicit $exec - ; GCN-NEXT: %3:vreg_64 = REG_SEQUENCE %1, %subreg.sub0, %0, %subreg.sub1 - ; GCN-NEXT: %4:vreg_64 = REG_SEQUENCE %1, %subreg.sub0, %0, %subreg.sub1 + ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + ; GCN-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec + ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_1]], %subreg.sub0, [[V_MOV_B32_e32_]], %subreg.sub1 + ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_1]], %subreg.sub0, [[V_MOV_B32_e32_]], %subreg.sub1 ; GCN-NEXT: S_ENDPGM 0 - %0:vgpr_32 = V_MOV_B32_e32 0, implicit $exec %1:vgpr_32 = V_MOV_B32_e32 1, implicit $exec @@ -890,3 +890,88 @@ body: | S_ENDPGM 0 ... + +--- +name: fold_s_mov_b32_inline_imm_valu_inst_0 +body: | + bb.0: + liveins: $vgpr0 + + ; GCN-LABEL: name: fold_s_mov_b32_inline_imm_valu_inst_0 + ; GCN: liveins: $vgpr0 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1 + ; GCN-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], 1, 0, implicit $exec + ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_ADD_U32_e64_]] + %0:vgpr_32 = COPY $vgpr0 + %1:sreg_32 = S_MOV_B32 1 + %2:vgpr_32 = V_ADD_U32_e64 %0, %1, 0, implicit $exec + SI_RETURN_TO_EPILOG %2 + +... + +--- +name: fold_s_mov_b32_inline_imm_valu_inst_1 +body: | + bb.0: + liveins: $vgpr0 + + ; GCN-LABEL: name: fold_s_mov_b32_inline_imm_valu_inst_1 + ; GCN: liveins: $vgpr0 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1 + ; GCN-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 1, [[COPY]], 0, implicit $exec + ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_ADD_U32_e64_]] + %0:vgpr_32 = COPY $vgpr0 + %1:sreg_32 = S_MOV_B32 1 + %2:vgpr_32 = V_ADD_U32_e64 %1, %0, 0, implicit $exec + SI_RETURN_TO_EPILOG %2 + +... + +--- +name: fold_s_mov_b32_inline_imm_valu_inst_with_multi_uses +body: | + bb.0: + + ; GCN-LABEL: name: fold_s_mov_b32_inline_imm_valu_inst_with_multi_uses + ; GCN: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1 + ; GCN-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 1, 1, 0, implicit $exec + ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_ADD_U32_e64_]] + %0:sreg_32 = S_MOV_B32 1 + %1:vgpr_32 = V_ADD_U32_e64 %0, %0, 0, implicit $exec + SI_RETURN_TO_EPILOG %1 + +... + +--- +name: fold_s_mov_b64_inline_imm_valu_inst_with_multi_uses +body: | + bb.0: + + ; GCN-LABEL: name: fold_s_mov_b64_inline_imm_valu_inst_with_multi_uses + ; GCN: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 1 + ; GCN-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 1, 0, 0, implicit $exec + ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_ADD_U32_e64_]] + %0:sreg_64 = S_MOV_B64 1 + %1:vgpr_32 = V_ADD_U32_e64 %0.sub0, %0.sub1, 0, implicit $exec + SI_RETURN_TO_EPILOG %1 + +... + +--- +name: fold_s_mov_b64_imm_subregs_partial_inline_imm_one_inst +body: | + bb.0: + + ; GCN-LABEL: name: fold_s_mov_b64_imm_subregs_partial_inline_imm_one_inst + ; GCN: [[S_MOV_B:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO 4096 + ; GCN-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[S_MOV_B]].sub0, 0, 0, implicit $exec + ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_ADD_U32_e64_]] + %0:sreg_64 = S_MOV_B64_IMM_PSEUDO 4096 + %1:vgpr_32 = V_ADD_U32_e64 %0.sub0, %0.sub1, 0, implicit $exec + SI_RETURN_TO_EPILOG %1 + +... diff --git a/llvm/test/CodeGen/AMDGPU/sdiv64.ll b/llvm/test/CodeGen/AMDGPU/sdiv64.ll index 9554a03942579..f92ea3f2519fc 100644 --- a/llvm/test/CodeGen/AMDGPU/sdiv64.ll +++ b/llvm/test/CodeGen/AMDGPU/sdiv64.ll @@ -2403,9 +2403,8 @@ define i64 @v_test_sdiv24_pow2_k_num_i64(i64 %x) { ; GCN-NEXT: v_max_i32_e32 v0, v2, v0 ; GCN-NEXT: v_cvt_f32_u32_e32 v2, v0 ; GCN-NEXT: v_sub_i32_e32 v3, vcc, 0, v0 -; GCN-NEXT: s_mov_b32 s4, 0x8000 -; GCN-NEXT: v_rcp_f32_e32 v2, v2 ; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v1 +; GCN-NEXT: v_rcp_f32_e32 v2, v2 ; GCN-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 ; GCN-NEXT: v_cvt_u32_f32_e32 v2, v2 ; GCN-NEXT: v_mul_lo_u32 v3, v3, v2 @@ -2414,7 +2413,7 @@ define i64 @v_test_sdiv24_pow2_k_num_i64(i64 %x) { ; GCN-NEXT: v_lshrrev_b32_e32 v2, 17, v2 ; GCN-NEXT: v_mul_u32_u24_e32 v3, v2, v0 ; GCN-NEXT: v_add_i32_e32 v4, vcc, 1, v2 -; GCN-NEXT: v_sub_i32_e32 v3, vcc, s4, v3 +; GCN-NEXT: v_sub_i32_e32 v3, vcc, 0x8000, v3 ; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 ; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc ; GCN-NEXT: v_sub_i32_e64 v4, s[4:5], v3, v0 @@ -2435,9 +2434,8 @@ define i64 @v_test_sdiv24_pow2_k_num_i64(i64 %x) { ; GCN-IR-NEXT: v_max_i32_e32 v0, v2, v0 ; GCN-IR-NEXT: v_cvt_f32_u32_e32 v2, v0 ; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, 0, v0 -; GCN-IR-NEXT: s_mov_b32 s4, 0x8000 -; GCN-IR-NEXT: v_rcp_f32_e32 v2, v2 ; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v1 +; GCN-IR-NEXT: v_rcp_f32_e32 v2, v2 ; GCN-IR-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 ; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v2 ; GCN-IR-NEXT: v_mul_lo_u32 v3, v3, v2 @@ -2446,7 +2444,7 @@ define i64 @v_test_sdiv24_pow2_k_num_i64(i64 %x) { ; GCN-IR-NEXT: v_lshrrev_b32_e32 v2, 17, v2 ; GCN-IR-NEXT: v_mul_u32_u24_e32 v3, v2, v0 ; GCN-IR-NEXT: v_add_i32_e32 v4, vcc, 1, v2 -; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, s4, v3 +; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, 0x8000, v3 ; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 ; GCN-IR-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc ; GCN-IR-NEXT: v_sub_i32_e64 v4, s[4:5], v3, v0 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
