https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/242072
>From dd98b04406aafd1e064ae0a2fc8ca570f738d9b3 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorov <[email protected]> Date: Thu, 12 Nov 2026 18:06:42 +0100 Subject: [PATCH 1/3] [AMDGPU] Prefer fma over fmac for f16 operands in the high half Without real true16 instructions fmac has no op_sel, so a high half operand needs a shift. v_fma_f16 reads it in place. --- llvm/lib/Target/AMDGPU/AMDGPUGISel.td | 4 + llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 10 + llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h | 1 + .../AMDGPU/AMDGPUInstructionSelector.cpp | 11 ++ .../Target/AMDGPU/AMDGPUInstructionSelector.h | 1 + llvm/lib/Target/AMDGPU/SIInstrInfo.td | 2 + llvm/lib/Target/AMDGPU/SIInstructions.td | 18 +- llvm/test/CodeGen/AMDGPU/fma.f16.ll | 177 ++++++------------ llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll | 8 +- .../AMDGPU/fmul-2-combine-multi-use.ll | 6 +- llvm/test/CodeGen/AMDGPU/strict_fma.f16.ll | 24 +-- 11 files changed, 108 insertions(+), 154 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td index 3c9c596b9c652..e7070294c3c25 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td @@ -39,6 +39,10 @@ def gi_vop3_no_mods : GIComplexOperandMatcher<s32, "selectVOP3NoMods">, GIComplexPatternEquiv<VOP3NoMods>; +def gi_vop3_no_op_sel_mods : + GIComplexOperandMatcher<s32, "selectVOP3NoOpSelMods">, + GIComplexPatternEquiv<VOP3NoOpSelMods>; + def gi_vop3omods : GIComplexOperandMatcher<s32, "selectVOP3OMods">, GIComplexPatternEquiv<VOP3OMods>; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp index afd8be7337742..eb8e342f2c33b 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp @@ -3680,6 +3680,16 @@ bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const { return true; } +bool AMDGPUDAGToDAGISel::SelectVOP3NoOpSelMods(SDValue In, + SDValue &Src) const { + SDValue Unused; + if (!Subtarget->useRealTrue16Insts() && In.getValueSizeInBits() == 16 && + isExtractHiElt(In, Unused)) + return false; + + return SelectVOP3NoMods(In, Src); +} + bool AMDGPUDAGToDAGISel::SelectVINTERPModsImpl(SDValue In, SDValue &Src, SDValue &SrcMods, bool OpSel) const { diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h index ca587544b0ea6..f97f2e8645063 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h @@ -227,6 +227,7 @@ class AMDGPUDAGToDAGISel : public SelectionDAGISel { SDValue &SrcMods) const; bool SelectVOP3BMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; bool SelectVOP3NoMods(SDValue In, SDValue &Src) const; + bool SelectVOP3NoOpSelMods(SDValue In, SDValue &Src) const; bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods, SDValue &Clamp, SDValue &Omod) const; bool SelectVOP3BMods0(SDValue In, SDValue &Src, SDValue &SrcMods, diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp index c0c90b3fcd1c6..c7af406dab056 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp @@ -4948,6 +4948,17 @@ AMDGPUInstructionSelector::selectVOP3NoMods(MachineOperand &Root) const { }}; } +InstructionSelector::ComplexRendererFns +AMDGPUInstructionSelector::selectVOP3NoOpSelMods(MachineOperand &Root) const { + Register Reg = Root.getReg(); + Register Unused; + if (!Subtarget->useRealTrue16Insts() && + MRI->getType(Reg).getSizeInBits() == 16 && + isExtractHiElt(*MRI, Reg, Unused)) + return {}; + return selectVOP3NoMods(Root); +} + enum class SrcStatus { IS_SAME, IS_UPPER_HALF, diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h index b12ac778fd085..f7b9c29745570 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h @@ -191,6 +191,7 @@ class AMDGPUInstructionSelector final : public InstructionSelector { selectVOP3BMods(MachineOperand &Root) const; ComplexRendererFns selectVOP3NoMods(MachineOperand &Root) const; + ComplexRendererFns selectVOP3NoOpSelMods(MachineOperand &Root) const; std::pair<Register, unsigned> selectVOP3PModsImpl(Register RootReg, const MachineRegisterInfo &MRI, diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td index c4dfed816f10a..1b1254d01b297 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td @@ -1754,6 +1754,8 @@ def VOP3ModsNonCanonicalizing : ComplexPattern<untyped, 2, def VOP3NoMods : ComplexPattern<untyped, 1, "SelectVOP3NoMods">; +def VOP3NoOpSelMods : ComplexPattern<untyped, 1, "SelectVOP3NoOpSelMods">; + def VOP3OMods : ComplexPattern<untyped, 3, "SelectVOP3OMods">; def VOP3PMods : ComplexPattern<untyped, 2, "SelectVOP3PMods">; diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td index c8c42b2809caa..4bd62c6c2107e 100644 --- a/llvm/lib/Target/AMDGPU/SIInstructions.td +++ b/llvm/lib/Target/AMDGPU/SIInstructions.td @@ -3945,25 +3945,25 @@ let SubtargetPredicate = isGFX10Plus in { // better to select fma instead of fmac. let True16Predicate = NotHasTrue16BitInsts in def : GCNPat < - (fma (f16 (VOP3NoMods f32:$src0)), - (f16 (VOP3NoMods f32:$src1)), - (f16 (VOP3NoMods f32:$src2))), + (fma (f16 (VOP3NoOpSelMods f32:$src0)), + (f16 (VOP3NoOpSelMods f32:$src1)), + (f16 (VOP3NoOpSelMods f32:$src2))), (V_FMAC_F16_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1, SRCMODS.NONE, $src2) >; let True16Predicate = UseRealTrue16Insts in def : GCNPat < - (fma (f16 (VOP3NoMods f16:$src0)), - (f16 (VOP3NoMods f16:$src1)), - (f16 (VOP3NoMods f16:$src2))), + (fma (f16 (VOP3NoOpSelMods f16:$src0)), + (f16 (VOP3NoOpSelMods f16:$src1)), + (f16 (VOP3NoOpSelMods f16:$src2))), (V_FMAC_F16_t16_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1, SRCMODS.NONE, $src2) >; let True16Predicate = UseFakeTrue16Insts in def : GCNPat < - (fma (f16 (VOP3NoMods f16:$src0)), - (f16 (VOP3NoMods f16:$src1)), - (f16 (VOP3NoMods f16:$src2))), + (fma (f16 (VOP3NoOpSelMods f16:$src0)), + (f16 (VOP3NoOpSelMods f16:$src1)), + (f16 (VOP3NoOpSelMods f16:$src2))), (V_FMAC_F16_fake16_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1, SRCMODS.NONE, $src2) >; diff --git a/llvm/test/CodeGen/AMDGPU/fma.f16.ll b/llvm/test/CodeGen/AMDGPU/fma.f16.ll index b517ad5f7eee1..27cec03324f64 100644 --- a/llvm/test/CodeGen/AMDGPU/fma.f16.ll +++ b/llvm/test/CodeGen/AMDGPU/fma.f16.ll @@ -796,8 +796,7 @@ define half @test_fma_hi_src0(<2 x half> %x, half %y, half %z) { ; GFX10-LABEL: test_fma_hi_src0: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX10-NEXT: v_fma_f16 v0, v0, v1, v2 +; GFX10-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0] ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-SDAG-TRUE16-LABEL: test_fma_hi_src0: @@ -809,9 +808,7 @@ define half @test_fma_hi_src0(<2 x half> %x, half %y, half %z) { ; GFX11-SDAG-FAKE16-LABEL: test_fma_hi_src0: ; GFX11-SDAG-FAKE16: ; %bb.0: ; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 +; GFX11-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0] ; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-GISEL-TRUE16-LABEL: test_fma_hi_src0: @@ -823,9 +820,7 @@ define half @test_fma_hi_src0(<2 x half> %x, half %y, half %z) { ; GFX11-GISEL-FAKE16-LABEL: test_fma_hi_src0: ; GFX11-GISEL-FAKE16: ; %bb.0: ; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 +; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0] ; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-SDAG-TRUE16-LABEL: test_fma_hi_src0: @@ -845,9 +840,7 @@ define half @test_fma_hi_src0(<2 x half> %x, half %y, half %z) { ; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 +; GFX12-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0] ; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-GISEL-TRUE16-LABEL: test_fma_hi_src0: @@ -867,9 +860,7 @@ define half @test_fma_hi_src0(<2 x half> %x, half %y, half %z) { ; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 +; GFX12-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0] ; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] %x1 = extractelement <2 x half> %x, i32 1 %r = call half @llvm.fma.f16(half %x1, half %y, half %z) @@ -886,10 +877,7 @@ define half @test_fma_hi_srcs(<2 x half> %x, <2 x half> %y, <2 x half> %z) { ; GFX10-LABEL: test_fma_hi_srcs: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v0 -; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v2 -; GFX10-NEXT: v_fmac_f16_e32 v0, v3, v1 +; GFX10-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,1,1,0] ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-SDAG-TRUE16-LABEL: test_fma_hi_srcs: @@ -901,11 +889,7 @@ define half @test_fma_hi_srcs(<2 x half> %x, <2 x half> %y, <2 x half> %z) { ; GFX11-SDAG-FAKE16-LABEL: test_fma_hi_srcs: ; GFX11-SDAG-FAKE16: ; %bb.0: ; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0 -; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v2 -; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-SDAG-FAKE16-NEXT: v_fmac_f16_e32 v0, v3, v1 +; GFX11-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,1,1,0] ; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-GISEL-TRUE16-LABEL: test_fma_hi_srcs: @@ -917,11 +901,7 @@ define half @test_fma_hi_srcs(<2 x half> %x, <2 x half> %y, <2 x half> %z) { ; GFX11-GISEL-FAKE16-LABEL: test_fma_hi_srcs: ; GFX11-GISEL-FAKE16: ; %bb.0: ; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0 -; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v2 -; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-GISEL-FAKE16-NEXT: v_fmac_f16_e32 v0, v3, v1 +; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,1,1,0] ; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-SDAG-TRUE16-LABEL: test_fma_hi_srcs: @@ -941,11 +921,7 @@ define half @test_fma_hi_srcs(<2 x half> %x, <2 x half> %y, <2 x half> %z) { ; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0 -; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v2 -; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-SDAG-FAKE16-NEXT: v_fmac_f16_e32 v0, v3, v1 +; GFX12-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,1,1,0] ; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-GISEL-TRUE16-LABEL: test_fma_hi_srcs: @@ -965,11 +941,7 @@ define half @test_fma_hi_srcs(<2 x half> %x, <2 x half> %y, <2 x half> %z) { ; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0 -; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v2 -; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-GISEL-FAKE16-NEXT: v_fmac_f16_e32 v0, v3, v1 +; GFX12-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,1,1,0] ; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] %x1 = extractelement <2 x half> %x, i32 1 %y1 = extractelement <2 x half> %y, i32 1 @@ -989,9 +961,8 @@ define half @test_fma_hi_src0_two_uses(<2 x half> %x, half %y, half %z) { ; GFX10-LABEL: test_fma_hi_src0_two_uses: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v0 -; GFX10-NEXT: v_fma_f16 v0, v3, v1, v2 -; GFX10-NEXT: v_fmac_f16_e32 v0, v3, v2 +; GFX10-NEXT: v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0] +; GFX10-NEXT: v_fma_f16 v0, v0, v2, v1 op_sel:[1,0,0,0] ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-SDAG-TRUE16-LABEL: test_fma_hi_src0_two_uses: @@ -1005,10 +976,9 @@ define half @test_fma_hi_src0_two_uses(<2 x half> %x, half %y, half %z) { ; GFX11-SDAG-FAKE16-LABEL: test_fma_hi_src0_two_uses: ; GFX11-SDAG-FAKE16: ; %bb.0: ; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0 -; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-FAKE16-NEXT: v_fma_f16 v0, v3, v1, v2 -; GFX11-SDAG-FAKE16-NEXT: v_fmac_f16_e32 v0, v3, v2 +; GFX11-SDAG-FAKE16-NEXT: v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0] +; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v2, v1 op_sel:[1,0,0,0] ; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-GISEL-TRUE16-LABEL: test_fma_hi_src0_two_uses: @@ -1022,10 +992,9 @@ define half @test_fma_hi_src0_two_uses(<2 x half> %x, half %y, half %z) { ; GFX11-GISEL-FAKE16-LABEL: test_fma_hi_src0_two_uses: ; GFX11-GISEL-FAKE16: ; %bb.0: ; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0 -; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v0, v3, v1, v2 -; GFX11-GISEL-FAKE16-NEXT: v_fmac_f16_e32 v0, v3, v2 +; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0] +; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, v2, v1 op_sel:[1,0,0,0] ; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-SDAG-TRUE16-LABEL: test_fma_hi_src0_two_uses: @@ -1047,10 +1016,9 @@ define half @test_fma_hi_src0_two_uses(<2 x half> %x, half %y, half %z) { ; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0 -; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-SDAG-FAKE16-NEXT: v_fma_f16 v0, v3, v1, v2 -; GFX12-SDAG-FAKE16-NEXT: v_fmac_f16_e32 v0, v3, v2 +; GFX12-SDAG-FAKE16-NEXT: v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0] +; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v2, v1 op_sel:[1,0,0,0] ; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-GISEL-TRUE16-LABEL: test_fma_hi_src0_two_uses: @@ -1072,10 +1040,9 @@ define half @test_fma_hi_src0_two_uses(<2 x half> %x, half %y, half %z) { ; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0 -; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-GISEL-FAKE16-NEXT: v_fma_f16 v0, v3, v1, v2 -; GFX12-GISEL-FAKE16-NEXT: v_fmac_f16_e32 v0, v3, v2 +; GFX12-GISEL-FAKE16-NEXT: v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0] +; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, v2, v1 op_sel:[1,0,0,0] ; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] %x1 = extractelement <2 x half> %x, i32 1 %r0 = call half @llvm.fma.f16(half %x1, half %y, half %z) @@ -1093,9 +1060,7 @@ define half @test_fma_hi_src2(half %x, half %y, <2 x half> %z) { ; GFX10-LABEL: test_fma_hi_src2: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX10-NEXT: v_fmac_f16_e32 v2, v0, v1 -; GFX10-NEXT: v_mov_b32_e32 v0, v2 +; GFX10-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[0,0,1,0] ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-SDAG-TRUE16-LABEL: test_fma_hi_src2: @@ -1107,10 +1072,7 @@ define half @test_fma_hi_src2(half %x, half %y, <2 x half> %z) { ; GFX11-SDAG-FAKE16-LABEL: test_fma_hi_src2: ; GFX11-SDAG-FAKE16: ; %bb.0: ; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-SDAG-FAKE16-NEXT: v_fmac_f16_e32 v2, v0, v1 -; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, v2 +; GFX11-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[0,0,1,0] ; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-GISEL-TRUE16-LABEL: test_fma_hi_src2: @@ -1122,10 +1084,7 @@ define half @test_fma_hi_src2(half %x, half %y, <2 x half> %z) { ; GFX11-GISEL-FAKE16-LABEL: test_fma_hi_src2: ; GFX11-GISEL-FAKE16: ; %bb.0: ; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-FAKE16-NEXT: v_fmac_f16_e32 v2, v0, v1 -; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, v2 +; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[0,0,1,0] ; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-SDAG-TRUE16-LABEL: test_fma_hi_src2: @@ -1145,10 +1104,7 @@ define half @test_fma_hi_src2(half %x, half %y, <2 x half> %z) { ; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-SDAG-FAKE16-NEXT: v_fmac_f16_e32 v2, v0, v1 -; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, v2 +; GFX12-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[0,0,1,0] ; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-GISEL-TRUE16-LABEL: test_fma_hi_src2: @@ -1168,10 +1124,7 @@ define half @test_fma_hi_src2(half %x, half %y, <2 x half> %z) { ; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-GISEL-FAKE16-NEXT: v_fmac_f16_e32 v2, v0, v1 -; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, v2 +; GFX12-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[0,0,1,0] ; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] %z1 = extractelement <2 x half> %z, i32 1 %r = call half @llvm.fma.f16(half %x, half %y, half %z1) @@ -1200,7 +1153,7 @@ define half @test_fma_hi_src0_stored(<2 x half> %x, half %y, half %z, ptr %p) { ; GFX10-SDAG: ; %bb.0: ; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v0 -; GFX10-SDAG-NEXT: v_fma_f16 v0, v5, v1, v2 +; GFX10-SDAG-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0] ; GFX10-SDAG-NEXT: flat_store_short v[3:4], v5 ; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31] @@ -1208,9 +1161,8 @@ define half @test_fma_hi_src0_stored(<2 x half> %x, half %y, half %z, ptr %p) { ; GFX10-GISEL-LABEL: test_fma_hi_src0_stored: ; GFX10-GISEL: ; %bb.0: ; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v0 +; GFX10-GISEL-NEXT: v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0] ; GFX10-GISEL-NEXT: flat_store_short_d16_hi v[3:4], v0 -; GFX10-GISEL-NEXT: v_fma_f16 v1, v5, v1, v2 ; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, v1 ; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] @@ -1227,8 +1179,7 @@ define half @test_fma_hi_src0_stored(<2 x half> %x, half %y, half %z, ptr %p) { ; GFX11-SDAG-FAKE16: ; %bb.0: ; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0 -; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-SDAG-FAKE16-NEXT: v_fma_f16 v0, v5, v1, v2 +; GFX11-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0] ; GFX11-SDAG-FAKE16-NEXT: flat_store_b16 v[3:4], v5 ; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] @@ -1247,10 +1198,8 @@ define half @test_fma_hi_src0_stored(<2 x half> %x, half %y, half %z, ptr %p) { ; GFX11-GISEL-FAKE16-LABEL: test_fma_hi_src0_stored: ; GFX11-GISEL-FAKE16: ; %bb.0: ; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0 +; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0] ; GFX11-GISEL-FAKE16-NEXT: flat_store_d16_hi_b16 v[3:4], v0 -; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v1, v5, v1, v2 -; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, v1 ; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] @@ -1275,8 +1224,7 @@ define half @test_fma_hi_src0_stored(<2 x half> %x, half %y, half %z, ptr %p) { ; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0 -; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-SDAG-FAKE16-NEXT: v_fma_f16 v0, v5, v1, v2 +; GFX12-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, v1, v2 op_sel:[1,0,0,0] ; GFX12-SDAG-FAKE16-NEXT: flat_store_b16 v[3:4], v5 ; GFX12-SDAG-FAKE16-NEXT: s_wait_dscnt 0x0 ; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] @@ -1303,10 +1251,8 @@ define half @test_fma_hi_src0_stored(<2 x half> %x, half %y, half %z, ptr %p) { ; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0 +; GFX12-GISEL-FAKE16-NEXT: v_fma_f16 v1, v0, v1, v2 op_sel:[1,0,0,0] ; GFX12-GISEL-FAKE16-NEXT: flat_store_d16_hi_b16 v[3:4], v0 -; GFX12-GISEL-FAKE16-NEXT: v_fma_f16 v1, v5, v1, v2 -; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, v1 ; GFX12-GISEL-FAKE16-NEXT: s_wait_dscnt 0x0 ; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] @@ -1334,8 +1280,7 @@ define half @test_fma_hi_src0_literal_src1(<2 x half> %x, half %z) { ; GFX10-LABEL: test_fma_hi_src0_literal_src1: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX10-NEXT: v_fmamk_f16 v0, v0, 0x6400, v1 +; GFX10-NEXT: v_fma_f16 v0, v0, 0x6400, v1 op_sel:[1,0,0,0] ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-SDAG-TRUE16-LABEL: test_fma_hi_src0_literal_src1: @@ -1347,9 +1292,7 @@ define half @test_fma_hi_src0_literal_src1(<2 x half> %x, half %z) { ; GFX11-SDAG-FAKE16-LABEL: test_fma_hi_src0_literal_src1: ; GFX11-SDAG-FAKE16: ; %bb.0: ; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-SDAG-FAKE16-NEXT: v_fmamk_f16 v0, v0, 0x6400, v1 +; GFX11-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, 0x6400, v1 op_sel:[1,0,0,0] ; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-GISEL-TRUE16-LABEL: test_fma_hi_src0_literal_src1: @@ -1361,9 +1304,7 @@ define half @test_fma_hi_src0_literal_src1(<2 x half> %x, half %z) { ; GFX11-GISEL-FAKE16-LABEL: test_fma_hi_src0_literal_src1: ; GFX11-GISEL-FAKE16: ; %bb.0: ; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-GISEL-FAKE16-NEXT: v_fmamk_f16 v0, v0, 0x6400, v1 +; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, 0x6400, v1 op_sel:[1,0,0,0] ; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-SDAG-TRUE16-LABEL: test_fma_hi_src0_literal_src1: @@ -1383,9 +1324,7 @@ define half @test_fma_hi_src0_literal_src1(<2 x half> %x, half %z) { ; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-SDAG-FAKE16-NEXT: v_fmamk_f16 v0, v0, 0x6400, v1 +; GFX12-SDAG-FAKE16-NEXT: v_fma_f16 v0, v0, 0x6400, v1 op_sel:[1,0,0,0] ; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-GISEL-TRUE16-LABEL: test_fma_hi_src0_literal_src1: @@ -1405,9 +1344,7 @@ define half @test_fma_hi_src0_literal_src1(<2 x half> %x, half %z) { ; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0 ; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0 -; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-GISEL-FAKE16-NEXT: v_fmamk_f16 v0, v0, 0x6400, v1 +; GFX12-GISEL-FAKE16-NEXT: v_fma_f16 v0, v0, 0x6400, v1 op_sel:[1,0,0,0] ; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] %x1 = extractelement <2 x half> %x, i32 1 %r = call half @llvm.fma.f16(half %x1, half 1024.0, half %z) @@ -1460,14 +1397,14 @@ define half @test_fma_hi_src0_other_block(<2 x half> %x, half %y, half %z, i1 %c ; GFX10-GISEL: ; %bb.0: ; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 1, v3 -; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX10-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3 +; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v0 ; GFX10-GISEL-NEXT: s_and_saveexec_b32 s4, vcc_lo ; GFX10-GISEL-NEXT: ; %bb.1: ; %fma -; GFX10-GISEL-NEXT: v_fmac_f16_e32 v2, v0, v1 -; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, v2 +; GFX10-GISEL-NEXT: v_fma_f16 v3, v0, v1, v2 op_sel:[1,0,0,0] ; GFX10-GISEL-NEXT: ; %bb.2: ; %UnifiedReturnBlock ; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s4 +; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, v3 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-SDAG-TRUE16-LABEL: test_fma_hi_src0_other_block: @@ -1525,16 +1462,16 @@ define half @test_fma_hi_src0_other_block(<2 x half> %x, half %y, half %z, i1 %c ; GFX11-GISEL-FAKE16: ; %bb.0: ; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v3, 1, v3 -; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s0, exec_lo -; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-GISEL-FAKE16-NEXT: v_cmpx_ne_u32_e32 0, v3 +; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3 +; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0 +; GFX11-GISEL-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX11-GISEL-FAKE16-NEXT: ; %bb.1: ; %fma -; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-FAKE16-NEXT: v_fmac_f16_e32 v2, v0, v1 -; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, v2 +; GFX11-GISEL-FAKE16-NEXT: v_fma_f16 v3, v0, v1, v2 op_sel:[1,0,0,0] ; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %UnifiedReturnBlock ; GFX11-GISEL-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, v3 ; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-SDAG-TRUE16-LABEL: test_fma_hi_src0_other_block: @@ -1611,17 +1548,17 @@ define half @test_fma_hi_src0_other_block(<2 x half> %x, half %y, half %z, i1 %c ; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0 ; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0 ; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v3, 1, v3 -; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s0, exec_lo -; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX12-GISEL-FAKE16-NEXT: v_cmpx_ne_u32_e32 0, v3 +; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3 +; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0 +; GFX12-GISEL-FAKE16-NEXT: s_and_saveexec_b32 s0, vcc_lo ; GFX12-GISEL-FAKE16-NEXT: ; %bb.1: ; %fma -; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-GISEL-FAKE16-NEXT: v_fmac_f16_e32 v2, v0, v1 -; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, v2 +; GFX12-GISEL-FAKE16-NEXT: v_fma_f16 v3, v0, v1, v2 op_sel:[1,0,0,0] ; GFX12-GISEL-FAKE16-NEXT: ; %bb.2: ; %UnifiedReturnBlock ; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0) ; GFX12-GISEL-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0 +; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, v3 ; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] %x1 = extractelement <2 x half> %x, i32 1 br i1 %c, label %fma, label %exit diff --git a/llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll b/llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll index ed62bf07ecd25..96ac9557f78dc 100644 --- a/llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll +++ b/llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll @@ -1,9 +1,11 @@ ; RUN: llc -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GCN,GFX1010 %s -; RUN: llc -mtriple=amdgpu11.00 < %s | FileCheck -check-prefixes=GCN,GFX1010 %s +; RUN: llc -mtriple=amdgpu11.00 < %s | FileCheck -check-prefixes=GCN,GFX1100 %s ; GCN-LABEL: {{^}}addMul2D: -; GFX1010: v_fmac_f16 -; GFX1010: v_fmac_f16 +; GFX1010: v_fma_f16 {{.*}} op_sel:[1,0,1,0] +; GFX1010: v_fma_f16 {{.*}} op_sel:[1,0,1,0] +; GFX1100: v_fmac_f16 +; GFX1100: v_fmac_f16 define hidden <4 x half> @addMul2D(ptr nocapture readonly %arg, ptr addrspace(4) nocapture readonly %arg1, <2 x i32> %arg2, i32 %arg3) local_unnamed_addr #0 { bb: %tmp = extractelement <2 x i32> %arg2, i64 1 diff --git a/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll b/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll index b0270a50790aa..f0f4f5627a5a2 100644 --- a/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll +++ b/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll @@ -601,9 +601,8 @@ define amdgpu_kernel void @multiple_use_fadd_fmac_f16(ptr addrspace(1) %out, i16 ; GFX10-DENORM-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; GFX10-DENORM-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-DENORM-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DENORM-NEXT: s_lshr_b32 s3, s2, 16 ; GFX10-DENORM-NEXT: v_add_f16_e64 v1, s2, s2 -; GFX10-DENORM-NEXT: v_fma_f16 v2, s2, 2.0, s3 +; GFX10-DENORM-NEXT: v_fma_f16 v2, s2, 2.0, s2 op_sel:[0,0,1,0] ; GFX10-DENORM-NEXT: global_store_short v0, v1, s[0:1] ; GFX10-DENORM-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX10-DENORM-NEXT: global_store_short v0, v2, s[0:1] offset:2 @@ -649,9 +648,8 @@ define amdgpu_kernel void @multiple_use_fadd_fmac_f16(ptr addrspace(1) %out, i16 ; GFX11-DENORM-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; GFX11-DENORM-FAKE16-NEXT: v_mov_b32_e32 v0, 0 ; GFX11-DENORM-FAKE16-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-DENORM-FAKE16-NEXT: s_lshr_b32 s3, s2, 16 ; GFX11-DENORM-FAKE16-NEXT: v_add_f16_e64 v1, s2, s2 -; GFX11-DENORM-FAKE16-NEXT: v_fma_f16 v2, s2, 2.0, s3 +; GFX11-DENORM-FAKE16-NEXT: v_fma_f16 v2, s2, 2.0, s2 op_sel:[0,0,1,0] ; GFX11-DENORM-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1] dlc ; GFX11-DENORM-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX11-DENORM-FAKE16-NEXT: global_store_b16 v0, v2, s[0:1] offset:2 dlc diff --git a/llvm/test/CodeGen/AMDGPU/strict_fma.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fma.f16.ll index 9c9d6e01e5c06..65e8c1e6d0fc9 100644 --- a/llvm/test/CodeGen/AMDGPU/strict_fma.f16.ll +++ b/llvm/test/CodeGen/AMDGPU/strict_fma.f16.ll @@ -148,18 +148,12 @@ define <4 x half> @v_constained_fma_v4f16_fpexcept_strict(<4 x half> %x, <4 x ha ; GFX10-LABEL: v_constained_fma_v4f16_fpexcept_strict: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_lshrrev_b32_e32 v6, 16, v5 -; GFX10-NEXT: v_lshrrev_b32_e32 v7, 16, v3 -; GFX10-NEXT: v_lshrrev_b32_e32 v8, 16, v1 -; GFX10-NEXT: v_lshrrev_b32_e32 v9, 16, v4 -; GFX10-NEXT: v_lshrrev_b32_e32 v10, 16, v2 -; GFX10-NEXT: v_lshrrev_b32_e32 v11, 16, v0 +; GFX10-NEXT: v_fma_f16 v6, v1, v3, v5 op_sel:[1,1,1,0] +; GFX10-NEXT: v_fma_f16 v7, v0, v2, v4 op_sel:[1,1,1,0] ; GFX10-NEXT: v_fmac_f16_e32 v4, v0, v2 -; GFX10-NEXT: v_fmac_f16_e32 v6, v8, v7 ; GFX10-NEXT: v_fmac_f16_e32 v5, v1, v3 -; GFX10-NEXT: v_fmac_f16_e32 v9, v11, v10 +; GFX10-NEXT: v_perm_b32 v0, v7, v4, 0x5040100 ; GFX10-NEXT: v_perm_b32 v1, v6, v5, 0x5040100 -; GFX10-NEXT: v_perm_b32 v0, v9, v4, 0x5040100 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-TRUE16-LABEL: v_constained_fma_v4f16_fpexcept_strict: @@ -175,18 +169,12 @@ define <4 x half> @v_constained_fma_v4f16_fpexcept_strict(<4 x half> %x, <4 x ha ; GFX11-FAKE16-LABEL: v_constained_fma_v4f16_fpexcept_strict: ; GFX11-FAKE16: ; %bb.0: ; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v5 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 16, v1 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v4 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v2 -; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 16, v0 +; GFX11-FAKE16-NEXT: v_fma_f16 v6, v1, v3, v5 op_sel:[1,1,1,0] +; GFX11-FAKE16-NEXT: v_fma_f16 v7, v0, v2, v4 op_sel:[1,1,1,0] ; GFX11-FAKE16-NEXT: v_fmac_f16_e32 v4, v0, v2 -; GFX11-FAKE16-NEXT: v_fmac_f16_e32 v6, v8, v7 ; GFX11-FAKE16-NEXT: v_fmac_f16_e32 v5, v1, v3 -; GFX11-FAKE16-NEXT: v_fmac_f16_e32 v9, v11, v10 +; GFX11-FAKE16-NEXT: v_perm_b32 v0, v7, v4, 0x5040100 ; GFX11-FAKE16-NEXT: v_perm_b32 v1, v6, v5, 0x5040100 -; GFX11-FAKE16-NEXT: v_perm_b32 v0, v9, v4, 0x5040100 ; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31] %val = call <4 x half> @llvm.experimental.constrained.fma.v4f16(<4 x half> %x, <4 x half> %y, <4 x half> %z, metadata !"round.tonearest", metadata !"fpexcept.strict") ret <4 x half> %val >From 86360c1e747767fd4d9f1477b4e933556ec1ad36 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorov <[email protected]> Date: Thu, 12 Nov 2026 23:05:37 +0100 Subject: [PATCH 2/3] format --- llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp index eb8e342f2c33b..7597f9c86f0ec 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp @@ -3680,8 +3680,7 @@ bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const { return true; } -bool AMDGPUDAGToDAGISel::SelectVOP3NoOpSelMods(SDValue In, - SDValue &Src) const { +bool AMDGPUDAGToDAGISel::SelectVOP3NoOpSelMods(SDValue In, SDValue &Src) const { SDValue Unused; if (!Subtarget->useRealTrue16Insts() && In.getValueSizeInBits() == 16 && isExtractHiElt(In, Unused)) >From 03df4e980e0940f569fbba1681ebba8ac6f7e729 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorov <[email protected]> Date: Fri, 13 Nov 2026 00:40:44 +0100 Subject: [PATCH 3/3] apply review --- llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 3 +-- llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp | 5 +---- llvm/lib/Target/AMDGPU/SIInstructions.td | 6 +++--- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp index 7597f9c86f0ec..5b436491cac3f 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp @@ -3682,8 +3682,7 @@ bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const { bool AMDGPUDAGToDAGISel::SelectVOP3NoOpSelMods(SDValue In, SDValue &Src) const { SDValue Unused; - if (!Subtarget->useRealTrue16Insts() && In.getValueSizeInBits() == 16 && - isExtractHiElt(In, Unused)) + if (isExtractHiElt(In, Unused)) return false; return SelectVOP3NoMods(In, Src); diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp index c7af406dab056..4f444b2a8735b 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp @@ -4950,11 +4950,8 @@ AMDGPUInstructionSelector::selectVOP3NoMods(MachineOperand &Root) const { InstructionSelector::ComplexRendererFns AMDGPUInstructionSelector::selectVOP3NoOpSelMods(MachineOperand &Root) const { - Register Reg = Root.getReg(); Register Unused; - if (!Subtarget->useRealTrue16Insts() && - MRI->getType(Reg).getSizeInBits() == 16 && - isExtractHiElt(*MRI, Reg, Unused)) + if (isExtractHiElt(*MRI, Root.getReg(), Unused)) return {}; return selectVOP3NoMods(Root); } diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td index 4bd62c6c2107e..916baa0a978ab 100644 --- a/llvm/lib/Target/AMDGPU/SIInstructions.td +++ b/llvm/lib/Target/AMDGPU/SIInstructions.td @@ -3953,9 +3953,9 @@ def : GCNPat < >; let True16Predicate = UseRealTrue16Insts in def : GCNPat < - (fma (f16 (VOP3NoOpSelMods f16:$src0)), - (f16 (VOP3NoOpSelMods f16:$src1)), - (f16 (VOP3NoOpSelMods f16:$src2))), + (fma (f16 (VOP3NoMods f16:$src0)), + (f16 (VOP3NoMods f16:$src1)), + (f16 (VOP3NoMods f16:$src2))), (V_FMAC_F16_t16_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1, SRCMODS.NONE, $src2) >; _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
