llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-backend-amdgpu Author: Dmitry Sidorov (MrSidims) <details> <summary>Changes</summary> After a <4 x i8> is split into i16 halves, a byte lane is extended from an i16 shift of a truncate. Rewrite it on the 32-bit source as an and of srl or a sign_extend_inreg of srl, which select to a single bit field extract. Uniform zero and any extends are left alone, their i16 shift is already promoted to i32. --- <sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub> --- Patch is 108.34 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/228424.diff 10 Files Affected: - (modified) llvm/lib/Target/AMDGPU/SIISelLowering.cpp (+64) - (modified) llvm/lib/Target/AMDGPU/SIISelLowering.h (+1) - (modified) llvm/test/CodeGen/AMDGPU/idot4-test.ll (+47-62) - (modified) llvm/test/CodeGen/AMDGPU/idot4s.ll (+32-47) - (modified) llvm/test/CodeGen/AMDGPU/idot4u.ll (+107-133) - (modified) llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll (+5-4) - (modified) llvm/test/CodeGen/AMDGPU/min.ll (+77-85) - (modified) llvm/test/CodeGen/AMDGPU/permute_i8.ll (+16-20) - (modified) llvm/test/CodeGen/AMDGPU/sad.ll (+8-10) - (modified) llvm/test/CodeGen/AMDGPU/v4i8-byte-lane-ext.ll (+154-227) ``````````diff diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index be0424dc379b4..318d0e501e732 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -1181,6 +1181,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, ISD::FCANONICALIZE, ISD::SCALAR_TO_VECTOR, ISD::ZERO_EXTEND, + ISD::SIGN_EXTEND, ISD::SIGN_EXTEND_INREG, ISD::ANY_EXTEND, ISD::EXTRACT_VECTOR_ELT, @@ -15061,6 +15062,22 @@ SDValue SITargetLowering::performAndCombine(SDNode *N, } } + // and (trunc (srl x, c)), mask => srl (trunc x), c + // if mask keeps exactly the bits that srl leaves in the low half + if (CRHS && VT == MVT::i16 && LHS.getOpcode() == ISD::TRUNCATE && + LHS.hasOneUse() && LHS.getOperand(0).getOpcode() == ISD::SRL) { + auto *Shift = dyn_cast<ConstantSDNode>(LHS.getOperand(0).getOperand(1)); + if (Shift && Shift->getZExtValue() > 0 && Shift->getZExtValue() < 16 && + CRHS->getAPIntValue().isMask(16 - Shift->getZExtValue())) { + SDLoc SL(N); + SDValue Trunc = + DAG.getNode(ISD::TRUNCATE, SL, VT, LHS.getOperand(0).getOperand(0)); + return DAG.getNode( + ISD::SRL, SL, VT, Trunc, + DAG.getShiftAmountConstant(Shift->getZExtValue(), VT, SL)); + } + } + // (and (fcmp ord x, x), (fcmp une (fabs x), inf)) -> // fp_class x, ~(s_nan | q_nan | n_infinity | p_infinity) if (LHS.getOpcode() == ISD::SETCC && RHS.getOpcode() == ISD::SETCC) { @@ -16061,6 +16078,9 @@ SITargetLowering::performZeroOrAnyExtendCombine(SDNode *N, if (!Src->hasOneUse()) return SDValue(); + if (SDValue Ext = performExtendTruncShiftCombine(N, DCI)) + return Ext; + // TODO: We bail out below if SrcOffset is not in the first dword (>= 4). It's // possible we're missing out on some combine opportunities, but we'd need to // weigh the cost of extracting the byte from the upper dwords. @@ -16098,6 +16118,48 @@ SITargetLowering::performZeroOrAnyExtendCombine(SDNode *N, DAG.getConstant(PermMask, DL, MVT::i32)); } +// zext (srl (trunc x), c) => and (srl x, c), lowbits +// aext (srl (trunc x), c) => and (srl x, c), lowbits +// sext (sra (trunc x), 8) => sext_inreg (srl x, 8), i8 +SDValue +SITargetLowering::performExtendTruncShiftCombine(SDNode *N, + DAGCombinerInfo &DCI) const { + if (DCI.getDAGCombineLevel() < AfterLegalizeTypes) + return SDValue(); + + const unsigned Opc = N->getOpcode(); + const bool IsSigned = Opc == ISD::SIGN_EXTEND; + SDValue Src = N->getOperand(0); + if (N->getValueType(0) != MVT::i32 || Src.getValueType() != MVT::i16 || + Src.getOpcode() != (IsSigned ? ISD::SRA : ISD::SRL) || !Src.hasOneUse() || + Src.getOperand(0).getOpcode() != ISD::TRUNCATE) + return SDValue(); + + // A uniform i16 srl is promoted to i32 later and yields the same nodes. + // Skip it so that the node order, and with it the schedule, stays the same. + if (!IsSigned && !Src->isDivergent()) + return SDValue(); + + SDValue X = Src.getOperand(0).getOperand(0); + auto *Shift = dyn_cast<ConstantSDNode>(Src.getOperand(1)); + if (X.getValueType() != MVT::i32 || !Shift || Shift->getZExtValue() == 0 || + Shift->getZExtValue() >= 16 || (IsSigned && Shift->getZExtValue() != 8)) + return SDValue(); + + SelectionDAG &DAG = DCI.DAG; + SDLoc SL(N); + const unsigned Width = 16 - Shift->getZExtValue(); + SDValue Srl = DAG.getNode( + ISD::SRL, SL, MVT::i32, X, + DAG.getShiftAmountConstant(Shift->getZExtValue(), MVT::i32, SL)); + if (IsSigned) + return DAG.getNode(ISD::SIGN_EXTEND_INREG, SL, MVT::i32, Srl, + DAG.getValueType(MVT::i8)); + return DAG.getNode( + ISD::AND, SL, MVT::i32, Srl, + DAG.getConstant(maskTrailingOnes<uint32_t>(Width), SL, MVT::i32)); +} + SDValue SITargetLowering::performSignExtendInRegCombine(SDNode *N, DAGCombinerInfo &DCI) const { @@ -19336,6 +19398,8 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N, case ISD::ANY_EXTEND: case ISD::ZERO_EXTEND: return performZeroOrAnyExtendCombine(N, DCI); + case ISD::SIGN_EXTEND: + return performExtendTruncShiftCombine(N, DCI); case ISD::SIGN_EXTEND_INREG: return performSignExtendInRegCombine(N, DCI); case AMDGPUISD::FP_CLASS: diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h index 7de2c529028c3..07bc2dc567296 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.h +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h @@ -227,6 +227,7 @@ class SITargetLowering final : public AMDGPUTargetLowering { SDValue performOrCombine(SDNode *N, DAGCombinerInfo &DCI) const; SDValue performXorCombine(SDNode *N, DAGCombinerInfo &DCI) const; SDValue performZeroOrAnyExtendCombine(SDNode *N, DAGCombinerInfo &DCI) const; + SDValue performExtendTruncShiftCombine(SDNode *N, DAGCombinerInfo &DCI) const; SDValue performSignExtendInRegCombine(SDNode *N, DAGCombinerInfo &DCI) const; SDValue performClassCombine(SDNode *N, DAGCombinerInfo &DCI) const; SDValue getCanonicalConstantFP(SelectionDAG &DAG, const SDLoc &SL, EVT VT, diff --git a/llvm/test/CodeGen/AMDGPU/idot4-test.ll b/llvm/test/CodeGen/AMDGPU/idot4-test.ll index 79017f3ea3cd7..99f518c7f256f 100644 --- a/llvm/test/CodeGen/AMDGPU/idot4-test.ll +++ b/llvm/test/CodeGen/AMDGPU/idot4-test.ll @@ -297,13 +297,12 @@ define i32 @udot4_unsat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) { ; GFX9-NODL-LABEL: udot4_unsat_bitcast: ; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v0 -; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v1 -; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 +; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 +; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 +; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 -; GFX9-NODL-NEXT: v_mad_u32_u24 v0, v3, v4, v0 -; GFX9-NODL-NEXT: v_add_u32_e32 v1, v5, v6 +; GFX9-NODL-NEXT: v_add_u32_e32 v0, v4, v0 +; GFX9-NODL-NEXT: v_add_u32_e32 v1, v3, v5 ; GFX9-NODL-NEXT: v_add3_u32 v0, v1, v0, v2 ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] entry: @@ -355,14 +354,12 @@ define i32 @sdot4_unsat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) { ; GFX9-NODL-LABEL: sdot4_unsat_bitcast: ; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1 -; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v0 -; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 +; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 +; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 +; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 -; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0 -; GFX9-NODL-NEXT: v_add_u32_e32 v1, v5, v4 +; GFX9-NODL-NEXT: v_add_u32_e32 v0, v4, v0 +; GFX9-NODL-NEXT: v_add_u32_e32 v1, v3, v5 ; GFX9-NODL-NEXT: v_add3_u32 v0, v1, v0, v2 ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] entry: @@ -407,13 +404,12 @@ define i32 @udot4_sat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) { ; GFX9-NODL-LABEL: udot4_sat_bitcast: ; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v0 -; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v1 -; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 +; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 +; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 +; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 -; GFX9-NODL-NEXT: v_mad_u32_u24 v0, v3, v4, v0 -; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v6, v0 +; GFX9-NODL-NEXT: v_add_u32_e32 v0, v4, v0 +; GFX9-NODL-NEXT: v_add3_u32 v0, v3, v5, v0 ; GFX9-NODL-NEXT: v_add_u32_e64 v0, v0, v2 clamp ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] entry: @@ -458,14 +454,12 @@ define i32 @sdot4_sat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) { ; GFX9-NODL-LABEL: sdot4_sat_bitcast: ; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1 -; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v0 -; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 +; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 +; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 +; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 -; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0 -; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v4, v0 +; GFX9-NODL-NEXT: v_add_u32_e32 v0, v4, v0 +; GFX9-NODL-NEXT: v_add3_u32 v0, v3, v5, v0 ; GFX9-NODL-NEXT: v_add_i32 v0, v0, v2 clamp ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] entry: @@ -803,13 +797,12 @@ define i32 @dot4_mismatched_sign_zext_sadd(i32 %a_packed, i32 %b_packed, i32 %c) ; GFX9-NODL-LABEL: dot4_mismatched_sign_zext_sadd: ; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v0 -; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v1 -; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 +; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 +; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 +; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 -; GFX9-NODL-NEXT: v_mad_u32_u24 v0, v3, v4, v0 -; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v6, v0 +; GFX9-NODL-NEXT: v_add_u32_e32 v0, v4, v0 +; GFX9-NODL-NEXT: v_add3_u32 v0, v3, v5, v0 ; GFX9-NODL-NEXT: v_add_i32 v0, v0, v2 clamp ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] entry: @@ -852,14 +845,12 @@ define i32 @dot4_mismatched_sign_sext_uadd(i32 %a_packed, i32 %b_packed, i32 %c) ; GFX9-NODL-LABEL: dot4_mismatched_sign_sext_uadd: ; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1 -; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v0 -; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 +; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 +; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 +; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 -; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0 -; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v4, v0 +; GFX9-NODL-NEXT: v_add_u32_e32 v0, v4, v0 +; GFX9-NODL-NEXT: v_add3_u32 v0, v3, v5, v0 ; GFX9-NODL-NEXT: v_add_u32_e64 v0, v0, v2 clamp ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] entry: @@ -878,54 +869,48 @@ define i32 @dot4_mixed_ext(i32 %a_packed, i32 %b_packed, i32 %c) { ; GFX9-DL-LABEL: dot4_mixed_ext: ; GFX9-DL: ; %bb.0: ; %entry ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-DL-NEXT: v_lshrrev_b16_e32 v3, 8, v1 -; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0 +; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 +; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 ; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 ; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 -; GFX9-DL-NEXT: v_add_u32_e32 v0, v3, v0 -; GFX9-DL-NEXT: v_add_u32_e32 v1, v4, v5 +; GFX9-DL-NEXT: v_add_u32_e32 v0, v4, v0 +; GFX9-DL-NEXT: v_add_u32_e32 v1, v3, v5 ; GFX9-DL-NEXT: v_add3_u32 v0, v1, v0, v2 ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-DL-LABEL: dot4_mixed_ext: ; GFX10-DL: ; %bb.0: ; %entry ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0xffff -; GFX10-DL-NEXT: v_lshrrev_b16 v4, 8, v1 -; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v6, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 -; GFX10-DL-NEXT: v_and_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1 +; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 +; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 +; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 ; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 -; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v3, v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 -; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v5, v0 -; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v3, v6 +; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v4, v5 +; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v3, v0 ; GFX10-DL-NEXT: v_add3_u32 v0, v0, v1, v2 ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] ; ; GFX950-LABEL: dot4_mixed_ext: ; GFX950: ; %bb.0: ; %entry ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX950-NEXT: v_lshrrev_b16_e32 v3, 8, v1 -; GFX950-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX950-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0 +; GFX950-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 +; GFX950-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 ; GFX950-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 ; GFX950-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 -; GFX950-NEXT: v_add_u32_e32 v0, v3, v0 -; GFX950-NEXT: v_add_u32_e32 v1, v4, v5 +; GFX950-NEXT: v_add_u32_e32 v0, v4, v0 +; GFX950-NEXT: v_add_u32_e32 v1, v3, v5 ; GFX950-NEXT: v_add3_u32 v0, v1, v0, v2 ; GFX950-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-NODL-LABEL: dot4_mixed_ext: ; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1 -; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0 +; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 +; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 -; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0 -; GFX9-NODL-NEXT: v_add_u32_e32 v1, v4, v5 +; GFX9-NODL-NEXT: v_add_u32_e32 v0, v4, v0 +; GFX9-NODL-NEXT: v_add_u32_e32 v1, v3, v5 ; GFX9-NODL-NEXT: v_add3_u32 v0, v1, v0, v2 ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] entry: diff --git a/llvm/test/CodeGen/AMDGPU/idot4s.ll b/llvm/test/CodeGen/AMDGPU/idot4s.ll index 53b522fa4d84b..c7cc5847636be 100644 --- a/llvm/test/CodeGen/AMDGPU/idot4s.ll +++ b/llvm/test/CodeGen/AMDGPU/idot4s.ll @@ -819,18 +819,15 @@ define amdgpu_kernel void @idot4_acc32_vecMul(ptr addrspace(1) %src1, ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GFX8-NEXT: flat_load_dword v0, v[0:1] ; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0 -; GFX8-NEXT: s_waitcnt vmcnt(1) -; GFX8-NEXT: v_lshrrev_b16_e32 v1, 8, v3 ; GFX8-NEXT: s_waitcnt vmcnt(0) -; GFX8-NEXT: v_lshrrev_b16_e32 v2, 8, v0 -; GFX8-NEXT: v_mul_i32_i24_sdwa v4, sext(v3), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 -; GFX8-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 +; GFX8-NEXT: v_mul_i32_i24_sdwa v1, sext(v3), sext(v0) dst_... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/228424 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
