https://github.com/gandhi56 updated https://github.com/llvm/llvm-project/pull/210156
>From bca744caf6189a47e04fca4b93e3d84a6bdbd67e Mon Sep 17 00:00:00 2001 From: Anshil Gandhi <[email protected]> Date: Wed, 22 Jul 2026 12:11:30 -0500 Subject: [PATCH] [AMDGPU] Lower uniform uaddsat.i16 to SALU instructions Promote uniform i16 uadd.sat to i32 in promoteUniformOpToI32 so it lowers to SALU (s_add_i32 + s_min_u32) instead of VALU + readfirstlane. The saturating add on zero-extended operands reduces to umin(add(lhs, rhs), 0xffff). Co-authored-by: Cursor <[email protected]> --- llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 1 + llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 12 +- llvm/test/CodeGen/AMDGPU/uaddsat.ll | 124 +++++++++--------- 3 files changed, 75 insertions(+), 62 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp index 962988ff97e39..7fa4b9a09aea9 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -1059,6 +1059,7 @@ bool AMDGPUTargetLowering::isNarrowingProfitable(SDNode *N, EVT SrcVT, case ISD::UMIN: case ISD::UMAX: case ISD::USUBSAT: + case ISD::UADDSAT: if (isTypeLegal(MVT::i16) && (!DestVT.isVector() || !isOperationLegal(ISD::ADD, MVT::v2i16))) { // Check if VOP3P diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index f2b670558de1c..11cf138b79f61 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -1078,6 +1078,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, ISD::UMIN, ISD::UMAX, ISD::USUBSAT, + ISD::UADDSAT, ISD::AND, ISD::OR, ISD::XOR, @@ -8911,6 +8912,7 @@ static unsigned getExtOpcodeForPromotedOp(SDValue Op) { case ISD::UMIN: case ISD::UMAX: case ISD::USUBSAT: + case ISD::UADDSAT: return ISD::ZERO_EXTEND; case ISD::ADD: case ISD::SUB: @@ -8960,7 +8962,7 @@ SDValue SITargetLowering::promoteUniformOpToI32(SDValue Op, Opc == ISD::OR || Opc == ISD::XOR || Opc == ISD::MUL || Opc == ISD::SETCC || Opc == ISD::SELECT || Opc == ISD::SMIN || Opc == ISD::SMAX || Opc == ISD::UMIN || Opc == ISD::UMAX || - Opc == ISD::USUBSAT); + Opc == ISD::USUBSAT || Opc == ISD::UADDSAT); EVT OpTy = (Opc != ISD::SETCC) ? Op.getValueType() : Op->getOperand(0).getValueType(); @@ -9002,7 +9004,12 @@ SDValue SITargetLowering::promoteUniformOpToI32(SDValue Op, SDValue NewVal; if (Opc == ISD::SELECT) NewVal = DAG.getNode(ISD::SELECT, DL, ExtTy, {Op->getOperand(0), LHS, RHS}); - else + else if (Opc == ISD::UADDSAT) { + SDValue Sum = DAG.getNode(ISD::ADD, DL, ExtTy, LHS, RHS); + SDValue MaxVal = DAG.getConstant( + APInt::getMaxValue(OpTy.getScalarSizeInBits()).zext(32), DL, ExtTy); + NewVal = DAG.getNode(ISD::UMIN, DL, ExtTy, Sum, MaxVal); + } else NewVal = DAG.getNode(Opc, DL, ExtTy, {LHS, RHS}); return DAG.getZExtOrTrunc(NewVal, DL, OpTy); @@ -18720,6 +18727,7 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N, case ISD::UMIN: case ISD::UMAX: case ISD::USUBSAT: + case ISD::UADDSAT: if (auto Res = promoteUniformOpToI32(SDValue(N, 0), DCI)) return Res; break; diff --git a/llvm/test/CodeGen/AMDGPU/uaddsat.ll b/llvm/test/CodeGen/AMDGPU/uaddsat.ll index b5e9611e15c27..e188bb032b029 100644 --- a/llvm/test/CodeGen/AMDGPU/uaddsat.ll +++ b/llvm/test/CodeGen/AMDGPU/uaddsat.ll @@ -73,37 +73,36 @@ define amdgpu_ps i16 @s_uaddsat_i16(i16 inreg %lhs, i16 inreg %rhs) { ; ; GFX8-LABEL: s_uaddsat_i16: ; GFX8: ; %bb.0: -; GFX8-NEXT: v_mov_b32_e32 v0, s1 -; GFX8-NEXT: v_add_u16_e64 v0, s0, v0 clamp -; GFX8-NEXT: v_readfirstlane_b32 s0, v0 +; GFX8-NEXT: s_and_b32 s1, 0xffff, s1 +; GFX8-NEXT: s_and_b32 s0, 0xffff, s0 +; GFX8-NEXT: s_add_i32 s0, s0, s1 +; GFX8-NEXT: s_min_u32 s0, s0, 0xffff ; GFX8-NEXT: ; return to shader part epilog ; ; GFX9-LABEL: s_uaddsat_i16: ; GFX9: ; %bb.0: -; GFX9-NEXT: v_mov_b32_e32 v0, s1 -; GFX9-NEXT: v_add_u16_e64 v0, s0, v0 clamp -; GFX9-NEXT: v_readfirstlane_b32 s0, v0 +; GFX9-NEXT: s_and_b32 s1, 0xffff, s1 +; GFX9-NEXT: s_and_b32 s0, 0xffff, s0 +; GFX9-NEXT: s_add_i32 s0, s0, s1 +; GFX9-NEXT: s_min_u32 s0, s0, 0xffff ; GFX9-NEXT: ; return to shader part epilog ; ; GFX10-LABEL: s_uaddsat_i16: ; GFX10: ; %bb.0: -; GFX10-NEXT: v_add_nc_u16 v0, s0, s1 clamp -; GFX10-NEXT: v_readfirstlane_b32 s0, v0 +; GFX10-NEXT: s_and_b32 s1, 0xffff, s1 +; GFX10-NEXT: s_and_b32 s0, 0xffff, s0 +; GFX10-NEXT: s_add_i32 s0, s0, s1 +; GFX10-NEXT: s_min_u32 s0, s0, 0xffff ; GFX10-NEXT: ; return to shader part epilog ; -; GFX11-TRUE16-LABEL: s_uaddsat_i16: -; GFX11-TRUE16: ; %bb.0: -; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, s0, s1 clamp -; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0 -; GFX11-TRUE16-NEXT: ; return to shader part epilog -; -; GFX11-FAKE16-LABEL: s_uaddsat_i16: -; GFX11-FAKE16: ; %bb.0: -; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, s0, s1 clamp -; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0 -; GFX11-FAKE16-NEXT: ; return to shader part epilog +; GFX11-LABEL: s_uaddsat_i16: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_and_b32 s1, 0xffff, s1 +; GFX11-NEXT: s_and_b32 s0, 0xffff, s0 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) +; GFX11-NEXT: s_add_i32 s0, s0, s1 +; GFX11-NEXT: s_min_u32 s0, s0, 0xffff +; GFX11-NEXT: ; return to shader part epilog %result = call i16 @llvm.uadd.sat.i16(i16 %lhs, i16 %rhs) ret i16 %result } @@ -125,15 +124,16 @@ define amdgpu_ps <2 x i16> @s_uaddsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inre ; ; GFX8-LABEL: s_uaddsat_v2i16: ; GFX8: ; %bb.0: -; GFX8-NEXT: s_lshr_b32 s2, s1, 16 -; GFX8-NEXT: s_lshr_b32 s3, s0, 16 -; GFX8-NEXT: v_mov_b32_e32 v0, s2 -; GFX8-NEXT: v_mov_b32_e32 v1, s3 -; GFX8-NEXT: v_add_u16_sdwa v0, v1, v0 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD -; GFX8-NEXT: v_mov_b32_e32 v1, s1 -; GFX8-NEXT: v_add_u16_e64 v1, s0, v1 clamp -; GFX8-NEXT: v_or_b32_e32 v0, v1, v0 -; GFX8-NEXT: v_readfirstlane_b32 s0, v0 +; GFX8-NEXT: s_and_b32 s2, s1, 0xffff +; GFX8-NEXT: s_and_b32 s3, s0, 0xffff +; GFX8-NEXT: s_lshr_b32 s1, s1, 16 +; GFX8-NEXT: s_lshr_b32 s0, s0, 16 +; GFX8-NEXT: s_add_i32 s0, s0, s1 +; GFX8-NEXT: s_add_i32 s3, s3, s2 +; GFX8-NEXT: s_min_u32 s0, s0, 0xffff +; GFX8-NEXT: s_min_u32 s2, s3, 0xffff +; GFX8-NEXT: s_lshl_b32 s0, s0, 16 +; GFX8-NEXT: s_or_b32 s0, s2, s0 ; GFX8-NEXT: ; return to shader part epilog ; ; GFX9-LABEL: s_uaddsat_v2i16: @@ -180,18 +180,20 @@ define amdgpu_ps <3 x i16> @s_uaddsat_v3i16(<3 x i16> inreg %lhs, <3 x i16> inre ; ; GFX8-LABEL: s_uaddsat_v3i16: ; GFX8: ; %bb.0: -; GFX8-NEXT: s_lshr_b32 s4, s2, 16 -; GFX8-NEXT: s_lshr_b32 s5, s0, 16 -; GFX8-NEXT: v_mov_b32_e32 v0, s4 -; GFX8-NEXT: v_mov_b32_e32 v1, s5 -; GFX8-NEXT: v_mov_b32_e32 v2, s2 -; GFX8-NEXT: v_add_u16_sdwa v0, v1, v0 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD -; GFX8-NEXT: v_mov_b32_e32 v1, s3 -; GFX8-NEXT: v_add_u16_e64 v2, s0, v2 clamp -; GFX8-NEXT: v_add_u16_e64 v1, s1, v1 clamp -; GFX8-NEXT: v_or_b32_e32 v0, v2, v0 -; GFX8-NEXT: v_readfirstlane_b32 s0, v0 -; GFX8-NEXT: v_readfirstlane_b32 s1, v1 +; GFX8-NEXT: s_and_b32 s3, s3, 0xffff +; GFX8-NEXT: s_and_b32 s1, s1, 0xffff +; GFX8-NEXT: s_add_i32 s1, s1, s3 +; GFX8-NEXT: s_and_b32 s3, s2, 0xffff +; GFX8-NEXT: s_and_b32 s4, s0, 0xffff +; GFX8-NEXT: s_lshr_b32 s2, s2, 16 +; GFX8-NEXT: s_lshr_b32 s0, s0, 16 +; GFX8-NEXT: s_add_i32 s0, s0, s2 +; GFX8-NEXT: s_add_i32 s4, s4, s3 +; GFX8-NEXT: s_min_u32 s0, s0, 0xffff +; GFX8-NEXT: s_min_u32 s3, s4, 0xffff +; GFX8-NEXT: s_lshl_b32 s0, s0, 16 +; GFX8-NEXT: s_min_u32 s1, s1, 0xffff +; GFX8-NEXT: s_or_b32 s0, s3, s0 ; GFX8-NEXT: ; return to shader part epilog ; ; GFX9-LABEL: s_uaddsat_v3i16: @@ -251,24 +253,26 @@ define amdgpu_ps <4 x i16> @s_uaddsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre ; ; GFX8-LABEL: s_uaddsat_v4i16: ; GFX8: ; %bb.0: -; GFX8-NEXT: s_lshr_b32 s4, s3, 16 -; GFX8-NEXT: s_lshr_b32 s5, s1, 16 -; GFX8-NEXT: v_mov_b32_e32 v0, s4 -; GFX8-NEXT: v_mov_b32_e32 v1, s5 -; GFX8-NEXT: s_lshr_b32 s4, s2, 16 -; GFX8-NEXT: s_lshr_b32 s5, s0, 16 -; GFX8-NEXT: v_add_u16_sdwa v0, v1, v0 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD -; GFX8-NEXT: v_mov_b32_e32 v1, s4 -; GFX8-NEXT: v_mov_b32_e32 v2, s5 -; GFX8-NEXT: v_add_u16_sdwa v1, v2, v1 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD -; GFX8-NEXT: v_mov_b32_e32 v2, s3 -; GFX8-NEXT: v_mov_b32_e32 v3, s2 -; GFX8-NEXT: v_add_u16_e64 v2, s1, v2 clamp -; GFX8-NEXT: v_add_u16_e64 v3, s0, v3 clamp -; GFX8-NEXT: v_or_b32_e32 v1, v3, v1 -; GFX8-NEXT: v_or_b32_e32 v0, v2, v0 -; GFX8-NEXT: v_readfirstlane_b32 s0, v1 -; GFX8-NEXT: v_readfirstlane_b32 s1, v0 +; GFX8-NEXT: s_and_b32 s4, s3, 0xffff +; GFX8-NEXT: s_and_b32 s5, s1, 0xffff +; GFX8-NEXT: s_add_i32 s5, s5, s4 +; GFX8-NEXT: s_lshr_b32 s3, s3, 16 +; GFX8-NEXT: s_lshr_b32 s1, s1, 16 +; GFX8-NEXT: s_min_u32 s4, s5, 0xffff +; GFX8-NEXT: s_add_i32 s1, s1, s3 +; GFX8-NEXT: s_and_b32 s3, s2, 0xffff +; GFX8-NEXT: s_and_b32 s5, s0, 0xffff +; GFX8-NEXT: s_lshr_b32 s2, s2, 16 +; GFX8-NEXT: s_lshr_b32 s0, s0, 16 +; GFX8-NEXT: s_add_i32 s0, s0, s2 +; GFX8-NEXT: s_min_u32 s1, s1, 0xffff +; GFX8-NEXT: s_add_i32 s5, s5, s3 +; GFX8-NEXT: s_min_u32 s0, s0, 0xffff +; GFX8-NEXT: s_min_u32 s3, s5, 0xffff +; GFX8-NEXT: s_lshl_b32 s0, s0, 16 +; GFX8-NEXT: s_lshl_b32 s1, s1, 16 +; GFX8-NEXT: s_or_b32 s0, s3, s0 +; GFX8-NEXT: s_or_b32 s1, s4, s1 ; GFX8-NEXT: ; return to shader part epilog ; ; GFX9-LABEL: s_uaddsat_v4i16: _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
