https://github.com/harrisonGPU updated https://github.com/llvm/llvm-project/pull/225322
>From da5b743dc74a4d6aebc6b95b72c36a425269b7ef Mon Sep 17 00:00:00 2001 From: Harrison Hao <[email protected]> Date: Tue, 22 Sep 2026 16:00:29 +0800 Subject: [PATCH] [AMDGPU] Fold a constant add/sub into the sudot4/sudot8 accumulator Fold a constant add into the accumulator operand of sudot4 and sudot8 when clamping is disabled: ``` sudot(a, b, C1, false) + C2 -> sudot(a, b, C1 + C2, false) ``` Subtraction by a constant is canonicalized to addition of its negation. --- .../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 26 ++++++++++++++++ .../InstCombine/AMDGPU/llvm.amdgcn.sudot.ll | 30 +++++++------------ 2 files changed, 36 insertions(+), 20 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp index 073682c526e7e..b08a5a708e4d6 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp @@ -2087,6 +2087,32 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const { } return std::nullopt; } + case Intrinsic::amdgcn_sudot4: + case Intrinsic::amdgcn_sudot8: { + // Reassociating across a saturating accumulate is not valid. + if (!II.hasOneUse() || !match(II.getArgOperand(5), m_Zero())) + break; + + const APInt *Acc; + if (!match(II.getArgOperand(4), m_APInt(Acc))) + break; + + auto *AccumUser = dyn_cast<BinaryOperator>(II.user_back()); + if (!AccumUser) + break; + + const APInt *AccumDelta; + Constant *NewAcc; + if (match(AccumUser, m_c_Add(m_Specific(&II), m_APInt(AccumDelta)))) + NewAcc = ConstantInt::get(II.getType(), *Acc + *AccumDelta); + else + break; + + IC.replaceOperand(II, 4, NewAcc); + IC.replaceInstUsesWith(*AccumUser, &II); + IC.eraseInstFromFunction(*AccumUser); + return &II; + } case Intrinsic::amdgcn_mfma_scale_f32_16x16x128_f8f6f4: case Intrinsic::amdgcn_mfma_scale_f32_32x32x64_f8f6f4: { Value *Src0 = II.getArgOperand(0); diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.sudot.ll b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.sudot.ll index a9840e1c7e489..97a32749dac39 100644 --- a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.sudot.ll +++ b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.sudot.ll @@ -4,8 +4,7 @@ define i32 @sudot4_add(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sudot4_add( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sudot4(i1 true, i32 %a, i1 false, i32 %b, i32 7, i1 false) @@ -16,8 +15,7 @@ define i32 @sudot4_add(i32 %a, i32 %b) { define i32 @sudot4_sub(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sudot4_sub( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 6, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sudot4(i1 true, i32 %a, i1 false, i32 %b, i32 7, i1 false) @@ -105,8 +103,7 @@ define i32 @sudot4_all_constants_clamp() { define i32 @sudot4_zero_acc_add_const(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sudot4_zero_acc_add_const( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 0, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 9, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sudot4(i1 true, i32 %a, i1 false, i32 %b, i32 0, i1 false) @@ -129,8 +126,7 @@ define i32 @sudot4_zero_acc_add_value(i32 %a, i32 %b, i32 %acc) { define i32 @sudot4_zero_acc_add_const_lhs(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sudot4_zero_acc_add_const_lhs( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 0, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 9, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sudot4(i1 true, i32 %a, i1 false, i32 %b, i32 0, i1 false) @@ -157,8 +153,7 @@ define i32 @sudot4_zero_acc_multiple_users(i32 %a, i32 %b) { define i32 @sudot4_zero_acc_sub_const(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sudot4_zero_acc_sub_const( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 0, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 -9, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sudot4(i1 true, i32 %a, i1 false, i32 %b, i32 0, i1 false) @@ -181,8 +176,7 @@ define i32 @sudot4_zero_acc_const_sub(i32 %a, i32 %b) { define i32 @sudot8_add(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sudot8_add( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sudot8(i1 false, i32 %a, i1 true, i32 %b, i32 7, i1 false) @@ -238,8 +232,7 @@ define i32 @sudot8_all_constants_clamp() { define i32 @sudot8_zero_acc_add_const(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sudot8_zero_acc_add_const( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 0, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 9, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sudot8(i1 false, i32 %a, i1 true, i32 %b, i32 0, i1 false) @@ -262,8 +255,7 @@ define i32 @sudot8_zero_acc_add_value(i32 %a, i32 %b, i32 %acc) { define i32 @sudot8_zero_acc_add_const_lhs(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sudot8_zero_acc_add_const_lhs( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 0, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 9, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sudot8(i1 false, i32 %a, i1 true, i32 %b, i32 0, i1 false) @@ -290,8 +282,7 @@ define i32 @sudot8_zero_acc_multiple_users(i32 %a, i32 %b) { define i32 @sudot8_zero_acc_sub_const(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sudot8_zero_acc_sub_const( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 0, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 -9, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sudot8(i1 false, i32 %a, i1 true, i32 %b, i32 0, i1 false) @@ -314,8 +305,7 @@ define i32 @sudot8_zero_acc_const_sub(i32 %a, i32 %b) { define i32 @sudot8_sub(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sudot8_sub( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 6, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sudot8(i1 false, i32 %a, i1 true, i32 %b, i32 7, i1 false) _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
