https://github.com/harrisonGPU updated https://github.com/llvm/llvm-project/pull/225322
>From 22785282601a54d9278cd339df5c9760899a5a4e Mon Sep 17 00:00:00 2001 From: Harrison Hao <[email protected]> Date: Tue, 22 Sep 2026 16:00:29 +0800 Subject: [PATCH] [AMDGPU] Fold a constant add/sub into the sudot4/sudot8 accumulator Fold a constant add into the accumulator operand of sudot4 and sudot8 when clamping is disabled: ``` sudot(a, b, C1, false) + C2 -> sudot(a, b, C1 + C2, false) ``` Subtraction by a constant is canonicalized to addition of its negation. --- .../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 26 +++++++++++++++++++ .../InstCombine/AMDGPU/llvm.amdgcn.sudot.ll | 12 +++------ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp index 073682c526e7e..b08a5a708e4d6 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp @@ -2087,6 +2087,32 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const { } return std::nullopt; } + case Intrinsic::amdgcn_sudot4: + case Intrinsic::amdgcn_sudot8: { + // Reassociating across a saturating accumulate is not valid. + if (!II.hasOneUse() || !match(II.getArgOperand(5), m_Zero())) + break; + + const APInt *Acc; + if (!match(II.getArgOperand(4), m_APInt(Acc))) + break; + + auto *AccumUser = dyn_cast<BinaryOperator>(II.user_back()); + if (!AccumUser) + break; + + const APInt *AccumDelta; + Constant *NewAcc; + if (match(AccumUser, m_c_Add(m_Specific(&II), m_APInt(AccumDelta)))) + NewAcc = ConstantInt::get(II.getType(), *Acc + *AccumDelta); + else + break; + + IC.replaceOperand(II, 4, NewAcc); + IC.replaceInstUsesWith(*AccumUser, &II); + IC.eraseInstFromFunction(*AccumUser); + return &II; + } case Intrinsic::amdgcn_mfma_scale_f32_16x16x128_f8f6f4: case Intrinsic::amdgcn_mfma_scale_f32_32x32x64_f8f6f4: { Value *Src0 = II.getArgOperand(0); diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.sudot.ll b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.sudot.ll index c9a1fee2530be..1c12dfaa0ac7f 100644 --- a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.sudot.ll +++ b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.sudot.ll @@ -4,8 +4,7 @@ define i32 @sudot4_add(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sudot4_add( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sudot4(i1 true, i32 %a, i1 false, i32 %b, i32 7, i1 false) @@ -16,8 +15,7 @@ define i32 @sudot4_add(i32 %a, i32 %b) { define i32 @sudot4_sub(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sudot4_sub( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot4(i1 true, i32 [[A]], i1 false, i32 [[B]], i32 6, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sudot4(i1 true, i32 %a, i1 false, i32 %b, i32 7, i1 false) @@ -60,8 +58,7 @@ define i32 @sudot4_clamp(i32 %a, i32 %b) { define i32 @sudot8_add(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sudot8_add( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sudot8(i1 false, i32 %a, i1 true, i32 %b, i32 7, i1 false) @@ -72,8 +69,7 @@ define i32 @sudot8_add(i32 %a, i32 %b) { define i32 @sudot8_sub(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sudot8_sub( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sudot8(i1 false, i32 [[A]], i1 true, i32 [[B]], i32 6, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sudot8(i1 false, i32 %a, i1 true, i32 %b, i32 7, i1 false) _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
