https://github.com/harrisonGPU updated https://github.com/llvm/llvm-project/pull/225002
>From 2d7088a6d08eee2267f4b8d9389c566cb679b80e Mon Sep 17 00:00:00 2001 From: Harrison Hao <[email protected]> Date: Mon, 21 Sep 2026 12:15:13 +0800 Subject: [PATCH] [AMDGPU][InstCombine] Fold constant add/sub into the dot accumulator `amdgcn.{s,u}dot{2,4,8}(a, b, C) +/- K -> dot(a, b, C +/- K)` when both the accumulator C and K are constants. The new constant is computed with wrapping APInt arithmetic to match the non-clamping accumulate. Only applies when clamp is false (the saturating accumulate does not reassociate) and the dot has a single use. K - dot is left alone since it would need the dot product negated. --- .../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 38 +++++++ .../InstCombine/AMDGPU/llvm.amdgcn.dot.ll | 99 +++++++------------ 2 files changed, 71 insertions(+), 66 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp index 073682c526e7e..dcbdde6ccbdfa 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp @@ -1968,6 +1968,44 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const { Result = scalbn(Result, Scale, RoundingMode::NearestTiesToEven); return IC.replaceInstUsesWith(II, ConstantFP::get(Src->getType(), Result)); } + case Intrinsic::amdgcn_sdot2: + case Intrinsic::amdgcn_udot2: + case Intrinsic::amdgcn_sdot4: + case Intrinsic::amdgcn_udot4: + case Intrinsic::amdgcn_sdot8: + case Intrinsic::amdgcn_udot8: { + if (!match(II.getArgOperand(3), m_Zero()) || !II.hasOneUse()) + break; + + const APInt *Acc; + if (!match(II.getArgOperand(2), m_APInt(Acc))) + break; + + auto *AccumUser = dyn_cast<BinaryOperator>(II.user_back()); + if (!AccumUser) + break; + + unsigned Opcode = AccumUser->getOpcode(); + if (Opcode != Instruction::Add && Opcode != Instruction::Sub) + break; + + // C - dot cannot be folded without negating the dot product. + if (Opcode == Instruction::Sub && AccumUser->getOperand(0) != &II) + break; + + const APInt *AccumDelta; + Value *ConstOp = + AccumUser->getOperand(AccumUser->getOperand(0) == &II ? 1 : 0); + if (!match(ConstOp, m_APInt(AccumDelta))) + break; + + Constant *NewAcc = ConstantInt::get(II.getType(), Opcode == Instruction::Add + ? *Acc + *AccumDelta + : *Acc - *AccumDelta); + IC.replaceInstUsesWith(*AccumUser, &II); + IC.eraseInstFromFunction(*AccumUser); + return IC.replaceOperand(II, 2, NewAcc); + } case Intrinsic::amdgcn_fmul_legacy: { Value *Op0 = II.getArgOperand(0); Value *Op1 = II.getArgOperand(1); diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.dot.ll b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.dot.ll index 0efbec8f597b8..c002c5ebecc27 100644 --- a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.dot.ll +++ b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.dot.ll @@ -4,8 +4,7 @@ define i32 @sdot2_add(<2 x i16> %a, <2 x i16> %b) { ; CHECK-LABEL: define i32 @sdot2_add( ; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot2(<2 x i16> %a, <2 x i16> %b, i32 7, i1 false) @@ -16,8 +15,7 @@ define i32 @sdot2_add(<2 x i16> %a, <2 x i16> %b) { define i32 @sdot2_sub(<2 x i16> %a, <2 x i16> %b) { ; CHECK-LABEL: define i32 @sdot2_sub( ; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 6, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot2(<2 x i16> %a, <2 x i16> %b, i32 7, i1 false) @@ -28,8 +26,7 @@ define i32 @sdot2_sub(<2 x i16> %a, <2 x i16> %b) { define i32 @sdot2_a_zero(<2 x i16> %b) { ; CHECK-LABEL: define i32 @sdot2_a_zero( ; CHECK-SAME: <2 x i16> [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> zeroinitializer, <2 x i16> [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> zeroinitializer, <2 x i16> [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot2(<2 x i16> zeroinitializer, <2 x i16> %b, i32 7, i1 false) @@ -40,8 +37,7 @@ define i32 @sdot2_a_zero(<2 x i16> %b) { define i32 @sdot2_b_zero(<2 x i16> %a) { ; CHECK-LABEL: define i32 @sdot2_b_zero( ; CHECK-SAME: <2 x i16> [[A:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> zeroinitializer, i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> zeroinitializer, i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot2(<2 x i16> %a, <2 x i16> zeroinitializer, i32 7, i1 false) @@ -52,8 +48,7 @@ define i32 @sdot2_b_zero(<2 x i16> %a) { define i32 @sdot2_const_nonzero(<2 x i16> %a, <2 x i16> %b) { ; CHECK-LABEL: define i32 @sdot2_const_nonzero( ; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> splat (i16 1), <2 x i16> splat (i16 2), i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> splat (i16 1), <2 x i16> splat (i16 2), i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot2(<2 x i16> <i16 1, i16 1>, <2 x i16> <i16 2, i16 2>, i32 7, i1 false) @@ -102,8 +97,7 @@ define i32 @sdot2_multi_use(<2 x i16> %a, <2 x i16> %b) { define i32 @udot2_add(<2 x i16> %a, <2 x i16> %b) { ; CHECK-LABEL: define i32 @udot2_add( ; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot2(<2 x i16> %a, <2 x i16> %b, i32 7, i1 false) @@ -114,8 +108,7 @@ define i32 @udot2_add(<2 x i16> %a, <2 x i16> %b) { define i32 @udot2_sub(<2 x i16> %a, <2 x i16> %b) { ; CHECK-LABEL: define i32 @udot2_sub( ; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 6, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot2(<2 x i16> %a, <2 x i16> %b, i32 7, i1 false) @@ -126,8 +119,7 @@ define i32 @udot2_sub(<2 x i16> %a, <2 x i16> %b) { define i32 @udot2_a_zero(<2 x i16> %b) { ; CHECK-LABEL: define i32 @udot2_a_zero( ; CHECK-SAME: <2 x i16> [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> zeroinitializer, <2 x i16> [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> zeroinitializer, <2 x i16> [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot2(<2 x i16> zeroinitializer, <2 x i16> %b, i32 7, i1 false) @@ -138,8 +130,7 @@ define i32 @udot2_a_zero(<2 x i16> %b) { define i32 @udot2_b_zero(<2 x i16> %a) { ; CHECK-LABEL: define i32 @udot2_b_zero( ; CHECK-SAME: <2 x i16> [[A:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> zeroinitializer, i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> zeroinitializer, i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot2(<2 x i16> %a, <2 x i16> zeroinitializer, i32 7, i1 false) @@ -150,8 +141,7 @@ define i32 @udot2_b_zero(<2 x i16> %a) { define i32 @udot2_const_nonzero(<2 x i16> %a, <2 x i16> %b) { ; CHECK-LABEL: define i32 @udot2_const_nonzero( ; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> splat (i16 1), <2 x i16> splat (i16 2), i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> splat (i16 1), <2 x i16> splat (i16 2), i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot2(<2 x i16> <i16 1, i16 1>, <2 x i16> <i16 2, i16 2>, i32 7, i1 false) @@ -174,8 +164,7 @@ define i32 @udot2_clamp(<2 x i16> %a, <2 x i16> %b) { define i32 @sdot4_add(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sdot4_add( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 %b, i32 7, i1 false) @@ -186,8 +175,7 @@ define i32 @sdot4_add(i32 %a, i32 %b) { define i32 @sdot4_add_commuted(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sdot4_add_commuted( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 %b, i32 7, i1 false) @@ -198,8 +186,7 @@ define i32 @sdot4_add_commuted(i32 %a, i32 %b) { define i32 @sdot4_sub(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sdot4_sub( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 6, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 %b, i32 7, i1 false) @@ -210,8 +197,7 @@ define i32 @sdot4_sub(i32 %a, i32 %b) { define i32 @sdot4_a_zero(i32 %b) { ; CHECK-LABEL: define i32 @sdot4_a_zero( ; CHECK-SAME: i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 0, i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 0, i32 [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot4(i32 0, i32 %b, i32 7, i1 false) @@ -222,8 +208,7 @@ define i32 @sdot4_a_zero(i32 %b) { define i32 @sdot4_b_zero(i32 %a) { ; CHECK-LABEL: define i32 @sdot4_b_zero( ; CHECK-SAME: i32 [[A:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 0, i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 0, i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 0, i32 7, i1 false) @@ -234,8 +219,7 @@ define i32 @sdot4_b_zero(i32 %a) { define i32 @sdot4_sub_to_zero(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sdot4_sub_to_zero( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 9, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 0, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 %b, i32 9, i1 false) @@ -246,8 +230,7 @@ define i32 @sdot4_sub_to_zero(i32 %a, i32 %b) { define i32 @sdot4_add_chain(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sdot4_add_chain( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 6 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 13, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 %b, i32 7, i1 false) @@ -259,8 +242,7 @@ define i32 @sdot4_add_chain(i32 %a, i32 %b) { define i32 @sdot4_const_nonzero(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sdot4_const_nonzero( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 1, i32 2, i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 1, i32 2, i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot4(i32 1, i32 2, i32 7, i1 false) @@ -333,8 +315,7 @@ define i32 @sdot4_add_x(i32 %a, i32 %b, i32 %x) { define i32 @udot4_add(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @udot4_add( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot4(i32 %a, i32 %b, i32 7, i1 false) @@ -345,8 +326,7 @@ define i32 @udot4_add(i32 %a, i32 %b) { define i32 @udot4_sub(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @udot4_sub( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 [[B]], i32 6, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot4(i32 %a, i32 %b, i32 7, i1 false) @@ -357,8 +337,7 @@ define i32 @udot4_sub(i32 %a, i32 %b) { define i32 @udot4_a_zero(i32 %b) { ; CHECK-LABEL: define i32 @udot4_a_zero( ; CHECK-SAME: i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 0, i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot4(i32 0, i32 [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot4(i32 0, i32 %b, i32 7, i1 false) @@ -369,8 +348,7 @@ define i32 @udot4_a_zero(i32 %b) { define i32 @udot4_b_zero(i32 %a) { ; CHECK-LABEL: define i32 @udot4_b_zero( ; CHECK-SAME: i32 [[A:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 0, i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 0, i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot4(i32 %a, i32 0, i32 7, i1 false) @@ -381,8 +359,7 @@ define i32 @udot4_b_zero(i32 %a) { define i32 @udot4_const_nonzero(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @udot4_const_nonzero( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 1, i32 2, i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot4(i32 1, i32 2, i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot4(i32 1, i32 2, i32 7, i1 false) @@ -405,8 +382,7 @@ define i32 @udot4_clamp(i32 %a, i32 %b) { define i32 @sdot8_add(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sdot8_add( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot8(i32 %a, i32 %b, i32 7, i1 false) @@ -417,8 +393,7 @@ define i32 @sdot8_add(i32 %a, i32 %b) { define i32 @sdot8_sub(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sdot8_sub( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 [[B]], i32 6, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot8(i32 %a, i32 %b, i32 7, i1 false) @@ -429,8 +404,7 @@ define i32 @sdot8_sub(i32 %a, i32 %b) { define i32 @sdot8_a_zero(i32 %b) { ; CHECK-LABEL: define i32 @sdot8_a_zero( ; CHECK-SAME: i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 0, i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 0, i32 [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot8(i32 0, i32 %b, i32 7, i1 false) @@ -441,8 +415,7 @@ define i32 @sdot8_a_zero(i32 %b) { define i32 @sdot8_b_zero(i32 %a) { ; CHECK-LABEL: define i32 @sdot8_b_zero( ; CHECK-SAME: i32 [[A:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 0, i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 0, i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot8(i32 %a, i32 0, i32 7, i1 false) @@ -453,8 +426,7 @@ define i32 @sdot8_b_zero(i32 %a) { define i32 @sdot8_const_nonzero(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sdot8_const_nonzero( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 1, i32 2, i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 1, i32 2, i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot8(i32 1, i32 2, i32 7, i1 false) @@ -477,8 +449,7 @@ define i32 @sdot8_clamp(i32 %a, i32 %b) { define i32 @udot8_add(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @udot8_add( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot8(i32 %a, i32 %b, i32 7, i1 false) @@ -489,8 +460,7 @@ define i32 @udot8_add(i32 %a, i32 %b) { define i32 @udot8_sub(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @udot8_sub( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 [[B]], i32 6, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot8(i32 %a, i32 %b, i32 7, i1 false) @@ -501,8 +471,7 @@ define i32 @udot8_sub(i32 %a, i32 %b) { define i32 @udot8_a_zero(i32 %b) { ; CHECK-LABEL: define i32 @udot8_a_zero( ; CHECK-SAME: i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot8(i32 0, i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot8(i32 0, i32 [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot8(i32 0, i32 %b, i32 7, i1 false) @@ -513,8 +482,7 @@ define i32 @udot8_a_zero(i32 %b) { define i32 @udot8_b_zero(i32 %a) { ; CHECK-LABEL: define i32 @udot8_b_zero( ; CHECK-SAME: i32 [[A:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 0, i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 0, i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot8(i32 %a, i32 0, i32 7, i1 false) @@ -525,8 +493,7 @@ define i32 @udot8_b_zero(i32 %a) { define i32 @udot8_const_nonzero(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @udot8_const_nonzero( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot8(i32 1, i32 2, i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot8(i32 1, i32 2, i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot8(i32 1, i32 2, i32 7, i1 false) _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
