https://github.com/harrisonGPU updated https://github.com/llvm/llvm-project/pull/225002
>From 3e55f4fb93eda65673ef3211c2a6094068d72071 Mon Sep 17 00:00:00 2001 From: Harrison Hao <[email protected]> Date: Mon, 21 Sep 2026 12:15:13 +0800 Subject: [PATCH] [AMDGPU][InstCombine] Fold constant add/sub into the dot accumulator `amdgcn.{s,u}dot{2,4,8}(a, b, C) +/- K -> dot(a, b, C +/- K)` when both the accumulator C and K are constants. The new constant is computed with wrapping APInt arithmetic to match the non-clamping accumulate. Only applies when clamp is false (the saturating accumulate does not reassociate) and the dot has a single use. K - dot is left alone since it would need the dot product negated. --- .../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 38 +++++++ .../InstCombine/AMDGPU/llvm.amdgcn.dot.ll | 99 +++++++------------ 2 files changed, 71 insertions(+), 66 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp index 073682c526e7e..dcbdde6ccbdfa 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp @@ -1968,6 +1968,44 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const { Result = scalbn(Result, Scale, RoundingMode::NearestTiesToEven); return IC.replaceInstUsesWith(II, ConstantFP::get(Src->getType(), Result)); } + case Intrinsic::amdgcn_sdot2: + case Intrinsic::amdgcn_udot2: + case Intrinsic::amdgcn_sdot4: + case Intrinsic::amdgcn_udot4: + case Intrinsic::amdgcn_sdot8: + case Intrinsic::amdgcn_udot8: { + if (!match(II.getArgOperand(3), m_Zero()) || !II.hasOneUse()) + break; + + const APInt *Acc; + if (!match(II.getArgOperand(2), m_APInt(Acc))) + break; + + auto *AccumUser = dyn_cast<BinaryOperator>(II.user_back()); + if (!AccumUser) + break; + + unsigned Opcode = AccumUser->getOpcode(); + if (Opcode != Instruction::Add && Opcode != Instruction::Sub) + break; + + // C - dot cannot be folded without negating the dot product. + if (Opcode == Instruction::Sub && AccumUser->getOperand(0) != &II) + break; + + const APInt *AccumDelta; + Value *ConstOp = + AccumUser->getOperand(AccumUser->getOperand(0) == &II ? 1 : 0); + if (!match(ConstOp, m_APInt(AccumDelta))) + break; + + Constant *NewAcc = ConstantInt::get(II.getType(), Opcode == Instruction::Add + ? *Acc + *AccumDelta + : *Acc - *AccumDelta); + IC.replaceInstUsesWith(*AccumUser, &II); + IC.eraseInstFromFunction(*AccumUser); + return IC.replaceOperand(II, 2, NewAcc); + } case Intrinsic::amdgcn_fmul_legacy: { Value *Op0 = II.getArgOperand(0); Value *Op1 = II.getArgOperand(1); diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.dot.ll b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.dot.ll index 05c8111537cd9..a719483577d8d 100644 --- a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.dot.ll +++ b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.dot.ll @@ -4,8 +4,7 @@ define i32 @sdot2_add(<2 x i16> %a, <2 x i16> %b) { ; CHECK-LABEL: define i32 @sdot2_add( ; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot2(<2 x i16> %a, <2 x i16> %b, i32 7, i1 false) @@ -16,8 +15,7 @@ define i32 @sdot2_add(<2 x i16> %a, <2 x i16> %b) { define i32 @sdot2_sub(<2 x i16> %a, <2 x i16> %b) { ; CHECK-LABEL: define i32 @sdot2_sub( ; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 6, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot2(<2 x i16> %a, <2 x i16> %b, i32 7, i1 false) @@ -28,8 +26,7 @@ define i32 @sdot2_sub(<2 x i16> %a, <2 x i16> %b) { define i32 @sdot2_a_zero(<2 x i16> %b) { ; CHECK-LABEL: define i32 @sdot2_a_zero( ; CHECK-SAME: <2 x i16> [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> zeroinitializer, <2 x i16> [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> zeroinitializer, <2 x i16> [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot2(<2 x i16> zeroinitializer, <2 x i16> %b, i32 7, i1 false) @@ -50,8 +47,7 @@ define i32 @sdot2_const_lhs(<2 x i16> %b, i32 %acc) { define i32 @sdot2_b_zero(<2 x i16> %a) { ; CHECK-LABEL: define i32 @sdot2_b_zero( ; CHECK-SAME: <2 x i16> [[A:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> zeroinitializer, i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> [[A]], <2 x i16> zeroinitializer, i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot2(<2 x i16> %a, <2 x i16> zeroinitializer, i32 7, i1 false) @@ -62,8 +58,7 @@ define i32 @sdot2_b_zero(<2 x i16> %a) { define i32 @sdot2_const_nonzero(<2 x i16> %a, <2 x i16> %b) { ; CHECK-LABEL: define i32 @sdot2_const_nonzero( ; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> splat (i16 1), <2 x i16> splat (i16 2), i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot2(<2 x i16> splat (i16 1), <2 x i16> splat (i16 2), i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot2(<2 x i16> splat (i16 1), <2 x i16> splat (i16 2), i32 7, i1 false) @@ -112,8 +107,7 @@ define i32 @sdot2_multi_use(<2 x i16> %a, <2 x i16> %b) { define i32 @udot2_add(<2 x i16> %a, <2 x i16> %b) { ; CHECK-LABEL: define i32 @udot2_add( ; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot2(<2 x i16> %a, <2 x i16> %b, i32 7, i1 false) @@ -124,8 +118,7 @@ define i32 @udot2_add(<2 x i16> %a, <2 x i16> %b) { define i32 @udot2_sub(<2 x i16> %a, <2 x i16> %b) { ; CHECK-LABEL: define i32 @udot2_sub( ; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> [[B]], i32 6, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot2(<2 x i16> %a, <2 x i16> %b, i32 7, i1 false) @@ -136,8 +129,7 @@ define i32 @udot2_sub(<2 x i16> %a, <2 x i16> %b) { define i32 @udot2_a_zero(<2 x i16> %b) { ; CHECK-LABEL: define i32 @udot2_a_zero( ; CHECK-SAME: <2 x i16> [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> zeroinitializer, <2 x i16> [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> zeroinitializer, <2 x i16> [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot2(<2 x i16> zeroinitializer, <2 x i16> %b, i32 7, i1 false) @@ -158,8 +150,7 @@ define i32 @udot2_const_lhs(<2 x i16> %b, i32 %acc) { define i32 @udot2_b_zero(<2 x i16> %a) { ; CHECK-LABEL: define i32 @udot2_b_zero( ; CHECK-SAME: <2 x i16> [[A:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> zeroinitializer, i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> [[A]], <2 x i16> zeroinitializer, i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot2(<2 x i16> %a, <2 x i16> zeroinitializer, i32 7, i1 false) @@ -170,8 +161,7 @@ define i32 @udot2_b_zero(<2 x i16> %a) { define i32 @udot2_const_nonzero(<2 x i16> %a, <2 x i16> %b) { ; CHECK-LABEL: define i32 @udot2_const_nonzero( ; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> splat (i16 1), <2 x i16> splat (i16 2), i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot2(<2 x i16> splat (i16 1), <2 x i16> splat (i16 2), i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot2(<2 x i16> splat (i16 1), <2 x i16> splat (i16 2), i32 7, i1 false) @@ -194,8 +184,7 @@ define i32 @udot2_clamp(<2 x i16> %a, <2 x i16> %b) { define i32 @sdot4_add(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sdot4_add( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 %b, i32 7, i1 false) @@ -206,8 +195,7 @@ define i32 @sdot4_add(i32 %a, i32 %b) { define i32 @sdot4_add_commuted(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sdot4_add_commuted( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 %b, i32 7, i1 false) @@ -218,8 +206,7 @@ define i32 @sdot4_add_commuted(i32 %a, i32 %b) { define i32 @sdot4_sub(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sdot4_sub( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 6, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 %b, i32 7, i1 false) @@ -230,8 +217,7 @@ define i32 @sdot4_sub(i32 %a, i32 %b) { define i32 @sdot4_a_zero(i32 %b) { ; CHECK-LABEL: define i32 @sdot4_a_zero( ; CHECK-SAME: i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 0, i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 0, i32 [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot4(i32 0, i32 %b, i32 7, i1 false) @@ -252,8 +238,7 @@ define i32 @sdot4_const_lhs(i32 %b, i32 %acc) { define i32 @sdot4_b_zero(i32 %a) { ; CHECK-LABEL: define i32 @sdot4_b_zero( ; CHECK-SAME: i32 [[A:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 0, i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 0, i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 0, i32 7, i1 false) @@ -264,8 +249,7 @@ define i32 @sdot4_b_zero(i32 %a) { define i32 @sdot4_sub_to_zero(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sdot4_sub_to_zero( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 9, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 0, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 %b, i32 9, i1 false) @@ -276,8 +260,7 @@ define i32 @sdot4_sub_to_zero(i32 %a, i32 %b) { define i32 @sdot4_add_chain(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sdot4_add_chain( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 6 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A]], i32 [[B]], i32 13, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 %b, i32 7, i1 false) @@ -289,8 +272,7 @@ define i32 @sdot4_add_chain(i32 %a, i32 %b) { define i32 @sdot4_const_nonzero(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sdot4_const_nonzero( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 1, i32 2, i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 1, i32 2, i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot4(i32 1, i32 2, i32 7, i1 false) @@ -372,8 +354,7 @@ define void @sdot4_no_use(i32 %a, i32 %b) { define i32 @udot4_add(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @udot4_add( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot4(i32 %a, i32 %b, i32 7, i1 false) @@ -384,8 +365,7 @@ define i32 @udot4_add(i32 %a, i32 %b) { define i32 @udot4_sub(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @udot4_sub( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 [[B]], i32 6, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot4(i32 %a, i32 %b, i32 7, i1 false) @@ -396,8 +376,7 @@ define i32 @udot4_sub(i32 %a, i32 %b) { define i32 @udot4_a_zero(i32 %b) { ; CHECK-LABEL: define i32 @udot4_a_zero( ; CHECK-SAME: i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 0, i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot4(i32 0, i32 [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot4(i32 0, i32 %b, i32 7, i1 false) @@ -418,8 +397,7 @@ define i32 @udot4_const_lhs(i32 %b, i32 %acc) { define i32 @udot4_b_zero(i32 %a) { ; CHECK-LABEL: define i32 @udot4_b_zero( ; CHECK-SAME: i32 [[A:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 0, i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A]], i32 0, i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot4(i32 %a, i32 0, i32 7, i1 false) @@ -430,8 +408,7 @@ define i32 @udot4_b_zero(i32 %a) { define i32 @udot4_const_nonzero(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @udot4_const_nonzero( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 1, i32 2, i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot4(i32 1, i32 2, i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot4(i32 1, i32 2, i32 7, i1 false) @@ -454,8 +431,7 @@ define i32 @udot4_clamp(i32 %a, i32 %b) { define i32 @sdot8_add(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sdot8_add( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot8(i32 %a, i32 %b, i32 7, i1 false) @@ -466,8 +442,7 @@ define i32 @sdot8_add(i32 %a, i32 %b) { define i32 @sdot8_sub(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sdot8_sub( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 [[B]], i32 6, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot8(i32 %a, i32 %b, i32 7, i1 false) @@ -478,8 +453,7 @@ define i32 @sdot8_sub(i32 %a, i32 %b) { define i32 @sdot8_a_zero(i32 %b) { ; CHECK-LABEL: define i32 @sdot8_a_zero( ; CHECK-SAME: i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 0, i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 0, i32 [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot8(i32 0, i32 %b, i32 7, i1 false) @@ -500,8 +474,7 @@ define i32 @sdot8_const_lhs(i32 %b, i32 %acc) { define i32 @sdot8_b_zero(i32 %a) { ; CHECK-LABEL: define i32 @sdot8_b_zero( ; CHECK-SAME: i32 [[A:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 0, i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 [[A]], i32 0, i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot8(i32 %a, i32 0, i32 7, i1 false) @@ -512,8 +485,7 @@ define i32 @sdot8_b_zero(i32 %a) { define i32 @sdot8_const_nonzero(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @sdot8_const_nonzero( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 1, i32 2, i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.sdot8(i32 1, i32 2, i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.sdot8(i32 1, i32 2, i32 7, i1 false) @@ -536,8 +508,7 @@ define i32 @sdot8_clamp(i32 %a, i32 %b) { define i32 @udot8_add(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @udot8_add( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot8(i32 %a, i32 %b, i32 7, i1 false) @@ -548,8 +519,7 @@ define i32 @udot8_add(i32 %a, i32 %b) { define i32 @udot8_sub(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @udot8_sub( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], -1 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 [[B]], i32 6, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot8(i32 %a, i32 %b, i32 7, i1 false) @@ -560,8 +530,7 @@ define i32 @udot8_sub(i32 %a, i32 %b) { define i32 @udot8_a_zero(i32 %b) { ; CHECK-LABEL: define i32 @udot8_a_zero( ; CHECK-SAME: i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot8(i32 0, i32 [[B]], i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot8(i32 0, i32 [[B]], i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot8(i32 0, i32 %b, i32 7, i1 false) @@ -582,8 +551,7 @@ define i32 @udot8_const_lhs(i32 %b, i32 %acc) { define i32 @udot8_b_zero(i32 %a) { ; CHECK-LABEL: define i32 @udot8_b_zero( ; CHECK-SAME: i32 [[A:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 0, i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot8(i32 [[A]], i32 0, i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot8(i32 %a, i32 0, i32 7, i1 false) @@ -594,8 +562,7 @@ define i32 @udot8_b_zero(i32 %a) { define i32 @udot8_const_nonzero(i32 %a, i32 %b) { ; CHECK-LABEL: define i32 @udot8_const_nonzero( ; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { -; CHECK-NEXT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot8(i32 1, i32 2, i32 7, i1 false) -; CHECK-NEXT: [[R:%.*]] = add i32 [[DOT]], 9 +; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.amdgcn.udot8(i32 1, i32 2, i32 16, i1 false) ; CHECK-NEXT: ret i32 [[R]] ; %dot = call i32 @llvm.amdgcn.udot8(i32 1, i32 2, i32 7, i1 false) _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
