https://github.com/harrisonGPU updated https://github.com/llvm/llvm-project/pull/227238
>From 84013e32c88648789086ba9a84ecdda7a90cd249 Mon Sep 17 00:00:00 2001 From: Harrison Hao <[email protected]> Date: Tue, 29 Sep 2026 16:51:08 +0800 Subject: [PATCH] [InstCombine] Fold fdiv by splat of pow/exp/powi into fmul Extend foldFDivPowDivisor to look through a one-use splat divisor. The exponent is negated on the scalar and the result is splatted again: $$\frac{Z}{\mathrm{splat}(x^{y})} \to Z \cdot \mathrm{splat}(x^{-y})$$ $$\frac{Z}{\mathrm{splat}(e^{y})} \to Z \cdot \mathrm{splat}(e^{-y})$$ $$\frac{Z}{\mathrm{splat}(2^{y})} \to Z \cdot \mathrm{splat}(2^{-y})$$ $$\frac{Z}{\mathrm{splat}(x^{n})} \to Z \cdot \mathrm{splat}(x^{-n}), \quad n \in \mathbb{Z}\ (\mathrm{powi})$$ Same FMF requirements as the scalar fold: reassoc and arcp, plus ninf for powi. This removes the reciprocal, e.g. v_rcp on AMDGPU. --- .../InstCombine/InstCombineMulDivRem.cpp | 38 ++++++++++++++----- llvm/test/Transforms/InstCombine/fdiv.ll | 25 +++++++----- 2 files changed, 43 insertions(+), 20 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp index 16a7b98666d743..cf3bf555855917 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp @@ -2107,17 +2107,31 @@ static Instruction *foldFDivConstantDividend(BinaryOperator &I) { /// Negate the exponent of pow/exp to fold division-by-pow() into multiply. static Instruction *foldFDivPowDivisor(BinaryOperator &I, InstCombiner::BuilderTy &Builder) { - Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); - auto *II = dyn_cast<IntrinsicInst>(Op1); - if (!II || !II->hasOneUse() || !I.hasAllowReassoc() || - !I.hasAllowReciprocal()) - return nullptr; - // Z / pow(X, Y) --> Z * pow(X, -Y) // Z / exp{2}(Y) --> Z * exp{2}(-Y) + // Z / splat(pow(X, Y)) --> Z * splat(pow(X, -Y)) // In the general case, this creates an extra instruction, but fmul allows // for better canonicalization and optimization than fdiv. + if (!I.hasAllowReassoc() || !I.hasAllowReciprocal()) + return nullptr; + + Value *Op0 = I.getOperand(0); + Value *Op1 = I.getOperand(1); + + Value *Divisor = Op1; + Value *Splat = nullptr; + if (match(Op1, + m_OneUse(m_Shuffle( + m_OneUse(m_InsertElt(m_Value(), m_Value(Splat), m_ZeroInt())), + m_Value(), m_ZeroMask())))) + Divisor = Splat; + + auto *II = dyn_cast<IntrinsicInst>(Divisor); + if (!II || !II->hasOneUse()) + return nullptr; + Intrinsic::ID IID = II->getIntrinsicID(); + SmallVector<Type *, 2> Tys = {II->getType()}; SmallVector<Value *> Args; switch (IID) { case Intrinsic::pow: @@ -2134,9 +2148,8 @@ static Instruction *foldFDivPowDivisor(BinaryOperator &I, return nullptr; Args.push_back(II->getArgOperand(0)); Args.push_back(Builder.CreateNeg(II->getArgOperand(1))); - Type *Tys[] = {I.getType(), II->getArgOperand(1)->getType()}; - Value *Pow = Builder.CreateIntrinsic(IID, Tys, Args, &I); - return BinaryOperator::CreateFMulFMF(Op0, Pow, &I); + Tys.push_back(II->getArgOperand(1)->getType()); + break; } case Intrinsic::exp: case Intrinsic::exp2: @@ -2145,7 +2158,12 @@ static Instruction *foldFDivPowDivisor(BinaryOperator &I, default: return nullptr; } - Value *Pow = Builder.CreateIntrinsic(IID, I.getType(), Args, &I); + + Value *Pow = Builder.CreateIntrinsic(IID, Tys, Args, &I); + if (Pow->getType() != I.getType()) + Pow = Builder.CreateVectorSplat( + cast<VectorType>(I.getType())->getElementCount(), Pow); + return BinaryOperator::CreateFMulFMF(Op0, Pow, &I); } diff --git a/llvm/test/Transforms/InstCombine/fdiv.ll b/llvm/test/Transforms/InstCombine/fdiv.ll index 212d1e15e76b5b..c38ffe0fd4c88b 100644 --- a/llvm/test/Transforms/InstCombine/fdiv.ll +++ b/llvm/test/Transforms/InstCombine/fdiv.ll @@ -941,10 +941,11 @@ declare void @use_v2f32(<2 x float>) define <2 x float> @pow_divisor_splat(float %x, float %y, <2 x float> %z) { ; CHECK-LABEL: @pow_divisor_splat( -; CHECK-NEXT: [[P:%.*]] = call float @llvm.pow.f32(float [[X:%.*]], float [[Y:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = fneg reassoc arcp float [[Y:%.*]] +; CHECK-NEXT: [[P:%.*]] = call reassoc arcp float @llvm.pow.f32(float [[X:%.*]], float [[TMP1]]) ; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> poison, float [[P]], i64 0 ; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <2 x float> [[INS]], <2 x float> poison, <2 x i32> zeroinitializer -; CHECK-NEXT: [[R:%.*]] = fdiv reassoc arcp <2 x float> [[Z:%.*]], [[SPLAT]] +; CHECK-NEXT: [[R:%.*]] = fmul reassoc arcp <2 x float> [[Z:%.*]], [[SPLAT]] ; CHECK-NEXT: ret <2 x float> [[R]] ; %p = call float @llvm.pow.f32(float %x, float %y) @@ -956,10 +957,11 @@ define <2 x float> @pow_divisor_splat(float %x, float %y, <2 x float> %z) { define <vscale x 2 x float> @pow_divisor_splat_scalable(float %x, float %y, <vscale x 2 x float> %z) { ; CHECK-LABEL: @pow_divisor_splat_scalable( -; CHECK-NEXT: [[P:%.*]] = call float @llvm.pow.f32(float [[X:%.*]], float [[Y:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = fneg reassoc arcp float [[Y:%.*]] +; CHECK-NEXT: [[P:%.*]] = call reassoc arcp float @llvm.pow.f32(float [[X:%.*]], float [[TMP1]]) ; CHECK-NEXT: [[INS:%.*]] = insertelement <vscale x 2 x float> poison, float [[P]], i64 0 ; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <vscale x 2 x float> [[INS]], <vscale x 2 x float> poison, <vscale x 2 x i32> zeroinitializer -; CHECK-NEXT: [[R:%.*]] = fdiv reassoc arcp <vscale x 2 x float> [[Z:%.*]], [[SPLAT]] +; CHECK-NEXT: [[R:%.*]] = fmul reassoc arcp <vscale x 2 x float> [[Z:%.*]], [[SPLAT]] ; CHECK-NEXT: ret <vscale x 2 x float> [[R]] ; %p = call float @llvm.pow.f32(float %x, float %y) @@ -1073,10 +1075,11 @@ define <2 x float> @pow_recip_splat(float %x, float %y) { define <2 x float> @exp_divisor_splat(float %y, <2 x float> %z) { ; CHECK-LABEL: @exp_divisor_splat( -; CHECK-NEXT: [[P:%.*]] = call float @llvm.exp.f32(float [[Y:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = fneg reassoc arcp float [[Y:%.*]] +; CHECK-NEXT: [[P:%.*]] = call reassoc arcp float @llvm.exp.f32(float [[TMP1]]) ; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> poison, float [[P]], i64 0 ; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <2 x float> [[INS]], <2 x float> poison, <2 x i32> zeroinitializer -; CHECK-NEXT: [[R:%.*]] = fdiv reassoc arcp <2 x float> [[Z:%.*]], [[SPLAT]] +; CHECK-NEXT: [[R:%.*]] = fmul reassoc arcp <2 x float> [[Z:%.*]], [[SPLAT]] ; CHECK-NEXT: ret <2 x float> [[R]] ; %p = call float @llvm.exp.f32(float %y) @@ -1088,10 +1091,11 @@ define <2 x float> @exp_divisor_splat(float %y, <2 x float> %z) { define <2 x float> @exp2_divisor_splat(float %y, <2 x float> %z) { ; CHECK-LABEL: @exp2_divisor_splat( -; CHECK-NEXT: [[P:%.*]] = call float @llvm.exp2.f32(float [[Y:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = fneg reassoc arcp float [[Y:%.*]] +; CHECK-NEXT: [[P:%.*]] = call reassoc arcp float @llvm.exp2.f32(float [[TMP1]]) ; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> poison, float [[P]], i64 0 ; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <2 x float> [[INS]], <2 x float> poison, <2 x i32> zeroinitializer -; CHECK-NEXT: [[R:%.*]] = fdiv reassoc arcp <2 x float> [[Z:%.*]], [[SPLAT]] +; CHECK-NEXT: [[R:%.*]] = fmul reassoc arcp <2 x float> [[Z:%.*]], [[SPLAT]] ; CHECK-NEXT: ret <2 x float> [[R]] ; %p = call float @llvm.exp2.f32(float %y) @@ -1103,10 +1107,11 @@ define <2 x float> @exp2_divisor_splat(float %y, <2 x float> %z) { define <2 x float> @powi_divisor_splat(float %x, i32 %y, <2 x float> %z) { ; CHECK-LABEL: @powi_divisor_splat( -; CHECK-NEXT: [[P:%.*]] = call float @llvm.powi.f32.i32(float [[X:%.*]], i32 [[Y:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = sub i32 0, [[Y:%.*]] +; CHECK-NEXT: [[P:%.*]] = call reassoc ninf arcp float @llvm.powi.f32.i32(float [[X:%.*]], i32 [[TMP1]]) ; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> poison, float [[P]], i64 0 ; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <2 x float> [[INS]], <2 x float> poison, <2 x i32> zeroinitializer -; CHECK-NEXT: [[R:%.*]] = fdiv reassoc ninf arcp <2 x float> [[Z:%.*]], [[SPLAT]] +; CHECK-NEXT: [[R:%.*]] = fmul reassoc ninf arcp <2 x float> [[Z:%.*]], [[SPLAT]] ; CHECK-NEXT: ret <2 x float> [[R]] ; %p = call float @llvm.powi.f32.i32(float %x, i32 %y) _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
