https://github.com/harrisonGPU updated https://github.com/llvm/llvm-project/pull/229357
>From 7d5adbdd46ffb1ec2a6879bcb1bd9ca21f2547bc Mon Sep 17 00:00:00 2001 From: Harrison Hao <[email protected]> Date: Tue, 6 Oct 2026 17:10:24 +0800 Subject: [PATCH 1/3] [AMDGPU] Fold fmul with zero checks to fmul.legacy Recognize f32 multiplication with zero checks in AMDGPUCodeGenPrepare, including fabs/fneg wrappers. Example: ``` (x == 0 || y == 0) ? +0.0 : x * y ==> llvm.amdgcn.fmul.legacy(x, y) ``` --- .../Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 77 ++++++++++++ .../amdgpu-codegenprepare-fmul-legacy.ll | 114 +++--------------- 2 files changed, 96 insertions(+), 95 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp index 4b669756d83f6..7fc17075a15cb 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp @@ -177,6 +177,9 @@ class AMDGPUCodeGenPrepareImpl /// we expand some divisions here, we need to perform this before obscuring. bool foldBinOpIntoSelect(BinaryOperator &I) const; + /// Fold select (X == 0 || Y == 0), +0.0, X * Y into fmul_legacy(X, Y). + bool foldSelectToFMulLegacy(SelectInst &I); + bool divHasSpecialOptimization(BinaryOperator &I, Value *Num, Value *Den) const; unsigned getDivNumBits(BinaryOperator &I, Value *Num, Value *Den, @@ -1578,11 +1581,85 @@ bool AMDGPUCodeGenPrepareImpl::visitLoadInst(LoadInst &I) { return false; } +// select (X == 0 || Y == 0), +0.0, (X * Y) -> fmul_legacy(X, Y) +bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) { + if (!I.getType()->isFloatTy()) + return false; + + // Inverted form: select (X != 0 && Y != 0), X * Y, +0.0. + bool Inverted = match(I.getFalseValue(), m_PosZeroFP()); + if (!Inverted && !match(I.getTrueValue(), m_PosZeroFP())) + return false; + + Value *X, *Y; + if (!match(Inverted ? I.getTrueValue() : I.getFalseValue(), + m_OneUse(m_FMul(m_Value(X), m_Value(Y))))) + return false; + + SmallVector<Value *, 4> Roots[2] = {{I.getCondition()}, {X, Y}}; + SmallPtrSet<Value *, 4> Leaves[2]; + for (auto [Worklist, Found] : zip(Roots, Leaves)) { + SmallPtrSet<Value *, 8> Visited; + + while (!Worklist.empty()) { + Value *Cur = Worklist.pop_back_val(); + if (!Visited.insert(Cur).second) + continue; + + Value *Src, *LHS, *RHS; + if (Cur->getType()->isIntegerTy(1)) { + if (Inverted ? match(Cur, m_And(m_Value(LHS), m_Value(RHS))) + : match(Cur, m_Or(m_Value(LHS), m_Value(RHS)))) { + Worklist.append({LHS, RHS}); + continue; + } + + if (!match(Cur, m_SpecificFCmp(Inverted ? FCmpInst::FCMP_UNE + : FCmpInst::FCMP_OEQ, + m_Value(Src), m_AnyZeroFP()))) + return false; + + Worklist.push_back(Src); + continue; + } + + // Peeling a nnan/ninf op could expose poison that the select hides. + auto *FPOp = dyn_cast<FPMathOperator>(Cur); + if (FPOp && (FPOp->hasNoNaNs() || FPOp->hasNoInfs())) { + Found.insert(Cur); + continue; + } + + // m_UnOp, not m_FNeg: fsub -0.0, X may flush a denormal result to zero. + if (match(Cur, m_CombineOr(m_UnOp(m_Value(Src)), m_FAbs(m_Value(Src))))) { + Worklist.push_back(Src); + continue; + } + + Found.insert(Cur); + } + } + + if (Leaves[0] != Leaves[1]) + return false; + + IRBuilder<> Builder(&I); + Value *LegacyMul = + Builder.CreateIntrinsic(Intrinsic::amdgcn_fmul_legacy, {}, {X, Y}); + LegacyMul->takeName(&I); + I.replaceAllUsesWith(LegacyMul); + DeadVals.push_back(&I); + return true; +} + bool AMDGPUCodeGenPrepareImpl::visitSelectInst(SelectInst &I) { FPMathOperator *FPOp = dyn_cast<FPMathOperator>(&I); if (!FPOp) return false; + if (foldSelectToFMulLegacy(I)) + return true; + Value *X; Value *Fract = nullptr; diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll index 05b603a449c3f..09a1233cd1833 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll @@ -4,11 +4,7 @@ define float @select_fmul_legacy(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -22,11 +18,7 @@ define float @select_fmul_legacy(float %x, float %y) { define float @select_fmul_legacy_uniform(float inreg %x, float inreg %y) { ; CHECK-LABEL: define float @select_fmul_legacy_uniform( ; CHECK-SAME: float inreg [[X:%.*]], float inreg [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -40,11 +32,7 @@ define float @select_fmul_legacy_uniform(float inreg %x, float inreg %y) { define float @select_fmul_legacy_commute_fmul(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_commute_fmul( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[Y]], [[X]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[Y]], float [[X]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -58,11 +46,7 @@ define float @select_fmul_legacy_commute_fmul(float %x, float %y) { define float @select_fmul_legacy_commute_or(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_commute_or( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[YZ]], [[XZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -76,11 +60,7 @@ define float @select_fmul_legacy_commute_or(float %x, float %y) { define float @select_fmul_legacy_cmp_neg_zero(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_cmp_neg_zero( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], -0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], -0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, -0.0 @@ -189,11 +169,7 @@ define float @select_fmul_legacy_logical_or(float %x, float %y) { define float @select_fmul_legacy_inverted(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_inverted( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XNZ:%.*]] = fcmp une float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YNZ:%.*]] = fcmp une float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = and i1 [[XNZ]], [[YNZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float [[MUL]], float 0.000000e+00 +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xnz = fcmp une float %x, 0.0 @@ -438,9 +414,7 @@ define <2 x float> @select_fmul_legacy_v2f32(<2 x float> %x, <2 x float> %y) { define float @select_fmul_legacy_square(float %x) { ; CHECK-LABEL: define float @select_fmul_legacy_square( ; CHECK-SAME: float [[X:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[X]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[XZ]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[X]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -452,11 +426,7 @@ define float @select_fmul_legacy_square(float %x) { define float @select_fmul_legacy_square_or(float %x) { ; CHECK-LABEL: define float @select_fmul_legacy_square_or( ; CHECK-SAME: float [[X:%.*]]) { -; CHECK-NEXT: [[XZ0:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[XZ1:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ0]], [[XZ1]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[X]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[X]]) ; CHECK-NEXT: ret float [[R]] ; %xz0 = fcmp oeq float %x, 0.0 @@ -542,12 +512,8 @@ define float @select_fmul_legacy_const_operand(float %x) { define float @select_fmul_legacy_fabs_mul_operand(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_fabs_mul_operand( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] ; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X]]) -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[FABS_X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[FABS_X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -562,12 +528,7 @@ define float @select_fmul_legacy_fabs_mul_operand(float %x, float %y) { define float @select_fmul_legacy_fabs_cmp_operand(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_fabs_cmp_operand( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X]]) -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[FABS_X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %fabs.x = call float @llvm.fabs.f32(float %x) @@ -582,12 +543,8 @@ define float @select_fmul_legacy_fabs_cmp_operand(float %x, float %y) { define float @select_fmul_legacy_fneg_mul_operand(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_fneg_mul_operand( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] ; CHECK-NEXT: [[FNEG_X:%.*]] = fneg float [[X]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[FNEG_X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[FNEG_X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -602,12 +559,7 @@ define float @select_fmul_legacy_fneg_mul_operand(float %x, float %y) { define float @select_fmul_legacy_fneg_cmp_operand(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_fneg_cmp_operand( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[FNEG_X:%.*]] = fneg float [[X]] -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[FNEG_X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %fneg.x = fneg float %x @@ -624,11 +576,7 @@ define float @select_fmul_legacy_fneg_fabs(float %x, float %y) { ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { ; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X]]) ; CHECK-NEXT: [[FNEG_FABS_X:%.*]] = fneg float [[FABS_X]] -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[FABS_X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[FNEG_FABS_X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[FNEG_FABS_X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %fabs.x = call float @llvm.fabs.f32(float %x) @@ -644,12 +592,8 @@ define float @select_fmul_legacy_fneg_fabs(float %x, float %y) { define float @select_fmul_legacy_fneg_nsz(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_fneg_nsz( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] ; CHECK-NEXT: [[FNEG_X:%.*]] = fneg nsz float [[X]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[FNEG_X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[FNEG_X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -748,11 +692,7 @@ define float @select_fmul_legacy_copysign_mul_operand(float %x, float %y, float define float @select_fmul_legacy_select_flags(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_select_flags( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select nsz i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -767,11 +707,7 @@ define float @select_fmul_legacy_select_flags(float %x, float %y) { define float @select_fmul_legacy_select_nnan_ninf(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_select_nnan_ninf( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select nnan ninf i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -785,11 +721,7 @@ define float @select_fmul_legacy_select_nnan_ninf(float %x, float %y) { define float @select_fmul_legacy_fmul_flags(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_fmul_flags( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul nnan ninf float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -821,11 +753,7 @@ define float @select_fmul_legacy_fmul_fast(float %x, float %y) { define float @select_fmul_legacy_fcmp_flags(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_fcmp_flags( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp nnan oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp nnan oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp nnan oeq float %x, 0.0 @@ -839,11 +767,7 @@ define float @select_fmul_legacy_fcmp_flags(float %x, float %y) { define float @select_fmul_legacy_daz(float %x, float %y) #0 { ; CHECK-LABEL: define float @select_fmul_legacy_daz( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR0:[0-9]+]] { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 >From 2220b5b3fada3f461369731104927cc129819745 Mon Sep 17 00:00:00 2001 From: Harrison Hao <[email protected]> Date: Wed, 21 Oct 2026 11:00:52 +0800 Subject: [PATCH 2/3] Update for comments --- .../Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 49 +++++++++++-------- .../amdgpu-codegenprepare-fmul-legacy.ll | 7 +-- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp index 7fc17075a15cb..2216c8b1acb96 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp @@ -178,7 +178,7 @@ class AMDGPUCodeGenPrepareImpl bool foldBinOpIntoSelect(BinaryOperator &I) const; /// Fold select (X == 0 || Y == 0), +0.0, X * Y into fmul_legacy(X, Y). - bool foldSelectToFMulLegacy(SelectInst &I); + bool foldSelectToFMulLegacy(SelectInst &I) const; bool divHasSpecialOptimization(BinaryOperator &I, Value *Num, Value *Den) const; @@ -1582,7 +1582,7 @@ bool AMDGPUCodeGenPrepareImpl::visitLoadInst(LoadInst &I) { } // select (X == 0 || Y == 0), +0.0, (X * Y) -> fmul_legacy(X, Y) -bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) { +bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) const { if (!I.getType()->isFloatTy()) return false; @@ -1596,7 +1596,30 @@ bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) { m_OneUse(m_FMul(m_Value(X), m_Value(Y))))) return false; - SmallVector<Value *, 4> Roots[2] = {{I.getCondition()}, {X, Y}}; + // Collect the values compared against zero in the condition. + SmallVector<Value *, 4> CondWorklist = {I.getCondition()}; + SmallVector<Value *, 4> CmpSrcs; + SmallPtrSet<Value *, 8> VisitedConds; + while (!CondWorklist.empty()) { + Value *Cond = CondWorklist.pop_back_val(); + if (!VisitedConds.insert(Cond).second) + continue; + + Value *LHS, *RHS, *Src; + if (Inverted ? match(Cond, m_And(m_Value(LHS), m_Value(RHS))) + : match(Cond, m_Or(m_Value(LHS), m_Value(RHS)))) { + CondWorklist.append({LHS, RHS}); + continue; + } + + if (!match(Cond, m_SpecificFCmp(Inverted ? FCmpInst::FCMP_UNE + : FCmpInst::FCMP_OEQ, + m_Value(Src), m_AnyZeroFP()))) + return false; + CmpSrcs.push_back(Src); + } + + SmallVector<Value *, 4> Roots[2] = {std::move(CmpSrcs), {X, Y}}; SmallPtrSet<Value *, 4> Leaves[2]; for (auto [Worklist, Found] : zip(Roots, Leaves)) { SmallPtrSet<Value *, 8> Visited; @@ -1606,23 +1629,6 @@ bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) { if (!Visited.insert(Cur).second) continue; - Value *Src, *LHS, *RHS; - if (Cur->getType()->isIntegerTy(1)) { - if (Inverted ? match(Cur, m_And(m_Value(LHS), m_Value(RHS))) - : match(Cur, m_Or(m_Value(LHS), m_Value(RHS)))) { - Worklist.append({LHS, RHS}); - continue; - } - - if (!match(Cur, m_SpecificFCmp(Inverted ? FCmpInst::FCMP_UNE - : FCmpInst::FCMP_OEQ, - m_Value(Src), m_AnyZeroFP()))) - return false; - - Worklist.push_back(Src); - continue; - } - // Peeling a nnan/ninf op could expose poison that the select hides. auto *FPOp = dyn_cast<FPMathOperator>(Cur); if (FPOp && (FPOp->hasNoNaNs() || FPOp->hasNoInfs())) { @@ -1630,6 +1636,7 @@ bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) { continue; } + Value *Src; // m_UnOp, not m_FNeg: fsub -0.0, X may flush a denormal result to zero. if (match(Cur, m_CombineOr(m_UnOp(m_Value(Src)), m_FAbs(m_Value(Src))))) { Worklist.push_back(Src); @@ -1645,7 +1652,7 @@ bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) { IRBuilder<> Builder(&I); Value *LegacyMul = - Builder.CreateIntrinsic(Intrinsic::amdgcn_fmul_legacy, {}, {X, Y}); + Builder.CreateIntrinsic(Intrinsic::amdgcn_fmul_legacy, {X, Y}); LegacyMul->takeName(&I); I.replaceAllUsesWith(LegacyMul); DeadVals.push_back(&I); diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll index 09a1233cd1833..a577113adc640 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll @@ -240,12 +240,7 @@ define float @select_fmul_legacy_inverted_logical_and(float %x, float %y) { define float @select_fmul_legacy_inverted_fabs(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_inverted_fabs( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X]]) -; CHECK-NEXT: [[XNZ:%.*]] = fcmp une float [[FABS_X]], 0.000000e+00 -; CHECK-NEXT: [[YNZ:%.*]] = fcmp une float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = and i1 [[XNZ]], [[YNZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float [[MUL]], float 0.000000e+00 +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %fabs.x = call float @llvm.fabs.f32(float %x) >From 74fea14fa792211d1af8ea168f4ca7c5452222c7 Mon Sep 17 00:00:00 2001 From: Harrison Hao <[email protected]> Date: Wed, 21 Oct 2026 12:33:27 +0800 Subject: [PATCH 3/3] Remove worklist --- .../Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 79 +++++++------------ .../amdgpu-codegenprepare-fmul-legacy.ll | 8 +- 2 files changed, 30 insertions(+), 57 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp index 2216c8b1acb96..7dd9e41c7cfa4 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp @@ -32,6 +32,7 @@ #include "llvm/Pass.h" #include "llvm/Support/KnownBits.h" #include "llvm/Support/KnownFPClass.h" +#include "llvm/Transforms/InstCombine/InstCombiner.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/IntegerDivision.h" #include "llvm/Transforms/Utils/Local.h" @@ -1591,68 +1592,44 @@ bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) const { if (!Inverted && !match(I.getTrueValue(), m_PosZeroFP())) return false; + Value *Mul = Inverted ? I.getTrueValue() : I.getFalseValue(); Value *X, *Y; - if (!match(Inverted ? I.getTrueValue() : I.getFalseValue(), - m_OneUse(m_FMul(m_Value(X), m_Value(Y))))) + if (!match(Mul, m_OneUse(m_FMul(m_Value(X), m_Value(Y))))) return false; - // Collect the values compared against zero in the condition. - SmallVector<Value *, 4> CondWorklist = {I.getCondition()}; - SmallVector<Value *, 4> CmpSrcs; - SmallPtrSet<Value *, 8> VisitedConds; - while (!CondWorklist.empty()) { - Value *Cond = CondWorklist.pop_back_val(); - if (!VisitedConds.insert(Cond).second) - continue; - - Value *LHS, *RHS, *Src; - if (Inverted ? match(Cond, m_And(m_Value(LHS), m_Value(RHS))) - : match(Cond, m_Or(m_Value(LHS), m_Value(RHS)))) { - CondWorklist.append({LHS, RHS}); - continue; - } - - if (!match(Cond, m_SpecificFCmp(Inverted ? FCmpInst::FCMP_UNE - : FCmpInst::FCMP_OEQ, - m_Value(Src), m_AnyZeroFP()))) + FCmpInst::Predicate ZeroCmpPred = + Inverted ? FCmpInst::FCMP_UNE : FCmpInst::FCMP_OEQ; + Value *A, *B; + auto ZeroCmpA = m_SpecificFCmp(ZeroCmpPred, m_Value(A), m_AnyZeroFP()); + auto ZeroCmpB = m_SpecificFCmp(ZeroCmpPred, m_Value(B), m_AnyZeroFP()); + Value *Cond = I.getCondition(); + if (Inverted ? !match(Cond, m_And(ZeroCmpA, ZeroCmpB)) + : !match(Cond, m_Or(ZeroCmpA, ZeroCmpB))) { + if (!match(Cond, ZeroCmpA)) return false; - CmpSrcs.push_back(Src); + B = A; } - SmallVector<Value *, 4> Roots[2] = {std::move(CmpSrcs), {X, Y}}; - SmallPtrSet<Value *, 4> Leaves[2]; - for (auto [Worklist, Found] : zip(Roots, Leaves)) { - SmallPtrSet<Value *, 8> Visited; - - while (!Worklist.empty()) { - Value *Cur = Worklist.pop_back_val(); - if (!Visited.insert(Cur).second) - continue; - - // Peeling a nnan/ninf op could expose poison that the select hides. - auto *FPOp = dyn_cast<FPMathOperator>(Cur); - if (FPOp && (FPOp->hasNoNaNs() || FPOp->hasNoInfs())) { - Found.insert(Cur); - continue; - } - - Value *Src; - // m_UnOp, not m_FNeg: fsub -0.0, X may flush a denormal result to zero. - if (match(Cur, m_CombineOr(m_UnOp(m_Value(Src)), m_FAbs(m_Value(Src))))) { - Worklist.push_back(Src); - continue; - } - - Found.insert(Cur); - } - } + Value *SA = InstCombiner::stripSignOnlyFPOps(A); + Value *SB = InstCombiner::stripSignOnlyFPOps(B); + Value *SX = InstCombiner::stripSignOnlyFPOps(X); + Value *SY = InstCombiner::stripSignOnlyFPOps(Y); + if (!((SA == SX && SB == SY) || (SA == SY && SB == SX))) + return false; - if (Leaves[0] != Leaves[1]) + // A stripped nnan/ninf op could create poison that the select hides. + if (!impliesPoison(X, SX) || !impliesPoison(Y, SY)) return false; + // The select may hide NaN, Inf or a signed zero from the fmul. + FastMathFlags FMF = cast<Instruction>(Mul)->getFastMathFlags(); + FMF.setNoNaNs(false); + FMF.setNoInfs(false); + FMF.setNoSignedZeros(I.hasNoSignedZeros()); + IRBuilder<> Builder(&I); Value *LegacyMul = - Builder.CreateIntrinsic(Intrinsic::amdgcn_fmul_legacy, {X, Y}); + Builder.CreateIntrinsic(Intrinsic::amdgcn_fmul_legacy, {X, Y}, FMF); LegacyMul->takeName(&I); I.replaceAllUsesWith(LegacyMul); DeadVals.push_back(&I); diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll index a577113adc640..224269bf3dfce 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll @@ -687,7 +687,7 @@ define float @select_fmul_legacy_copysign_mul_operand(float %x, float %y, float define float @select_fmul_legacy_select_flags(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_select_flags( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) +; CHECK-NEXT: [[R:%.*]] = call nsz float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -730,11 +730,7 @@ define float @select_fmul_legacy_fmul_flags(float %x, float %y) { define float @select_fmul_legacy_fmul_fast(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_fmul_fast( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul fast float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call reassoc arcp contract afn float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
