https://github.com/harrisonGPU updated https://github.com/llvm/llvm-project/pull/229357
>From e3b261a4c71295ea8988e64e1db410b7013e4ac8 Mon Sep 17 00:00:00 2001 From: Harrison Hao <[email protected]> Date: Tue, 6 Oct 2026 17:10:24 +0800 Subject: [PATCH 1/2] [AMDGPU] Fold fmul with zero checks to fmul.legacy Recognize f32 multiplication with zero checks in AMDGPUCodeGenPrepare, including fabs/fneg wrappers. Example: ``` (x == 0 || y == 0) ? +0.0 : x * y ==> llvm.amdgcn.fmul.legacy(x, y) ``` --- .../Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 77 +++++++++++ .../amdgpu-codegenprepare-fmul-legacy.ll | 128 +++--------------- 2 files changed, 98 insertions(+), 107 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp index 4b669756d83f63..7fc17075a15cba 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp @@ -177,6 +177,9 @@ class AMDGPUCodeGenPrepareImpl /// we expand some divisions here, we need to perform this before obscuring. bool foldBinOpIntoSelect(BinaryOperator &I) const; + /// Fold select (X == 0 || Y == 0), +0.0, X * Y into fmul_legacy(X, Y). + bool foldSelectToFMulLegacy(SelectInst &I); + bool divHasSpecialOptimization(BinaryOperator &I, Value *Num, Value *Den) const; unsigned getDivNumBits(BinaryOperator &I, Value *Num, Value *Den, @@ -1578,11 +1581,85 @@ bool AMDGPUCodeGenPrepareImpl::visitLoadInst(LoadInst &I) { return false; } +// select (X == 0 || Y == 0), +0.0, (X * Y) -> fmul_legacy(X, Y) +bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) { + if (!I.getType()->isFloatTy()) + return false; + + // Inverted form: select (X != 0 && Y != 0), X * Y, +0.0. + bool Inverted = match(I.getFalseValue(), m_PosZeroFP()); + if (!Inverted && !match(I.getTrueValue(), m_PosZeroFP())) + return false; + + Value *X, *Y; + if (!match(Inverted ? I.getTrueValue() : I.getFalseValue(), + m_OneUse(m_FMul(m_Value(X), m_Value(Y))))) + return false; + + SmallVector<Value *, 4> Roots[2] = {{I.getCondition()}, {X, Y}}; + SmallPtrSet<Value *, 4> Leaves[2]; + for (auto [Worklist, Found] : zip(Roots, Leaves)) { + SmallPtrSet<Value *, 8> Visited; + + while (!Worklist.empty()) { + Value *Cur = Worklist.pop_back_val(); + if (!Visited.insert(Cur).second) + continue; + + Value *Src, *LHS, *RHS; + if (Cur->getType()->isIntegerTy(1)) { + if (Inverted ? match(Cur, m_And(m_Value(LHS), m_Value(RHS))) + : match(Cur, m_Or(m_Value(LHS), m_Value(RHS)))) { + Worklist.append({LHS, RHS}); + continue; + } + + if (!match(Cur, m_SpecificFCmp(Inverted ? FCmpInst::FCMP_UNE + : FCmpInst::FCMP_OEQ, + m_Value(Src), m_AnyZeroFP()))) + return false; + + Worklist.push_back(Src); + continue; + } + + // Peeling a nnan/ninf op could expose poison that the select hides. + auto *FPOp = dyn_cast<FPMathOperator>(Cur); + if (FPOp && (FPOp->hasNoNaNs() || FPOp->hasNoInfs())) { + Found.insert(Cur); + continue; + } + + // m_UnOp, not m_FNeg: fsub -0.0, X may flush a denormal result to zero. + if (match(Cur, m_CombineOr(m_UnOp(m_Value(Src)), m_FAbs(m_Value(Src))))) { + Worklist.push_back(Src); + continue; + } + + Found.insert(Cur); + } + } + + if (Leaves[0] != Leaves[1]) + return false; + + IRBuilder<> Builder(&I); + Value *LegacyMul = + Builder.CreateIntrinsic(Intrinsic::amdgcn_fmul_legacy, {}, {X, Y}); + LegacyMul->takeName(&I); + I.replaceAllUsesWith(LegacyMul); + DeadVals.push_back(&I); + return true; +} + bool AMDGPUCodeGenPrepareImpl::visitSelectInst(SelectInst &I) { FPMathOperator *FPOp = dyn_cast<FPMathOperator>(&I); if (!FPOp) return false; + if (foldSelectToFMulLegacy(I)) + return true; + Value *X; Value *Fract = nullptr; diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll index 8731f24dbd3aae..c0a5d20503f7a1 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll @@ -4,11 +4,7 @@ define float @select_fmul_legacy(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -22,11 +18,7 @@ define float @select_fmul_legacy(float %x, float %y) { define float @select_fmul_legacy_uniform(float inreg %x, float inreg %y) { ; CHECK-LABEL: define float @select_fmul_legacy_uniform( ; CHECK-SAME: float inreg [[X:%.*]], float inreg [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -40,11 +32,7 @@ define float @select_fmul_legacy_uniform(float inreg %x, float inreg %y) { define float @select_fmul_legacy_commute_fmul(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_commute_fmul( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[Y]], [[X]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[Y]], float [[X]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -58,11 +46,7 @@ define float @select_fmul_legacy_commute_fmul(float %x, float %y) { define float @select_fmul_legacy_commute_or(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_commute_or( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[YZ]], [[XZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -76,11 +60,7 @@ define float @select_fmul_legacy_commute_or(float %x, float %y) { define float @select_fmul_legacy_cmp_neg_zero(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_cmp_neg_zero( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], -0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], -0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, -0.0 @@ -189,11 +169,7 @@ define float @select_fmul_legacy_logical_or(float %x, float %y) { define float @select_fmul_legacy_inverted(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_inverted( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XNZ:%.*]] = fcmp une float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YNZ:%.*]] = fcmp une float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = and i1 [[XNZ]], [[YNZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float [[MUL]], float 0.000000e+00 +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xnz = fcmp une float %x, 0.0 @@ -438,9 +414,7 @@ define <2 x float> @select_fmul_legacy_v2f32(<2 x float> %x, <2 x float> %y) { define float @select_fmul_legacy_square(float %x) { ; CHECK-LABEL: define float @select_fmul_legacy_square( ; CHECK-SAME: float [[X:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[X]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[XZ]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[X]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -452,11 +426,7 @@ define float @select_fmul_legacy_square(float %x) { define float @select_fmul_legacy_square_or(float %x) { ; CHECK-LABEL: define float @select_fmul_legacy_square_or( ; CHECK-SAME: float [[X:%.*]]) { -; CHECK-NEXT: [[XZ0:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[XZ1:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ0]], [[XZ1]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[X]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[X]]) ; CHECK-NEXT: ret float [[R]] ; %xz0 = fcmp oeq float %x, 0.0 @@ -470,12 +440,7 @@ define float @select_fmul_legacy_square_or(float %x) { define float @select_fmul_legacy_redundant_cmp(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_redundant_cmp( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C0:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[C:%.*]] = or i1 [[C0]], [[XZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -490,12 +455,7 @@ define float @select_fmul_legacy_redundant_cmp(float %x, float %y) { define float @select_fmul_legacy_shared_or(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_shared_or( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C0:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[C:%.*]] = or i1 [[C0]], [[C0]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -582,12 +542,8 @@ define float @select_fmul_legacy_const_operand(float %x) { define float @select_fmul_legacy_fabs_mul_operand(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_fabs_mul_operand( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] ; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X]]) -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[FABS_X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[FABS_X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -602,12 +558,7 @@ define float @select_fmul_legacy_fabs_mul_operand(float %x, float %y) { define float @select_fmul_legacy_fabs_cmp_operand(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_fabs_cmp_operand( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X]]) -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[FABS_X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %fabs.x = call float @llvm.fabs.f32(float %x) @@ -622,12 +573,8 @@ define float @select_fmul_legacy_fabs_cmp_operand(float %x, float %y) { define float @select_fmul_legacy_fneg_mul_operand(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_fneg_mul_operand( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] ; CHECK-NEXT: [[FNEG_X:%.*]] = fneg float [[X]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[FNEG_X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[FNEG_X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -642,12 +589,7 @@ define float @select_fmul_legacy_fneg_mul_operand(float %x, float %y) { define float @select_fmul_legacy_fneg_cmp_operand(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_fneg_cmp_operand( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[FNEG_X:%.*]] = fneg float [[X]] -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[FNEG_X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %fneg.x = fneg float %x @@ -664,11 +606,7 @@ define float @select_fmul_legacy_fneg_fabs(float %x, float %y) { ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { ; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X]]) ; CHECK-NEXT: [[FNEG_FABS_X:%.*]] = fneg float [[FABS_X]] -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[FABS_X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[FNEG_FABS_X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[FNEG_FABS_X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %fabs.x = call float @llvm.fabs.f32(float %x) @@ -684,12 +622,8 @@ define float @select_fmul_legacy_fneg_fabs(float %x, float %y) { define float @select_fmul_legacy_fneg_nsz(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_fneg_nsz( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] ; CHECK-NEXT: [[FNEG_X:%.*]] = fneg nsz float [[X]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[FNEG_X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[FNEG_X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -788,11 +722,7 @@ define float @select_fmul_legacy_canonicalize(float %x, float %y) { define float @select_fmul_legacy_select_flags(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_select_flags( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select nsz i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -807,11 +737,7 @@ define float @select_fmul_legacy_select_flags(float %x, float %y) { define float @select_fmul_legacy_select_nnan_ninf(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_select_nnan_ninf( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select nnan ninf i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -825,11 +751,7 @@ define float @select_fmul_legacy_select_nnan_ninf(float %x, float %y) { define float @select_fmul_legacy_fmul_flags(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_fmul_flags( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul nnan ninf float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 @@ -843,11 +765,7 @@ define float @select_fmul_legacy_fmul_flags(float %x, float %y) { define float @select_fmul_legacy_fcmp_flags(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_fcmp_flags( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[XZ:%.*]] = fcmp nnan oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp nnan oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp nnan oeq float %x, 0.0 @@ -861,11 +779,7 @@ define float @select_fmul_legacy_fcmp_flags(float %x, float %y) { define float @select_fmul_legacy_daz(float %x, float %y) #0 { ; CHECK-LABEL: define float @select_fmul_legacy_daz( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR0:[0-9]+]] { -; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00 -; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]] +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %xz = fcmp oeq float %x, 0.0 >From fe93783ea4c1291e6b534896eb858cbed259ba02 Mon Sep 17 00:00:00 2001 From: Harrison Hao <[email protected]> Date: Wed, 21 Oct 2026 11:00:52 +0800 Subject: [PATCH 2/2] Update for comments --- .../Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 49 +++++++++++-------- .../amdgpu-codegenprepare-fmul-legacy.ll | 7 +-- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp index 7fc17075a15cba..2216c8b1acb96a 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp @@ -178,7 +178,7 @@ class AMDGPUCodeGenPrepareImpl bool foldBinOpIntoSelect(BinaryOperator &I) const; /// Fold select (X == 0 || Y == 0), +0.0, X * Y into fmul_legacy(X, Y). - bool foldSelectToFMulLegacy(SelectInst &I); + bool foldSelectToFMulLegacy(SelectInst &I) const; bool divHasSpecialOptimization(BinaryOperator &I, Value *Num, Value *Den) const; @@ -1582,7 +1582,7 @@ bool AMDGPUCodeGenPrepareImpl::visitLoadInst(LoadInst &I) { } // select (X == 0 || Y == 0), +0.0, (X * Y) -> fmul_legacy(X, Y) -bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) { +bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) const { if (!I.getType()->isFloatTy()) return false; @@ -1596,7 +1596,30 @@ bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) { m_OneUse(m_FMul(m_Value(X), m_Value(Y))))) return false; - SmallVector<Value *, 4> Roots[2] = {{I.getCondition()}, {X, Y}}; + // Collect the values compared against zero in the condition. + SmallVector<Value *, 4> CondWorklist = {I.getCondition()}; + SmallVector<Value *, 4> CmpSrcs; + SmallPtrSet<Value *, 8> VisitedConds; + while (!CondWorklist.empty()) { + Value *Cond = CondWorklist.pop_back_val(); + if (!VisitedConds.insert(Cond).second) + continue; + + Value *LHS, *RHS, *Src; + if (Inverted ? match(Cond, m_And(m_Value(LHS), m_Value(RHS))) + : match(Cond, m_Or(m_Value(LHS), m_Value(RHS)))) { + CondWorklist.append({LHS, RHS}); + continue; + } + + if (!match(Cond, m_SpecificFCmp(Inverted ? FCmpInst::FCMP_UNE + : FCmpInst::FCMP_OEQ, + m_Value(Src), m_AnyZeroFP()))) + return false; + CmpSrcs.push_back(Src); + } + + SmallVector<Value *, 4> Roots[2] = {std::move(CmpSrcs), {X, Y}}; SmallPtrSet<Value *, 4> Leaves[2]; for (auto [Worklist, Found] : zip(Roots, Leaves)) { SmallPtrSet<Value *, 8> Visited; @@ -1606,23 +1629,6 @@ bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) { if (!Visited.insert(Cur).second) continue; - Value *Src, *LHS, *RHS; - if (Cur->getType()->isIntegerTy(1)) { - if (Inverted ? match(Cur, m_And(m_Value(LHS), m_Value(RHS))) - : match(Cur, m_Or(m_Value(LHS), m_Value(RHS)))) { - Worklist.append({LHS, RHS}); - continue; - } - - if (!match(Cur, m_SpecificFCmp(Inverted ? FCmpInst::FCMP_UNE - : FCmpInst::FCMP_OEQ, - m_Value(Src), m_AnyZeroFP()))) - return false; - - Worklist.push_back(Src); - continue; - } - // Peeling a nnan/ninf op could expose poison that the select hides. auto *FPOp = dyn_cast<FPMathOperator>(Cur); if (FPOp && (FPOp->hasNoNaNs() || FPOp->hasNoInfs())) { @@ -1630,6 +1636,7 @@ bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) { continue; } + Value *Src; // m_UnOp, not m_FNeg: fsub -0.0, X may flush a denormal result to zero. if (match(Cur, m_CombineOr(m_UnOp(m_Value(Src)), m_FAbs(m_Value(Src))))) { Worklist.push_back(Src); @@ -1645,7 +1652,7 @@ bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) { IRBuilder<> Builder(&I); Value *LegacyMul = - Builder.CreateIntrinsic(Intrinsic::amdgcn_fmul_legacy, {}, {X, Y}); + Builder.CreateIntrinsic(Intrinsic::amdgcn_fmul_legacy, {X, Y}); LegacyMul->takeName(&I); I.replaceAllUsesWith(LegacyMul); DeadVals.push_back(&I); diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll index c0a5d20503f7a1..9d98ad7645bcda 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll @@ -240,12 +240,7 @@ define float @select_fmul_legacy_inverted_logical_and(float %x, float %y) { define float @select_fmul_legacy_inverted_fabs(float %x, float %y) { ; CHECK-LABEL: define float @select_fmul_legacy_inverted_fabs( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) { -; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X]]) -; CHECK-NEXT: [[XNZ:%.*]] = fcmp une float [[FABS_X]], 0.000000e+00 -; CHECK-NEXT: [[YNZ:%.*]] = fcmp une float [[Y]], 0.000000e+00 -; CHECK-NEXT: [[C:%.*]] = and i1 [[XNZ]], [[YNZ]] -; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float [[MUL]], float 0.000000e+00 +; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]]) ; CHECK-NEXT: ret float [[R]] ; %fabs.x = call float @llvm.fabs.f32(float %x) _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
