https://github.com/arsenm updated https://github.com/llvm/llvm-project/pull/172998
>From b5ced4709ff40b8fbbc1abbaa4daf7772208d301 Mon Sep 17 00:00:00 2001 From: Matt Arsenault <[email protected]> Date: Fri, 19 Dec 2025 11:08:58 +0100 Subject: [PATCH 1/2] InstCombine: Fold out nanless canonicalize pattern Pattern match a wrapper around llvm.canonicalize which weakens the semantics to not require quieting signaling nans. Depending on the denormal mode and FP type, we can either drop the pattern entirely or reduce it only to a canonicalize call. I'm inventing this pattern to deal with LLVM's lax canonicalization model in math library code. The math library code currently has explicit checks for the denormal mode, and conditionally canonicalizes the result if there is flushing. Semantically, this could be directly replaced with a simple call to llvm.canonicalize, but doing so would incur an additional cost when using standard IEEE behavior. If we do not care about quieting a signaling nan, this should be a no-op unless the denormal mode may flush. This will allow replacement of the conditional code with a zero cost abstraction utility function. Note we need a standard LLVM floating-point operation in the nan case to assert we do not care about preserving the nan payload and sign bit. This could be any no-op fp instruction; a normal choice would be fmul by 1.0. Using that presents an ordering problem - since LLVM fp operations are not required to canonicalize, instcombine would fold out the fmul before reaching this select combine. The galaxy brain solution here is to use fdiv 1.0, %x as the no-op. This is not a no-op - it could potentially return infinity if %x were 0 (or very close to 0) so it will not be dropped. For the purposes here, that's fine since it's only being used as a nan sink. https://alive2.llvm.org/ce/z/QYS4en --- .../InstCombine/InstCombineSelect.cpp | 65 +++++++++++ .../nanless-canonicalize-combine.ll | 101 ++++-------------- 2 files changed, 85 insertions(+), 81 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp index f52bac5e600cb..02ff40298dcbe 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp @@ -4343,6 +4343,71 @@ Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) { matchFMulByZeroIfResultEqZero(*this, Cmp0, Cmp1, MatchCmp1, MatchCmp0, SI, SIFPOp->hasNoSignedZeros())) return replaceInstUsesWith(SI, Cmp0); + + if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) { + // Fold out only-canonicalize-non-nans pattern. This implements a + // wrapper around llvm.canonicalize which is not required to quiet + // signaling nans or preserve nan payload bits. + // + // %hard.canonical = call @llvm.canonicalize(%x) + // %soft.canonical = fdiv 1.0, %x + // %ord = fcmp ord %x, 0.0 + // %x.canon = select i1 %ord, %hard.canonical, %soft.canonical + // + // With known IEEE handling: + // => %x + // + // With other denormal behaviors: + // => llvm.canonicalize(%x) + // + // Note the fdiv could be any value preserving, potentially + // canonicalizing floating-point operation such as fmul by 1.0. However, + // since in the llvm model canonicalization is not mandatory, the fmul + // would have been dropped by the time we reached here. The trick here + // is to use a reciprocal fdiv. It's not a droppable no-op, as it could + // return an infinity if %x were sufficiently small, but in this pattern + // we're only using the output for nan values. + + if (Pred == CmpInst::FCMP_ORD) { + MatchCmp0 = TrueVal; + MatchCmp1 = FalseVal; + } else { + MatchCmp0 = FalseVal; + MatchCmp1 = TrueVal; + } + + if (match(MatchCmp0, m_FCanonicalize(m_Specific(Cmp0))) && + match(Cmp1, m_PosZeroFP())) { + const fltSemantics &FPSem = + SelType->getScalarType()->getFltSemantics(); + if (APFloat::isIEEELikeFP(FPSem)) { + // IEEE handling does not have non-canonical values, so the + // canonicalize can be dropped for direct replacement without + // looking for the intermediate maybe-canonicalizing operation. + if (Cmp0 == MatchCmp1 && SI.getFunction()->getDenormalMode(FPSem) == + DenormalMode::getIEEE()) + return replaceInstUsesWith(SI, Cmp0); + + // If denormals may be flushed, we need to retain the canonicalize + // call. This introduces a canonicalization on the nan path, which + // we are not free to do as that could change the sign bit or + // payload bits. We can only do this if there were a no-op like + // floating-point instruction which may have changed the nan bits + // anyway. + if (match(MatchCmp1, m_FDiv(m_FPOne(), m_Specific(Cmp0)))) { + DenormalMode Mode = SI.getFunction()->getDenormalMode(FPSem); + if (Mode == DenormalMode::getIEEE()) + return replaceInstUsesWith(SI, Cmp0); + + if (Mode.inputsAreZero() || Mode.outputsAreZero()) + return replaceInstUsesWith(SI, MatchCmp0); + } + + // Leave the dynamic mode case alone. This would introduce new + // constraints if the mode may be refined later. + } + } + } } } diff --git a/llvm/test/Transforms/InstCombine/nanless-canonicalize-combine.ll b/llvm/test/Transforms/InstCombine/nanless-canonicalize-combine.ll index 56e0808c650ff..c26a914c7a1f3 100644 --- a/llvm/test/Transforms/InstCombine/nanless-canonicalize-combine.ll +++ b/llvm/test/Transforms/InstCombine/nanless-canonicalize-combine.ll @@ -9,11 +9,7 @@ define float @canonicalize_ieee_0(float %x) #0 { ; CHECK-LABEL: define float @canonicalize_ieee_0( ; CHECK-SAME: float [[X:%.*]]) #[[ATTR0:[0-9]+]] { -; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call float @llvm.canonicalize.f32(float [[X]]) -; CHECK-NEXT: [[SOFT_CANONICAL:%.*]] = fdiv float 1.000000e+00, [[X]] -; CHECK-NEXT: [[ORD:%.*]] = fcmp ord float [[X]], 0.000000e+00 -; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], float [[HARD_CANONICAL]], float [[SOFT_CANONICAL]] -; CHECK-NEXT: ret float [[X_CANON]] +; CHECK-NEXT: ret float [[X]] ; %hard.canonical = call float @llvm.canonicalize.f32(float %x) %soft.canonical = fdiv float 1.0, %x @@ -26,11 +22,7 @@ define float @canonicalize_ieee_0(float %x) #0 { define float @canonicalize_ieee_1(float %x) #0 { ; CHECK-LABEL: define float @canonicalize_ieee_1( ; CHECK-SAME: float [[X:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call float @llvm.canonicalize.f32(float [[X]]) -; CHECK-NEXT: [[SOFT_CANONICAL:%.*]] = fdiv float 1.000000e+00, [[X]] -; CHECK-NEXT: [[UNO:%.*]] = fcmp uno float [[X]], 0.000000e+00 -; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[UNO]], float [[SOFT_CANONICAL]], float [[HARD_CANONICAL]] -; CHECK-NEXT: ret float [[X_CANON]] +; CHECK-NEXT: ret float [[X]] ; %hard.canonical = call float @llvm.canonicalize.f32(float %x) %soft.canonical = fdiv float 1.0, %x @@ -44,10 +36,7 @@ define float @canonicalize_ieee_1(float %x) #0 { define float @canonicalize_ieee_0_fmul(float %x) #0 { ; CHECK-LABEL: define float @canonicalize_ieee_0_fmul( ; CHECK-SAME: float [[X:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call float @llvm.canonicalize.f32(float [[X]]) -; CHECK-NEXT: [[ORD:%.*]] = fcmp ord float [[X]], 0.000000e+00 -; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], float [[HARD_CANONICAL]], float [[X]] -; CHECK-NEXT: ret float [[X_CANON]] +; CHECK-NEXT: ret float [[X]] ; %hard.canonical = call float @llvm.canonicalize.f32(float %x) %soft.canonical = fmul float %x, 1.0 @@ -61,10 +50,7 @@ define float @canonicalize_ieee_0_fmul(float %x) #0 { define float @canonicalize_ieee_0_fdiv_commute(float %x) #0 { ; CHECK-LABEL: define float @canonicalize_ieee_0_fdiv_commute( ; CHECK-SAME: float [[X:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call float @llvm.canonicalize.f32(float [[X]]) -; CHECK-NEXT: [[ORD:%.*]] = fcmp ord float [[X]], 0.000000e+00 -; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], float [[HARD_CANONICAL]], float [[X]] -; CHECK-NEXT: ret float [[X_CANON]] +; CHECK-NEXT: ret float [[X]] ; %hard.canonical = call float @llvm.canonicalize.f32(float %x) %soft.canonical = fdiv float %x, 1.0 @@ -79,10 +65,7 @@ define float @canonicalize_daz_0(float %x) #1 { ; CHECK-LABEL: define float @canonicalize_daz_0( ; CHECK-SAME: float [[X:%.*]]) #[[ATTR1:[0-9]+]] { ; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call float @llvm.canonicalize.f32(float [[X]]) -; CHECK-NEXT: [[SOFT_CANONICAL:%.*]] = fdiv float 1.000000e+00, [[X]] -; CHECK-NEXT: [[ORD:%.*]] = fcmp ord float [[X]], 0.000000e+00 -; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], float [[HARD_CANONICAL]], float [[SOFT_CANONICAL]] -; CHECK-NEXT: ret float [[X_CANON]] +; CHECK-NEXT: ret float [[HARD_CANONICAL]] ; %hard.canonical = call float @llvm.canonicalize.f32(float %x) %soft.canonical = fdiv float 1.0, %x @@ -96,11 +79,8 @@ define float @canonicalize_daz_0(float %x) #1 { define float @canonicalize_daz_1(float %x) #1 { ; CHECK-LABEL: define float @canonicalize_daz_1( ; CHECK-SAME: float [[X:%.*]]) #[[ATTR1]] { -; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call float @llvm.canonicalize.f32(float [[X]]) -; CHECK-NEXT: [[SOFT_CANONICAL:%.*]] = fdiv float 1.000000e+00, [[X]] -; CHECK-NEXT: [[UNO:%.*]] = fcmp uno float [[X]], 0.000000e+00 -; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[UNO]], float [[SOFT_CANONICAL]], float [[HARD_CANONICAL]] -; CHECK-NEXT: ret float [[X_CANON]] +; CHECK-NEXT: [[SOFT_CANONICAL:%.*]] = call float @llvm.canonicalize.f32(float [[X]]) +; CHECK-NEXT: ret float [[SOFT_CANONICAL]] ; %hard.canonical = call float @llvm.canonicalize.f32(float %x) %soft.canonical = fdiv float 1.0, %x @@ -146,11 +126,7 @@ define float @canonicalize_dynamic_1(float %x) #2 { define <2 x float> @canonicalize_ieee_0_vec(<2 x float> %x) #0 { ; CHECK-LABEL: define <2 x float> @canonicalize_ieee_0_vec( ; CHECK-SAME: <2 x float> [[X:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call <2 x float> @llvm.canonicalize.v2f32(<2 x float> [[X]]) -; CHECK-NEXT: [[SOFT_CANONICAL:%.*]] = fdiv <2 x float> splat (float 1.000000e+00), [[X]] -; CHECK-NEXT: [[ORD:%.*]] = fcmp ord <2 x float> [[X]], zeroinitializer -; CHECK-NEXT: [[X_CANON:%.*]] = select <2 x i1> [[ORD]], <2 x float> [[HARD_CANONICAL]], <2 x float> [[SOFT_CANONICAL]] -; CHECK-NEXT: ret <2 x float> [[X_CANON]] +; CHECK-NEXT: ret <2 x float> [[X]] ; %hard.canonical = call <2 x float> @llvm.canonicalize.v2f32(<2 x float> %x) %soft.canonical = fdiv <2 x float> splat (float 1.0), %x @@ -162,11 +138,7 @@ define <2 x float> @canonicalize_ieee_0_vec(<2 x float> %x) #0 { define <2 x float> @canonicalize_ieee_1_vec(<2 x float> %x) #0 { ; CHECK-LABEL: define <2 x float> @canonicalize_ieee_1_vec( ; CHECK-SAME: <2 x float> [[X:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call <2 x float> @llvm.canonicalize.v2f32(<2 x float> [[X]]) -; CHECK-NEXT: [[SOFT_CANONICAL:%.*]] = fdiv <2 x float> splat (float 1.000000e+00), [[X]] -; CHECK-NEXT: [[UNO:%.*]] = fcmp uno <2 x float> [[X]], zeroinitializer -; CHECK-NEXT: [[X_CANON:%.*]] = select <2 x i1> [[UNO]], <2 x float> [[SOFT_CANONICAL]], <2 x float> [[HARD_CANONICAL]] -; CHECK-NEXT: ret <2 x float> [[X_CANON]] +; CHECK-NEXT: ret <2 x float> [[X]] ; %hard.canonical = call <2 x float> @llvm.canonicalize.v2f32(<2 x float> %x) %soft.canonical = fdiv <2 x float> splat (float 1.0), %x @@ -195,10 +167,7 @@ define <2 x float> @canonicalize_daz_0_vec(<2 x float> %x) #1 { ; CHECK-LABEL: define <2 x float> @canonicalize_daz_0_vec( ; CHECK-SAME: <2 x float> [[X:%.*]]) #[[ATTR1]] { ; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call <2 x float> @llvm.canonicalize.v2f32(<2 x float> [[X]]) -; CHECK-NEXT: [[SOFT_CANONICAL:%.*]] = fdiv <2 x float> splat (float 1.000000e+00), [[X]] -; CHECK-NEXT: [[ORD:%.*]] = fcmp ord <2 x float> [[X]], zeroinitializer -; CHECK-NEXT: [[X_CANON:%.*]] = select <2 x i1> [[ORD]], <2 x float> [[HARD_CANONICAL]], <2 x float> [[SOFT_CANONICAL]] -; CHECK-NEXT: ret <2 x float> [[X_CANON]] +; CHECK-NEXT: ret <2 x float> [[HARD_CANONICAL]] ; %hard.canonical = call <2 x float> @llvm.canonicalize.v2f32(<2 x float> %x) %soft.canonical = fdiv <2 x float> splat (float 1.0), %x @@ -210,11 +179,8 @@ define <2 x float> @canonicalize_daz_0_vec(<2 x float> %x) #1 { define <2 x float> @canonicalize_daz_1_vec(<2 x float> %x) #1 { ; CHECK-LABEL: define <2 x float> @canonicalize_daz_1_vec( ; CHECK-SAME: <2 x float> [[X:%.*]]) #[[ATTR1]] { -; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call <2 x float> @llvm.canonicalize.v2f32(<2 x float> [[X]]) -; CHECK-NEXT: [[SOFT_CANONICAL:%.*]] = fdiv <2 x float> splat (float 1.000000e+00), [[X]] -; CHECK-NEXT: [[UNO:%.*]] = fcmp uno <2 x float> [[X]], zeroinitializer -; CHECK-NEXT: [[X_CANON:%.*]] = select <2 x i1> [[UNO]], <2 x float> [[SOFT_CANONICAL]], <2 x float> [[HARD_CANONICAL]] -; CHECK-NEXT: ret <2 x float> [[X_CANON]] +; CHECK-NEXT: [[SOFT_CANONICAL:%.*]] = call <2 x float> @llvm.canonicalize.v2f32(<2 x float> [[X]]) +; CHECK-NEXT: ret <2 x float> [[SOFT_CANONICAL]] ; %hard.canonical = call <2 x float> @llvm.canonicalize.v2f32(<2 x float> %x) %soft.canonical = fdiv <2 x float> splat (float 1.0), %x @@ -226,11 +192,7 @@ define <2 x float> @canonicalize_daz_1_vec(<2 x float> %x) #1 { define bfloat @canonicalize_ieee_bf16(bfloat %x) #0 { ; CHECK-LABEL: define bfloat @canonicalize_ieee_bf16( ; CHECK-SAME: bfloat [[X:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call bfloat @llvm.canonicalize.bf16(bfloat [[X]]) -; CHECK-NEXT: [[SOFT_CANONICAL:%.*]] = fdiv bfloat 0xR3F80, [[X]] -; CHECK-NEXT: [[ORD:%.*]] = fcmp ord bfloat [[X]], 0xR0000 -; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], bfloat [[HARD_CANONICAL]], bfloat [[SOFT_CANONICAL]] -; CHECK-NEXT: ret bfloat [[X_CANON]] +; CHECK-NEXT: ret bfloat [[X]] ; %hard.canonical = call bfloat @llvm.canonicalize.bf16(bfloat %x) %soft.canonical = fdiv bfloat 1.0, %x @@ -242,11 +204,7 @@ define bfloat @canonicalize_ieee_bf16(bfloat %x) #0 { define half @canonicalize_ieee_f16(half %x) #0 { ; CHECK-LABEL: define half @canonicalize_ieee_f16( ; CHECK-SAME: half [[X:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call half @llvm.canonicalize.f16(half [[X]]) -; CHECK-NEXT: [[SOFT_CANONICAL:%.*]] = fdiv half 0xH3C00, [[X]] -; CHECK-NEXT: [[ORD:%.*]] = fcmp ord half [[X]], 0xH0000 -; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], half [[HARD_CANONICAL]], half [[SOFT_CANONICAL]] -; CHECK-NEXT: ret half [[X_CANON]] +; CHECK-NEXT: ret half [[X]] ; %hard.canonical = call half @llvm.canonicalize.f16(half %x) %soft.canonical = fdiv half 1.0, %x @@ -258,11 +216,7 @@ define half @canonicalize_ieee_f16(half %x) #0 { define double @canonicalize_ieee_f64(double %x) #0 { ; CHECK-LABEL: define double @canonicalize_ieee_f64( ; CHECK-SAME: double [[X:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call double @llvm.canonicalize.f64(double [[X]]) -; CHECK-NEXT: [[SOFT_CANONICAL:%.*]] = fdiv double 1.000000e+00, [[X]] -; CHECK-NEXT: [[ORD:%.*]] = fcmp ord double [[X]], 0.000000e+00 -; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], double [[HARD_CANONICAL]], double [[SOFT_CANONICAL]] -; CHECK-NEXT: ret double [[X_CANON]] +; CHECK-NEXT: ret double [[X]] ; %hard.canonical = call double @llvm.canonicalize.f64(double %x) %soft.canonical = fdiv double 1.0, %x @@ -274,10 +228,7 @@ define double @canonicalize_ieee_f64(double %x) #0 { define fp128 @canonicalize_ieee_f128(fp128 %x) #0 { ; CHECK-LABEL: define fp128 @canonicalize_ieee_f128( ; CHECK-SAME: fp128 [[X:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call fp128 @llvm.canonicalize.f128(fp128 [[X]]) -; CHECK-NEXT: [[ORD:%.*]] = fcmp ord fp128 [[X]], 0xL00000000000000000000000000000000 -; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], fp128 [[HARD_CANONICAL]], fp128 [[X]] -; CHECK-NEXT: ret fp128 [[X_CANON]] +; CHECK-NEXT: ret fp128 [[X]] ; %hard.canonical = call fp128 @llvm.canonicalize.f128(fp128 %x) %ord = fcmp ord fp128 %x, 0xL00000000000000000000000000000000 @@ -503,10 +454,7 @@ define ppc_fp128 @ignore_ppc_fp128(ppc_fp128 %x) #0 { define float @canonicalize_ieee_0_missing_noop(float %x) #0 { ; CHECK-LABEL: define float @canonicalize_ieee_0_missing_noop( ; CHECK-SAME: float [[X:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call float @llvm.canonicalize.f32(float [[X]]) -; CHECK-NEXT: [[ORD:%.*]] = fcmp ord float [[X]], 0.000000e+00 -; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], float [[HARD_CANONICAL]], float [[X]] -; CHECK-NEXT: ret float [[X_CANON]] +; CHECK-NEXT: ret float [[X]] ; %hard.canonical = call float @llvm.canonicalize.f32(float %x) %ord = fcmp ord float %x, 0.0 @@ -519,10 +467,7 @@ define float @canonicalize_ieee_0_missing_noop(float %x) #0 { define float @canonicalize_ieee_1_missing_noop(float %x) #0 { ; CHECK-LABEL: define float @canonicalize_ieee_1_missing_noop( ; CHECK-SAME: float [[X:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call float @llvm.canonicalize.f32(float [[X]]) -; CHECK-NEXT: [[UNO:%.*]] = fcmp uno float [[X]], 0.000000e+00 -; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[UNO]], float [[X]], float [[HARD_CANONICAL]] -; CHECK-NEXT: ret float [[X_CANON]] +; CHECK-NEXT: ret float [[X]] ; %hard.canonical = call float @llvm.canonicalize.f32(float %x) %uno = fcmp uno float %x, 0.0 @@ -567,10 +512,7 @@ define float @canonicalize_only_ftz(float %x) "denormal-fp-math"="preserve-sign, ; CHECK-LABEL: define float @canonicalize_only_ftz( ; CHECK-SAME: float [[X:%.*]]) #[[ATTR3:[0-9]+]] { ; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call float @llvm.canonicalize.f32(float [[X]]) -; CHECK-NEXT: [[SOFT_CANONICAL:%.*]] = fdiv float 1.000000e+00, [[X]] -; CHECK-NEXT: [[ORD:%.*]] = fcmp ord float [[X]], 0.000000e+00 -; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], float [[HARD_CANONICAL]], float [[SOFT_CANONICAL]] -; CHECK-NEXT: ret float [[X_CANON]] +; CHECK-NEXT: ret float [[HARD_CANONICAL]] ; %hard.canonical = call float @llvm.canonicalize.f32(float %x) %soft.canonical = fdiv float 1.0, %x @@ -584,10 +526,7 @@ define float @canonicalize_only_daz(float %x) "denormal-fp-math"="ieee,preserve- ; CHECK-LABEL: define float @canonicalize_only_daz( ; CHECK-SAME: float [[X:%.*]]) #[[ATTR4:[0-9]+]] { ; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call float @llvm.canonicalize.f32(float [[X]]) -; CHECK-NEXT: [[SOFT_CANONICAL:%.*]] = fdiv float 1.000000e+00, [[X]] -; CHECK-NEXT: [[ORD:%.*]] = fcmp ord float [[X]], 0.000000e+00 -; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], float [[HARD_CANONICAL]], float [[SOFT_CANONICAL]] -; CHECK-NEXT: ret float [[X_CANON]] +; CHECK-NEXT: ret float [[HARD_CANONICAL]] ; %hard.canonical = call float @llvm.canonicalize.f32(float %x) %soft.canonical = fdiv float 1.0, %x >From 6e74f3b37587bb5ea3b347326903a84f1b99a061 Mon Sep 17 00:00:00 2001 From: Matt Arsenault <[email protected]> Date: Fri, 19 Dec 2025 23:28:18 +0100 Subject: [PATCH 2/2] Partially handle as independent folds --- .../InstCombine/InstCombineSelect.cpp | 60 +++++++++++++++---- .../nanless-canonicalize-combine.ll | 53 +++++++--------- 2 files changed, 70 insertions(+), 43 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp index 02ff40298dcbe..aad0c0f73d8f6 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp @@ -4344,7 +4344,11 @@ Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) { SI, SIFPOp->hasNoSignedZeros())) return replaceInstUsesWith(SI, Cmp0); - if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) { + Type *EltTy = SelType->getScalarType(); + + // TODO: Generalize to any ordered / unordered compare. + if ((Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) && + match(Cmp1, m_PosZeroFP()) && EltTy->isIEEELikeFPTy()) { // Fold out only-canonicalize-non-nans pattern. This implements a // wrapper around llvm.canonicalize which is not required to quiet // signaling nans or preserve nan payload bits. @@ -4376,17 +4380,52 @@ Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) { MatchCmp1 = TrueVal; } - if (match(MatchCmp0, m_FCanonicalize(m_Specific(Cmp0))) && - match(Cmp1, m_PosZeroFP())) { - const fltSemantics &FPSem = - SelType->getScalarType()->getFltSemantics(); - if (APFloat::isIEEELikeFP(FPSem)) { + bool RcpIfNan = match(MatchCmp1, m_FDiv(m_FPOne(), m_Specific(Cmp0))); + bool CanonicalizeIfNotNan = + match(MatchCmp0, m_FCanonicalize(m_Specific(Cmp0))); + + if (RcpIfNan || CanonicalizeIfNotNan) { + const fltSemantics &FPSem = EltTy->getFltSemantics(); + DenormalMode Mode = F.getDenormalMode(FPSem); + + if (RcpIfNan) { + if (Mode == DenormalMode::getIEEE()) { + // Special case for the other select operand. Otherwise, we may + // need to insert freeze on Cmp0 in the compare and select. + if (CanonicalizeIfNotNan) + return replaceInstUsesWith(SI, Cmp0); + + if (isGuaranteedNotToBeUndef(Cmp0, &AC, &SI, &DT)) { + // select (fcmp ord %cmp0, 0), y, (fdiv 1, x) + // => select (fcmp ord %cmp0, 0), y, x + // + // select (fcmp uno %cmp0, 0), (fdiv 1, x), y + // => select (fcmp uno %cmp0, 0), x, y + replaceOperand(SI, Pred == CmpInst::FCMP_ORD ? 2 : 1, Cmp0); + return &SI; + } + + auto *FrCmp0 = InsertNewInstBefore( + new FreezeInst(Cmp0, Cmp0->getName() + ".fr"), + FCmp->getIterator()); + + replaceOperand(*FCmp, 0, FrCmp0); + return replaceOperand(SI, Pred == CmpInst::FCMP_ORD ? 2 : 1, + FrCmp0); + } + } + + if (CanonicalizeIfNotNan) { // IEEE handling does not have non-canonical values, so the // canonicalize can be dropped for direct replacement without // looking for the intermediate maybe-canonicalizing operation. - if (Cmp0 == MatchCmp1 && SI.getFunction()->getDenormalMode(FPSem) == - DenormalMode::getIEEE()) - return replaceInstUsesWith(SI, Cmp0); + if (Mode == DenormalMode::getIEEE()) { + // select (fcmp ord %cmp0, 0), canonicalize(x), y + // => select (fcmp ord %cmp0, 0), x, y + + replaceOperand(SI, Pred == CmpInst::FCMP_ORD ? 1 : 2, Cmp0); + return &SI; + } // If denormals may be flushed, we need to retain the canonicalize // call. This introduces a canonicalization on the nan path, which @@ -4394,8 +4433,7 @@ Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) { // payload bits. We can only do this if there were a no-op like // floating-point instruction which may have changed the nan bits // anyway. - if (match(MatchCmp1, m_FDiv(m_FPOne(), m_Specific(Cmp0)))) { - DenormalMode Mode = SI.getFunction()->getDenormalMode(FPSem); + if (RcpIfNan) { if (Mode == DenormalMode::getIEEE()) return replaceInstUsesWith(SI, Cmp0); diff --git a/llvm/test/Transforms/InstCombine/nanless-canonicalize-combine.ll b/llvm/test/Transforms/InstCombine/nanless-canonicalize-combine.ll index c26a914c7a1f3..af142cc2b2365 100644 --- a/llvm/test/Transforms/InstCombine/nanless-canonicalize-combine.ll +++ b/llvm/test/Transforms/InstCombine/nanless-canonicalize-combine.ll @@ -150,10 +150,9 @@ define <2 x float> @canonicalize_ieee_1_vec(<2 x float> %x) #0 { define <2 x float> @canonicalize_ieee_0_vec_poison_elt(<2 x float> %x) #0 { ; CHECK-LABEL: define <2 x float> @canonicalize_ieee_0_vec_poison_elt( ; CHECK-SAME: <2 x float> [[X:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call <2 x float> @llvm.canonicalize.v2f32(<2 x float> [[X]]) ; CHECK-NEXT: [[SOFT_CANONICAL:%.*]] = fdiv <2 x float> <float 1.000000e+00, float poison>, [[X]] ; CHECK-NEXT: [[ORD:%.*]] = fcmp ord <2 x float> [[X]], zeroinitializer -; CHECK-NEXT: [[X_CANON:%.*]] = select <2 x i1> [[ORD]], <2 x float> [[HARD_CANONICAL]], <2 x float> [[SOFT_CANONICAL]] +; CHECK-NEXT: [[X_CANON:%.*]] = select <2 x i1> [[ORD]], <2 x float> [[X]], <2 x float> [[SOFT_CANONICAL]] ; CHECK-NEXT: ret <2 x float> [[X_CANON]] ; %hard.canonical = call <2 x float> @llvm.canonicalize.v2f32(<2 x float> %x) @@ -240,10 +239,9 @@ define fp128 @canonicalize_ieee_f128(fp128 %x) #0 { define float @div_not_one(float %x) #0 { ; CHECK-LABEL: define float @div_not_one( ; CHECK-SAME: float [[X:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call float @llvm.canonicalize.f32(float [[X]]) ; CHECK-NEXT: [[SOFT_CANONICAL:%.*]] = fdiv float 2.000000e+00, [[X]] ; CHECK-NEXT: [[ORD:%.*]] = fcmp ord float [[X]], 0.000000e+00 -; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], float [[HARD_CANONICAL]], float [[SOFT_CANONICAL]] +; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], float [[X]], float [[SOFT_CANONICAL]] ; CHECK-NEXT: ret float [[X_CANON]] ; %hard.canonical = call float @llvm.canonicalize.f32(float %x) @@ -257,10 +255,9 @@ define float @div_not_one(float %x) #0 { define float @not_fdiv(float %x) #0 { ; CHECK-LABEL: define float @not_fdiv( ; CHECK-SAME: float [[X:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call float @llvm.canonicalize.f32(float [[X]]) ; CHECK-NEXT: [[SOFT_CANONICAL:%.*]] = fadd float [[X]], 1.000000e+00 ; CHECK-NEXT: [[ORD:%.*]] = fcmp ord float [[X]], 0.000000e+00 -; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], float [[HARD_CANONICAL]], float [[SOFT_CANONICAL]] +; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], float [[X]], float [[SOFT_CANONICAL]] ; CHECK-NEXT: ret float [[X_CANON]] ; %hard.canonical = call float @llvm.canonicalize.f32(float %x) @@ -293,10 +290,10 @@ declare float @func(float) define float @not_canonicalize(float %x) #0 { ; CHECK-LABEL: define float @not_canonicalize( ; CHECK-SAME: float [[X:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[NOT_CANONICAL:%.*]] = call noundef float @func(float [[X]]) -; CHECK-NEXT: [[SOFT_CANONICAL:%.*]] = fdiv float 1.000000e+00, [[X]] -; CHECK-NEXT: [[ORD:%.*]] = fcmp ord float [[X]], 0.000000e+00 -; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], float [[NOT_CANONICAL]], float [[SOFT_CANONICAL]] +; CHECK-NEXT: [[X_FR:%.*]] = freeze float [[X]] +; CHECK-NEXT: [[NOT_CANONICAL:%.*]] = call noundef float @func(float [[X_FR]]) +; CHECK-NEXT: [[ORD:%.*]] = fcmp ord float [[X_FR]], 0.000000e+00 +; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], float [[NOT_CANONICAL]], float [[X_FR]] ; CHECK-NEXT: ret float [[X_CANON]] ; %not.canonical = call noundef float @func(float %x) @@ -327,10 +324,9 @@ define float @compared_value_different(float %x, float %y) #0 { define float @fdiv_value_different_ieee(float %x, float %y) #0 { ; CHECK-LABEL: define float @fdiv_value_different_ieee( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call float @llvm.canonicalize.f32(float [[X]]) ; CHECK-NEXT: [[SOFT_CANONICAL:%.*]] = fdiv float 1.000000e+00, [[Y]] ; CHECK-NEXT: [[ORD:%.*]] = fcmp ord float [[X]], 0.000000e+00 -; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], float [[HARD_CANONICAL]], float [[SOFT_CANONICAL]] +; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], float [[X]], float [[SOFT_CANONICAL]] ; CHECK-NEXT: ret float [[X_CANON]] ; %hard.canonical = call float @llvm.canonicalize.f32(float %x) @@ -344,10 +340,9 @@ define float @fdiv_value_different_ieee(float %x, float %y) #0 { define float @fdiv_value_different_ieee_commute(float %x, float %y) #0 { ; CHECK-LABEL: define float @fdiv_value_different_ieee_commute( ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call float @llvm.canonicalize.f32(float [[X]]) ; CHECK-NEXT: [[SOFT_CANONICAL:%.*]] = fdiv float 1.000000e+00, [[Y]] ; CHECK-NEXT: [[ORD:%.*]] = fcmp uno float [[X]], 0.000000e+00 -; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], float [[SOFT_CANONICAL]], float [[HARD_CANONICAL]] +; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], float [[SOFT_CANONICAL]], float [[X]] ; CHECK-NEXT: ret float [[X_CANON]] ; %hard.canonical = call float @llvm.canonicalize.f32(float %x) @@ -568,9 +563,8 @@ define float @canonicalize_missing_noop_only_daz(float %x) "denormal-fp-math"="i define half @independent_hands_canonicalize_ieee_0(half noundef %x, half %y) #0 { ; CHECK-LABEL: define half @independent_hands_canonicalize_ieee_0( ; CHECK-SAME: half noundef [[X:%.*]], half [[Y:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call half @llvm.canonicalize.f16(half [[X]]) ; CHECK-NEXT: [[ORD:%.*]] = fcmp ord half [[X]], 0xH0000 -; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], half [[HARD_CANONICAL]], half [[Y]] +; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], half [[X]], half [[Y]] ; CHECK-NEXT: ret half [[X_CANON]] ; %hard.canonical = call half @llvm.canonicalize.f16(half %x) @@ -582,9 +576,8 @@ define half @independent_hands_canonicalize_ieee_0(half noundef %x, half %y) #0 define half @independent_hands_canonicalize_ieee_1(half noundef %x, half %y) #0 { ; CHECK-LABEL: define half @independent_hands_canonicalize_ieee_1( ; CHECK-SAME: half noundef [[X:%.*]], half [[Y:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call half @llvm.canonicalize.f16(half [[X]]) ; CHECK-NEXT: [[UNO:%.*]] = fcmp uno half [[X]], 0xH0000 -; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[UNO]], half [[Y]], half [[HARD_CANONICAL]] +; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[UNO]], half [[Y]], half [[X]] ; CHECK-NEXT: ret half [[X_CANON]] ; %hard.canonical = call half @llvm.canonicalize.f16(half %x) @@ -596,9 +589,8 @@ define half @independent_hands_canonicalize_ieee_1(half noundef %x, half %y) #0 define half @independent_hands_fdiv_ieee_0(half noundef %x, half %y) #0 { ; CHECK-LABEL: define half @independent_hands_fdiv_ieee_0( ; CHECK-SAME: half noundef [[X:%.*]], half [[Y:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[RCP:%.*]] = fdiv half 0xH3C00, [[X]] ; CHECK-NEXT: [[ORD:%.*]] = fcmp ord half [[X]], 0xH0000 -; CHECK-NEXT: [[SEL:%.*]] = select i1 [[ORD]], half [[Y]], half [[RCP]] +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[ORD]], half [[Y]], half [[X]] ; CHECK-NEXT: ret half [[SEL]] ; %rcp = fdiv half 1.0, %x @@ -610,9 +602,8 @@ define half @independent_hands_fdiv_ieee_0(half noundef %x, half %y) #0 { define half @independent_hands_fdiv_ieee_1(half noundef %x, half %y) #0 { ; CHECK-LABEL: define half @independent_hands_fdiv_ieee_1( ; CHECK-SAME: half noundef [[X:%.*]], half [[Y:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[RCP:%.*]] = fdiv half 0xH3C00, [[X]] ; CHECK-NEXT: [[UNO:%.*]] = fcmp uno half [[X]], 0xH0000 -; CHECK-NEXT: [[SEL:%.*]] = select i1 [[UNO]], half [[RCP]], half [[Y]] +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[UNO]], half [[X]], half [[Y]] ; CHECK-NEXT: ret half [[SEL]] ; %rcp = fdiv half 1.0, %x @@ -710,9 +701,8 @@ define x86_fp80 @independent_hands_fdiv_ieee_0_x86_fp80(x86_fp80 noundef %x, x86 define half @independent_hands_canonicalize_ieee_0_maybe_undef(half %x, half %y) #0 { ; CHECK-LABEL: define half @independent_hands_canonicalize_ieee_0_maybe_undef( ; CHECK-SAME: half [[X:%.*]], half [[Y:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call half @llvm.canonicalize.f16(half [[X]]) ; CHECK-NEXT: [[ORD:%.*]] = fcmp ord half [[X]], 0xH0000 -; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], half [[HARD_CANONICAL]], half [[Y]] +; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[ORD]], half [[X]], half [[Y]] ; CHECK-NEXT: ret half [[X_CANON]] ; %hard.canonical = call half @llvm.canonicalize.f16(half %x) @@ -725,9 +715,8 @@ define half @independent_hands_canonicalize_ieee_0_maybe_undef(half %x, half %y) define half @independent_hands_canonicalize_ieee_1_maybe_undef(half %x, half %y) #0 { ; CHECK-LABEL: define half @independent_hands_canonicalize_ieee_1_maybe_undef( ; CHECK-SAME: half [[X:%.*]], half [[Y:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[HARD_CANONICAL:%.*]] = call half @llvm.canonicalize.f16(half [[X]]) ; CHECK-NEXT: [[UNO:%.*]] = fcmp uno half [[X]], 0xH0000 -; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[UNO]], half [[Y]], half [[HARD_CANONICAL]] +; CHECK-NEXT: [[X_CANON:%.*]] = select i1 [[UNO]], half [[Y]], half [[X]] ; CHECK-NEXT: ret half [[X_CANON]] ; %hard.canonical = call half @llvm.canonicalize.f16(half %x) @@ -740,9 +729,9 @@ define half @independent_hands_canonicalize_ieee_1_maybe_undef(half %x, half %y) define half @independent_hands_fdiv_ieee_0_maybe_undef(half %x, half %y) #0 { ; CHECK-LABEL: define half @independent_hands_fdiv_ieee_0_maybe_undef( ; CHECK-SAME: half [[X:%.*]], half [[Y:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[RCP:%.*]] = fdiv half 0xH3C00, [[X]] -; CHECK-NEXT: [[ORD:%.*]] = fcmp ord half [[X]], 0xH0000 -; CHECK-NEXT: [[SEL:%.*]] = select i1 [[ORD]], half [[Y]], half [[RCP]] +; CHECK-NEXT: [[X_FR:%.*]] = freeze half [[X]] +; CHECK-NEXT: [[ORD:%.*]] = fcmp ord half [[X_FR]], 0xH0000 +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[ORD]], half [[Y]], half [[X_FR]] ; CHECK-NEXT: ret half [[SEL]] ; %rcp = fdiv half 1.0, %x @@ -755,9 +744,9 @@ define half @independent_hands_fdiv_ieee_0_maybe_undef(half %x, half %y) #0 { define half @independent_hands_fdiv_ieee_1_maybe_undef(half %x, half %y) #0 { ; CHECK-LABEL: define half @independent_hands_fdiv_ieee_1_maybe_undef( ; CHECK-SAME: half [[X:%.*]], half [[Y:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[RCP:%.*]] = fdiv half 0xH3C00, [[X]] -; CHECK-NEXT: [[UNO:%.*]] = fcmp uno half [[X]], 0xH0000 -; CHECK-NEXT: [[SEL:%.*]] = select i1 [[UNO]], half [[RCP]], half [[Y]] +; CHECK-NEXT: [[X_FR:%.*]] = freeze half [[X]] +; CHECK-NEXT: [[UNO:%.*]] = fcmp uno half [[X_FR]], 0xH0000 +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[UNO]], half [[X_FR]], half [[Y]] ; CHECK-NEXT: ret half [[SEL]] ; %rcp = fdiv half 1.0, %x _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
