https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/211363
Backport: 53da348 48c2598 The current checks in materializeConstantVectorTripCount miss tail-folded cases when vectorizing loops with fmax without fast-math flags. Use recently added hasTailFolded helper to ensure we always bail out on tail-folded loops. Fixes https://github.com/llvm/llvm-project/issues/209159. >From 31c0a4c3220bcef7ff4f8ed5fb324d499d0ef2e7 Mon Sep 17 00:00:00 2001 From: Florian Hahn <[email protected]> Date: Tue, 14 Jul 2026 15:16:33 +0100 Subject: [PATCH 1/2] [LV] Add test showing incorrect trip count materialization (NFC) (#209460) Add test case showing mis-compile for https://github.com/llvm/llvm-project/issues/209159. --- .../fmax-without-fast-math-flags.ll | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll index a4f7631435bb3..ef01bb29c1589 100644 --- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll +++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll @@ -758,3 +758,108 @@ exit: %sub = fsub float %max.next, %min.next ret float %sub } + +; Test fmax reduction with tail folding and a constant trip count that is not a +; multiple of the vector factor (optsize forces tail folding). +; Test for https://github.com/llvm/llvm-project/issues/209159. +; FIXME: Currently the trip count is computed incorrectly. +define float @fmaxnum_constant_trip_count_tailfold(ptr %src) #0 { +; CHECK-LABEL: define float @fmaxnum_constant_trip_count_tailfold( +; CHECK-SAME: ptr [[SRC:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[PRED_LOAD_CONTINUE6]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6]] ] +; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 29) +; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0 +; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]] +; CHECK: [[PRED_LOAD_IF]]: +; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[TMP2]], align 4 +; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x float> poison, float [[TMP3]], i64 0 +; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]] +; CHECK: [[PRED_LOAD_CONTINUE]]: +; CHECK-NEXT: [[TMP5:%.*]] = phi <4 x float> [ poison, %[[VECTOR_BODY]] ], [ [[TMP4]], %[[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP0]], i64 1 +; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]] +; CHECK: [[PRED_LOAD_IF1]]: +; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP7]] +; CHECK-NEXT: [[TMP9:%.*]] = load float, ptr [[TMP8]], align 4 +; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x float> [[TMP5]], float [[TMP9]], i64 1 +; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]] +; CHECK: [[PRED_LOAD_CONTINUE2]]: +; CHECK-NEXT: [[TMP11:%.*]] = phi <4 x float> [ [[TMP5]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP10]], %[[PRED_LOAD_IF1]] ] +; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP0]], i64 2 +; CHECK-NEXT: br i1 [[TMP12]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]] +; CHECK: [[PRED_LOAD_IF3]]: +; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP13]] +; CHECK-NEXT: [[TMP15:%.*]] = load float, ptr [[TMP14]], align 4 +; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x float> [[TMP11]], float [[TMP15]], i64 2 +; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]] +; CHECK: [[PRED_LOAD_CONTINUE4]]: +; CHECK-NEXT: [[TMP17:%.*]] = phi <4 x float> [ [[TMP11]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP16]], %[[PRED_LOAD_IF3]] ] +; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP0]], i64 3 +; CHECK-NEXT: br i1 [[TMP18]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]] +; CHECK: [[PRED_LOAD_IF5]]: +; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[INDEX]], 3 +; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[TMP19]] +; CHECK-NEXT: [[TMP21:%.*]] = load float, ptr [[TMP20]], align 4 +; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x float> [[TMP17]], float [[TMP21]], i64 3 +; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]] +; CHECK: [[PRED_LOAD_CONTINUE6]]: +; CHECK-NEXT: [[TMP23:%.*]] = phi <4 x float> [ [[TMP17]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP22]], %[[PRED_LOAD_IF5]] ] +; CHECK-NEXT: [[TMP24]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[TMP23]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP25:%.*]] = fcmp uno <4 x float> [[TMP23]], [[TMP23]] +; CHECK-NEXT: [[TMP26:%.*]] = freeze <4 x i1> [[TMP25]] +; CHECK-NEXT: [[TMP27:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP26]]) +; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 28 +; CHECK-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]] +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4) +; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP30:%.*]] = select <4 x i1> [[TMP0]], <4 x float> [[TMP24]], <4 x float> [[VEC_PHI]] +; CHECK-NEXT: [[TMP31:%.*]] = select i1 [[TMP27]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP30]] +; CHECK-NEXT: [[TMP32:%.*]] = select i1 [[TMP27]], i64 [[INDEX]], i64 30 +; CHECK-NEXT: [[TMP33:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP31]]) +; CHECK-NEXT: [[TMP34:%.*]] = xor i1 [[TMP27]], true +; CHECK-NEXT: br i1 [[TMP34]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP32]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[MAX:%.*]] = phi float [ [[TMP33]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[IV]] +; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4 +; CHECK-NEXT: [[MAX_NEXT]] = tail call float @llvm.maxnum.f32(float [[MAX]], float [[L]]) +; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 +; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 30 +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP33]], %[[MIDDLE_BLOCK]] ] +; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]] +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %max = phi float [ 0.000000e+00, %entry ], [ %max.next, %loop ] + %gep = getelementptr inbounds float, ptr %src, i64 %iv + %l = load float, ptr %gep, align 4 + %max.next = tail call float @llvm.maxnum.f32(float %max, float %l) + %iv.next = add i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 30 + br i1 %ec, label %exit, label %loop + +exit: + ret float %max.next +} + +attributes #0 = { optsize } >From 478d4d31c57c850c2e2d77c4581005de56d3a3af Mon Sep 17 00:00:00 2001 From: Florian Hahn <[email protected]> Date: Tue, 14 Jul 2026 17:30:24 +0100 Subject: [PATCH 2/2] [VPlan] Use VPlan::hasTailFolded in materializeConstantVectorTC. (#209502) The current checks in materializeConstantVectorTripCount miss tail-folded cases when vectorizing loops with fmax without fast-math flags. Use recently added hasTailFolded helper to ensure we always bail out on tail-folded loops. Fixes https://github.com/llvm/llvm-project/issues/209159. --- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 2 +- .../Transforms/LoopVectorize/fmax-without-fast-math-flags.ll | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index f37d61f8464fa..b0e5a5c4449b8 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -5451,7 +5451,7 @@ void VPlanTransforms::materializeConstantVectorTripCount( // Skip cases for which the trip count may be non-trivial to materialize. // I.e., when a scalar tail is absent - due to tail folding, or when a scalar // tail is required. - if (!Plan.hasScalarTail() || + if (Plan.hasTailFolded() || !Plan.hasScalarTail() || Plan.getMiddleBlock()->getSingleSuccessor() == Plan.getScalarPreheader() || !isa<VPIRValue>(TC)) diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll index ef01bb29c1589..e04caf61609f4 100644 --- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll +++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll @@ -819,7 +819,7 @@ define float @fmaxnum_constant_trip_count_tailfold(ptr %src) #0 { ; CHECK-NEXT: [[TMP25:%.*]] = fcmp uno <4 x float> [[TMP23]], [[TMP23]] ; CHECK-NEXT: [[TMP26:%.*]] = freeze <4 x i1> [[TMP25]] ; CHECK-NEXT: [[TMP27:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP26]]) -; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 28 +; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32 ; CHECK-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4) ; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
