Author: Florian Hahn Date: 2026-11-08T15:42:18+01:00 New Revision: b2307d24960ef211eb4392f23565fb03c532e9ad
URL: https://github.com/llvm/llvm-project/commit/b2307d24960ef211eb4392f23565fb03c532e9ad DIFF: https://github.com/llvm/llvm-project/commit/b2307d24960ef211eb4392f23565fb03c532e9ad.diff LOG: [VPlan] Bail out on FindLast reductions with extra phi users. (#230676) handleFindLastReductions uses the mask of the single non-phi incoming value of the find-last blend as the condition for updating the reduction. If the data value itself is the reduction phi on some paths, e.g. via a nested blend for an inner if, the mask is also true for lanes that keep the old value, and extract-last-active may then pick a stale lane. Conservatively bail out if the reduction phi has users other than the find-last select/blend and the header mask select. (cherry picked from commit 79caa03343df4da7b8ae0338d932125ef4bcf20b) Added: Modified: llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp llvm/test/Transforms/LoopVectorize/find-last.ll Removed: ################################################################################ diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp index 1a135b85495146..4e68c58048204b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp @@ -1864,6 +1864,14 @@ bool VPlanTransforms::handleFindLastReductions(VPlan &Plan) { !MatchBlend(SelectR)) return false; + // Bail out if PhiR has users other than the find-last select/blend and the + // header mask select, e.g. nested blends. Then the data operand may be PhiR + // on some paths and Cond does not determine the lanes to update. + if (any_of(PhiR->users(), [&](VPUser *U) { + return U != SelectR && U != BackedgeSelect->getDefiningRecipe(); + })) + return false; + assert(Cond != HeaderMask && "Cond must not be HeaderMask"); // Find final reduction computation and replace it with an diff --git a/llvm/test/Transforms/LoopVectorize/find-last.ll b/llvm/test/Transforms/LoopVectorize/find-last.ll index 4116cb5abd09df..78070f6cedd403 100644 --- a/llvm/test/Transforms/LoopVectorize/find-last.ll +++ b/llvm/test/Transforms/LoopVectorize/find-last.ll @@ -359,43 +359,10 @@ exit: } ; The data value %p is itself a phi that may yield %rdx. -; FIXME: Currently miscompiled, as %c.outer is used as condition to update -; the reduction. define i32 @find_last_nested_phi_in_latch(ptr noalias %a, ptr noalias %b, i64 %n) { ; CHECK-LABEL: define i32 @find_last_nested_phi_in_latch( ; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) { -; CHECK-NEXT: [[ENTRY:.*]]: -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 -; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] -; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[TMP0:%.*]] = urem i64 [[N]], 4 -; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] -; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] -; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], zeroinitializer -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP4]], <4 x i1> [[TMP3]], <4 x i32> poison) -; CHECK-NEXT: [[TMP5:%.*]] = icmp slt <4 x i32> [[WIDE_MASKED_LOAD]], zeroinitializer -; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> [[WIDE_MASKED_LOAD]], <4 x i32> [[VEC_PHI]] -; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP3]] -; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]]) -; CHECK-NEXT: [[TMP8]] = select i1 [[TMP7]], <4 x i1> [[TMP3]], <4 x i1> [[TMP1]] -; CHECK-NEXT: [[TMP10]] = select i1 [[TMP7]], <4 x i32> [[PREDPHI]], <4 x i32> [[VEC_PHI]] -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] -; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP10]], <4 x i1> [[TMP8]], i32 -1) -; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] -; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] -; CHECK: [[SCALAR_PH]]: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] -; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP12]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ] +; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] ; CHECK: [[LOOP_HEADER]]: ; @@ -434,42 +401,10 @@ exit: } ; Same as above, but the outer merge is not in the latch. -; FIXME: Currently miscompiled. define i32 @find_last_nested_phi(ptr noalias %a, ptr noalias %b, i64 %n) { ; CHECK-LABEL: define i32 @find_last_nested_phi( ; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) { -; CHECK-NEXT: [[ENTRY:.*]]: -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 -; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] -; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[TMP0:%.*]] = urem i64 [[N]], 4 -; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] -; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] -; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], zeroinitializer -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP4]], <4 x i1> [[TMP3]], <4 x i32> poison) -; CHECK-NEXT: [[TMP5:%.*]] = icmp slt <4 x i32> [[WIDE_MASKED_LOAD]], zeroinitializer -; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> [[WIDE_MASKED_LOAD]], <4 x i32> [[VEC_PHI]] -; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP3]] -; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]]) -; CHECK-NEXT: [[TMP8]] = select i1 [[TMP7]], <4 x i1> [[TMP3]], <4 x i1> [[TMP1]] -; CHECK-NEXT: [[TMP10]] = select i1 [[TMP7]], <4 x i32> [[PREDPHI]], <4 x i32> [[VEC_PHI]] -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] -; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP10]], <4 x i1> [[TMP8]], i32 -1) -; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] -; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] -; CHECK: [[SCALAR_PH]]: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] -; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP12]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ] +; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] ; CHECK: [[LOOP_HEADER]]: ; _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
