llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-llvm-transforms Author: Luke Lau (lukel97) <details> <summary>Changes</summary> A poison value can be replaced with anything, so we don't need to create an incoming value for it when converting phis to blends. --- <sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub> --- Full diff: https://github.com/llvm/llvm-project/pull/237121.diff 3 Files Affected: - (modified) llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp (+2) - (modified) llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll (+83) - (modified) llvm/test/Transforms/LoopVectorize/predicator.ll (+71-1) ``````````diff diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp index c5d3c611a8773..e54920789dda6 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp @@ -310,6 +310,8 @@ VPPredicator::computeBlendEdges(VPPhi *Phi) { } } + Edges.remove_if([](auto X) { return match(X.second, m_Poison()); }); + return Edges; } diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll index 483ab8ad94b50..46887ec305304 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll @@ -751,6 +751,89 @@ exit: ret void } +define void @blend_poison_incoming(ptr noalias %a, ptr noalias %dst) { +; CHECK-LABEL: VPlan for loop in 'blend_poison_incoming' +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> +; CHECK-NEXT: EMIT ir<%gep.a> = getelementptr ir<%a>, ir<%iv> +; CHECK-NEXT: EMIT-SCALAR ir<%x> = load ir<%gep.a> +; CHECK-NEXT: EMIT ir<%c0> = icmp sgt ir<%x>, ir<0> +; CHECK-NEXT: Successor(s): bb2 +; CHECK-EMPTY: +; CHECK-NEXT: bb2: +; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c0> +; CHECK-NEXT: EMIT ir<%mul> = mul ir<%x>, ir<3>, vp<[[VP4]]> (!vplan.execution.frequency 6917529027641081855 (37.5%, estimated)) +; CHECK-NEXT: Successor(s): bb1 +; CHECK-EMPTY: +; CHECK-NEXT: bb1: +; CHECK-NEXT: EMIT ir<%c1> = icmp slt ir<%x>, ir<100>, ir<%c0> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated)) +; CHECK-NEXT: EMIT ir<%add> = add ir<%x>, ir<1>, ir<%c0> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated)) +; CHECK-NEXT: Successor(s): bb3 +; CHECK-EMPTY: +; CHECK-NEXT: bb3: +; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%c0>, ir<%c1> +; CHECK-NEXT: Successor(s): bb4 +; CHECK-EMPTY: +; CHECK-NEXT: bb4: +; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = not ir<%c1> +; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%c0>, vp<[[VP6]]> +; CHECK-NEXT: BLEND ir<%phi> = ir<%mul>/vp<[[VP4]]> ir<%add>/vp<[[VP7]]> +; CHECK-NEXT: EMIT ir<%gep.dst> = getelementptr ir<%dst>, ir<%iv> +; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep.dst> +; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1> +; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; +entry: + br label %bb0 + +bb0: +; bb0 +; / \ +; bb1 bb2 +; | \ | +; bb3 | | +; \ | / +; bb4 +; Lanes that reach bb4 from bb3 can take any value, so the blend for %phi +; doesn't need the bb3->bb4 edge. + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ] + %gep.a = getelementptr i32, ptr %a, i64 %iv + %x = load i32, ptr %gep.a + %c0 = icmp sgt i32 %x, 0 + br i1 %c0, label %bb1, label %bb2 + +bb1: + %c1 = icmp slt i32 %x, 100 + %add = add i32 %x, 1 + br i1 %c1, label %bb3, label %bb4 + +bb3: + br label %bb4 + +bb2: + %mul = mul i32 %x, 3 + br label %bb4 + +bb4: + %phi = phi i32 [ %add, %bb1 ], [ poison, %bb3 ], [ %mul, %bb2 ] + %gep.dst = getelementptr i32, ptr %dst, i64 %iv + store i32 %phi, ptr %gep.dst + %iv.next = add nsw nuw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p) { ; CHECK-LABEL: VPlan for loop in 'simplifiable_blend' ; CHECK-NEXT: <x1> vector loop: { diff --git a/llvm/test/Transforms/LoopVectorize/predicator.ll b/llvm/test/Transforms/LoopVectorize/predicator.ll index 57414ae62c341..8a7b4fb2f7181 100644 --- a/llvm/test/Transforms/LoopVectorize/predicator.ll +++ b/llvm/test/Transforms/LoopVectorize/predicator.ll @@ -232,6 +232,76 @@ exit: ret void } +define void @blend_poison_incoming(ptr noalias %a, ptr noalias %dst) { +; CHECK-LABEL: define void @blend_poison_incoming( +; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[DST:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = mul <4 x i32> [[WIDE_LOAD]], splat (i32 3) +; CHECK-NEXT: [[TMP6:%.*]] = icmp sge <4 x i32> [[WIDE_LOAD]], splat (i32 100) +; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 1) +; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP6]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP7]], <4 x i32> [[TMP4]], <4 x i32> [[TMP2]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]] +; CHECK-NEXT: store <4 x i32> [[PREDPHI1]], ptr [[TMP8]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128 +; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %bb0 + +bb0: +; bb0 +; / \ +; bb1 bb2 +; | \ | +; bb3 | | +; \ | / +; bb4 +; Lanes that reach bb4 from bb3 can take any value, so the blend for %phi +; doesn't need the bb3->bb4 edge. + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ] + %gep.a = getelementptr i32, ptr %a, i64 %iv + %x = load i32, ptr %gep.a + %c0 = icmp sgt i32 %x, 0 + br i1 %c0, label %bb1, label %bb2 + +bb1: + %c1 = icmp slt i32 %x, 100 + %add = add i32 %x, 1 + br i1 %c1, label %bb3, label %bb4 + +bb3: + br label %bb4 + +bb2: + %mul = mul i32 %x, 3 + br label %bb4 + +bb4: + %phi = phi i32 [ %add, %bb1 ], [ poison, %bb3 ], [ %mul, %bb2 ] + %gep.dst = getelementptr i32, ptr %dst, i64 %iv + store i32 %phi, ptr %gep.dst + %iv.next = add nsw nuw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p) { ; CHECK-LABEL: define void @simplifiable_blend( ; CHECK-SAME: i1 [[C1:%.*]], i1 [[C2:%.*]], i1 [[C3:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]], ptr [[P:%.*]]) { @@ -250,7 +320,7 @@ define void @simplifiable_blend(i1 %c1, i1 %c2, i1 %c3, i32 %x, i32 %y, ptr %p) ; CHECK-NEXT: store <4 x i32> [[PREDPHI7]], ptr [[TMP3]], align 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 ; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128 -; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: ; CHECK-NEXT: br label %[[EXIT:.*]] ; CHECK: [[EXIT]]: `````````` </details> https://github.com/llvm/llvm-project/pull/237121 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
