https://github.com/lukel97 updated 
https://github.com/llvm/llvm-project/pull/237254

>From 7367914e2000252d9166892ef5442f193913cbb1 Mon Sep 17 00:00:00 2001
From: Luke Lau <[email protected]>
Date: Wed, 28 Oct 2026 22:33:08 +0800
Subject: [PATCH 1/2] Precommit tests

---
 .../LoopVectorize/early_exit_with_stores.ll   | 221 ++++++++++++++++++
 1 file changed, 221 insertions(+)

diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll 
b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
index ad1269aa3c852..139c9992cc38d 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
@@ -1555,3 +1555,224 @@ exit:
 }
 
 declare i64 @get_an_unknown_offset();
+
+define void @store_in_latch_after_exit(ptr align 2 dereferenceable(40) noalias 
%array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: define void @store_in_latch_after_exit(
+; CHECK-SAME: ptr noalias align 2 dereferenceable(40) [[ARRAY:%.*]], ptr 
readonly align 2 dereferenceable(40) [[PRED:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], 
i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 
500)
+; CHECK-NEXT:    [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x 
i1> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> 
@llvm.masked.load.v4i16.p0(ptr align 2 [[TMP4]], <4 x i1> [[TMP3]], <4 x i16> 
poison)
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat 
(i16 1)
+; CHECK-NEXT:    call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP5]], ptr 
align 2 [[TMP4]], <4 x i1> [[TMP3]])
+; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> 
[[TMP2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
+; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP9:%.*]] = call i64 
@llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[TMP10]], 20
+; CHECK-NEXT:    br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[TMP10]], %[[SCALAR_PH]] ], [ 
[[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr 
[[PRED]], i64 [[IV]]
+; CHECK-NEXT:    [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT:    [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT:    br i1 [[EE_COND]], label %[[EXIT]], label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr 
[[ARRAY]], i64 [[IV]]
+; CHECK-NEXT:    [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT:    store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT:    br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]], 
!llvm.loop [[LOOP21:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+  %ee.val = load i16, ptr %ee.addr
+  %ee.cond = icmp sgt i16 %ee.val, 500
+  br i1 %ee.cond, label %exit, label %loop.latch
+
+loop.latch:
+  %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+  %data = load i16, ptr %st.addr
+  %inc = add nsw i16 %data, 1
+  store i16 %inc, ptr %st.addr
+  %iv.next = add nuw nsw i64 %iv, 1
+  %counted.cond = icmp eq i64 %iv.next, 20
+  br i1 %counted.cond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @conditional_store_after_two_exits(ptr align 2 dereferenceable(40) 
noalias %array, ptr align 2 dereferenceable(40) readonly %pred, ptr align 2 
dereferenceable(40) readonly %pred2) {
+; CHECK-LABEL: define void @conditional_store_after_two_exits(
+; CHECK-SAME: ptr noalias align 2 dereferenceable(40) [[ARRAY:%.*]], ptr 
readonly align 2 dereferenceable(40) [[PRED:%.*]], ptr readonly align 2 
dereferenceable(40) [[PRED2:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], 
%[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr 
[[PRED]], i64 [[IV]]
+; CHECK-NEXT:    [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT:    [[EE2_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr 
[[PRED2]], i64 [[IV]]
+; CHECK-NEXT:    [[EE2_VAL:%.*]] = load i16, ptr [[EE2_ADDR]], align 2
+; CHECK-NEXT:    [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT:    br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[LOOP_2:.*]]
+; CHECK:       [[LOOP_2]]:
+; CHECK-NEXT:    [[EE2_COND:%.*]] = icmp eq i16 [[EE2_VAL]], 0
+; CHECK-NEXT:    br i1 [[EE2_COND]], label %[[EXIT_2:.*]], label %[[LOOP_3:.*]]
+; CHECK:       [[LOOP_3]]:
+; CHECK-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr 
[[ARRAY]], i64 [[IV]]
+; CHECK-NEXT:    [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT:    [[ST_COND:%.*]] = icmp slt i16 [[DATA]], 0
+; CHECK-NEXT:    br i1 [[ST_COND]], label %[[STORE:.*]], label %[[LOOP_LATCH]]
+; CHECK:       [[STORE]]:
+; CHECK-NEXT:    [[INC:%.*]] = add i16 [[DATA]], 1
+; CHECK-NEXT:    store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT:    br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[EXIT_2]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+  %ee.val = load i16, ptr %ee.addr
+  %ee2.addr = getelementptr inbounds nuw i16, ptr %pred2, i64 %iv
+  %ee2.val = load i16, ptr %ee2.addr
+  %ee.cond = icmp sgt i16 %ee.val, 500
+  br i1 %ee.cond, label %exit, label %loop.2
+
+loop.2:
+  %ee2.cond = icmp eq i16 %ee2.val, 0
+  br i1 %ee2.cond, label %exit.2, label %loop.3
+
+loop.3:
+  %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+  %data = load i16, ptr %st.addr
+  %st.cond = icmp slt i16 %data, 0
+  br i1 %st.cond, label %store, label %loop.latch
+
+store:
+  %inc = add i16 %data, 1
+  store i16 %inc, ptr %st.addr
+  br label %loop.latch
+
+loop.latch:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %counted.cond = icmp eq i64 %iv.next, 20
+  br i1 %counted.cond, label %exit, label %loop
+
+exit:
+  ret void
+
+exit.2:
+  ret void
+}
+
+define i16 @store_after_exit_live_out(ptr align 2 dereferenceable(40) noalias 
%array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: define i16 @store_after_exit_live_out(
+; CHECK-SAME: ptr noalias align 2 dereferenceable(40) [[ARRAY:%.*]], ptr 
readonly align 2 dereferenceable(40) [[PRED:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], 
i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 
500)
+; CHECK-NEXT:    [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x 
i1> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> 
@llvm.masked.load.v4i16.p0(ptr align 2 [[TMP4]], <4 x i1> [[TMP3]], <4 x i16> 
poison)
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat 
(i16 1)
+; CHECK-NEXT:    call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP5]], ptr 
align 2 [[TMP4]], <4 x i1> [[TMP3]])
+; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> 
[[TMP2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
+; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i16> [[TMP5]], i64 3
+; CHECK-NEXT:    [[TMP10:%.*]] = call i64 
@llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[TMP11]], %[[SCALAR_PH]] ], [ 
[[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr 
[[PRED]], i64 [[IV]]
+; CHECK-NEXT:    [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT:    [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT:    br i1 [[EE_COND]], label %[[EXIT]], label %[[LOOP_STORE:.*]]
+; CHECK:       [[LOOP_STORE]]:
+; CHECK-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr 
[[ARRAY]], i64 [[IV]]
+; CHECK-NEXT:    [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT:    store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT:    br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]], 
!llvm.loop [[LOOP23:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi i16 [ -1, %[[LOOP]] ], [ [[INC]], 
%[[LOOP_LATCH]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i16 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+  %ee.val = load i16, ptr %ee.addr
+  %ee.cond = icmp sgt i16 %ee.val, 500
+  br i1 %ee.cond, label %exit, label %loop.store
+
+loop.store:
+  %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+  %data = load i16, ptr %st.addr
+  %inc = add nsw i16 %data, 1
+  store i16 %inc, ptr %st.addr
+  br label %loop.latch
+
+loop.latch:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %counted.cond = icmp eq i64 %iv.next, 20
+  br i1 %counted.cond, label %exit, label %loop
+
+exit:
+  %res = phi i16 [ -1, %loop ], [ %inc, %loop.latch ]
+  ret i16 %res
+}

>From e7d606250955639ce349c0668d00ce3eb08410c7 Mon Sep 17 00:00:00 2001
From: Luke Lau <[email protected]>
Date: Fri, 30 Oct 2026 00:16:00 +0800
Subject: [PATCH 2/2] [VPlan] Allow early exit loops with side effects by
 masking in the vector loop.

When all blocks with side effects are dominated by every early exit, we can 
vectorize the loop if we mask off any lanes past the exit. We can do this with 
control flow:

```mermaid
flowchart TD
  header --> exiting
  exiting -->| exit mask | sideeffects --> latch --> latch.split
  exiting -->| !exit mask | latch.split
```

For every early exiting block we add a branch to the latch on 
`!mask.beforefirst(exitcond)`. This is implemented in 
UncountableExitStyle::ReadOnly which is now renamed to 
MaskedHandleExitInVectorLoop.

This is a complementary exit style to MaskedHandleExitInScalarLoop. If any of 
the side effects aren't dominated by the early exit it still falls back to the 
scalar loop style, which is why none of the existing early exit side effect 
tests changed. Masking in loop is profitable for RISC-V and I'm assuming it's 
also profitable for AArch64 too, but if not then we can add a TTI hook for a 
preferred exit style.
---
 .../Transforms/Vectorize/LoopVectorize.cpp    |   3 +-
 llvm/lib/Transforms/Vectorize/VPlan.h         |   9 +-
 .../Vectorize/VPlanConstruction.cpp           |   4 +-
 .../Transforms/Vectorize/VPlanTransforms.cpp  | 119 ++++++++++-----
 .../LoopVectorize/early_exit_with_stores.ll   | 135 ++++++++----------
 .../Vectorize/VPlanVerifierTest.cpp           |   5 +-
 6 files changed, 158 insertions(+), 117 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp 
b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index b0f596dacd840..8d4c219879d20 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6189,7 +6189,8 @@ 
LoopVectorizationPlanner::tryToBuildVPlan1(LoopVectorizationCostModel &PlanCM) {
       return nullptr;
 
     // TODO: Check target preference for style.
-    UncountableExitStyle EEStyle = UncountableExitStyle::ReadOnly;
+    UncountableExitStyle EEStyle =
+        UncountableExitStyle::MaskedHandleExitInVectorLoop;
     if (!RUN_VPLAN_PASS(VPlanTransforms::handleUncountableEarlyExits, *VPlan0,
                         ORE, OrigLoop, PSE, *DT, Legal->getAssumptionCache(),
                         EEStyle)) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h 
b/llvm/lib/Transforms/Vectorize/VPlan.h
index 596107debadf9..9f66cc53e85ac 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -83,10 +83,11 @@ using VPlanPtr = std::unique_ptr<VPlan>;
 /// Different methods of handling early exits.
 ///
 enum class UncountableExitStyle {
-  /// No side effects to worry about, so we can process any uncountable exits
-  /// in the loop and branch either to the middle block if the trip count was
-  /// reached, or an early exitblock to determine which exit was taken.
-  ReadOnly,
+  /// Any side effects come after all the uncountable exits and are masked off,
+  /// so we can process any uncountable exits in the loop and branch either to
+  /// the middle block if the trip count was reached, or an early exitblock to
+  /// determine which exit was taken.
+  MaskedHandleExitInVectorLoop,
   /// All memory operations other than the load(s) required to determine 
whether
   /// an uncountable exit occurre will be masked based on that condition. If an
   /// uncountable exit is taken, then all lanes before the exiting lane will
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp 
b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index f005d03abbe02..21878179d3105 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1260,7 +1260,9 @@ bool 
VPlanTransforms::areAllLoadsDereferenceable(VPBasicBlock *HeaderVPBB,
         continue;
       auto *VPI = dyn_cast<VPInstruction>(&R);
       if (!VPI || VPI->getOpcode() != Instruction::Load) {
-        assert(!R.mayReadFromMemory() && "unexpected recipe reading memory");
+        assert((!R.mayReadFromMemory() ||
+                (VPI && VPI->getOpcode() == Instruction::Store)) &&
+               "unexpected recipe reading memory");
         continue;
       }
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp 
b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index f39e10e94f886..c6ad73c8950aa 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3551,50 +3551,59 @@ static bool handleUncountableExitsWithSideEffects(
 
 // Returns true if the loop contains stores, the only writes legality accepts 
in
 // early-exit loops.
-static bool loopHasStores(VPBasicBlock *HeaderVPBB) {
-  return any_of(vp_rpo_plain_cfg_loop_body(HeaderVPBB), [](VPBasicBlock *VPBB) 
{
-    return any_of(*VPBB, [VPBB](VPRecipeBase &R) {
-      (void)VPBB;
-      if (match(&R, m_Intrinsic<Intrinsic::pseudoprobe>()))
-        return false;
-      if (!R.mayWriteToMemory()) {
-        assert((!R.mayHaveSideEffects() || &R == VPBB->getTerminator() ||
-                R.isSafeToSpeculativelyExecute()) &&
-               "legality only accepts recipes with side effects that are "
-               "safe to speculate");
-        return false;
-      }
-      assert(match(&R, m_VPInstruction<Instruction::Store>()) &&
-             "legality only accepts stores as writes in early-exit loops");
-      return true;
-    });
-  });
+static SmallVector<VPBasicBlock *> blocksWithStores(VPBasicBlock *HeaderVPBB) {
+  return to_vector(make_filter_range(
+      vp_rpo_plain_cfg_loop_body(HeaderVPBB), [](VPBasicBlock *VPBB) {
+        return any_of(*VPBB, [VPBB](VPRecipeBase &R) {
+          (void)VPBB;
+          if (match(&R, m_Intrinsic<Intrinsic::pseudoprobe>()))
+            return false;
+          if (!R.mayWriteToMemory()) {
+            assert((!R.mayHaveSideEffects() || &R == VPBB->getTerminator() ||
+                    R.isSafeToSpeculativelyExecute()) &&
+                   "legality only accepts recipes with side effects that are "
+                   "safe to speculate");
+            return false;
+          }
+          assert(match(&R, m_VPInstruction<Instruction::Store>()) &&
+                 "legality only accepts stores as writes in early-exit loops");
+          return true;
+        });
+      }));
 }
 
 bool VPlanTransforms::handleUncountableEarlyExits(
     VPlan &Plan, OptimizationRemarkEmitter *ORE, Loop *TheLoop,
     PredicatedScalarEvolution &PSE, DominatorTree &DT, AssumptionCache *AC,
     UncountableExitStyle Style) {
-#ifndef NDEBUG
   VPDominatorTree VPDT(Plan);
-#endif
 
   auto *MiddleVPBB = VPBlockUtils::getPlainCFGMiddleBlock(Plan);
   auto [HeaderVPBB, LatchVPBB] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
 
-  if (loopHasStores(HeaderVPBB))
-    Style = UncountableExitStyle::MaskedHandleExitInScalarLoop;
+  // If all side effects are dominated by the early exits, we can mask in the
+  // vector loop. Otherwise we need to handle side effects in the scalar loop.
+  // TODO: Legality only allows side effects that the scalar loop style can
+  // handle, relax it for side effects after all the early exits.
+  SmallVector<VPBasicBlock *> SideEffectVPBBs = blocksWithStores(HeaderVPBB);
+  if (!SideEffectVPBBs.empty()) {
+    auto EarlyExits = vputils::getEarlyExits(Plan, MiddleVPBB);
+    for (auto [EarlyExitingVPBB, _] : EarlyExits)
+      for (VPBasicBlock *VPBB : SideEffectVPBBs)
+        if (!VPDT.properlyDominates(EarlyExitingVPBB, VPBB))
+          Style = UncountableExitStyle::MaskedHandleExitInScalarLoop;
+  }
 
-  // Dereferenceability is checked separately for uncountable exit loops with
-  // stores, as only the loads contributing to the exit condition need to
-  // be checked.
-  if (Style == UncountableExitStyle::ReadOnly &&
+  if (Style == UncountableExitStyle::MaskedHandleExitInVectorLoop &&
       !areAllLoadsDereferenceable(HeaderVPBB, TheLoop, PSE, DT, AC)) {
-    reportVectorizationFailure(
-        "Auto-vectorization of early exit loops with potentially "
-        "faulting loads is not supported",
-        "EarlyExitFaultingLoads", ORE, TheLoop);
-    return false;
+    if (SideEffectVPBBs.empty()) {
+      reportVectorizationFailure(
+          "Auto-vectorization of early exit loops with potentially "
+          "faulting loads is not supported",
+          "EarlyExitFaultingLoads", ORE, TheLoop);
+      return false;
+    }
+    Style = UncountableExitStyle::MaskedHandleExitInScalarLoop;
   }
 
   VPBuilder LatchBuilder(LatchVPBB->getTerminator());
@@ -3646,7 +3655,7 @@ bool VPlanTransforms::handleUncountableEarlyExits(
     });
   }
 
-  if (Style == UncountableExitStyle::ReadOnly) {
+  if (Style == UncountableExitStyle::MaskedHandleExitInVectorLoop) {
     // Lanes past the exit can be poison that the scalar loop never branched 
on,
     // so freeze them to prevent branching on poison.
     for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(HeaderVPBB)) {
@@ -3656,6 +3665,49 @@ bool VPlanTransforms::handleUncountableEarlyExits(
       Term->setOperand(0, VPBuilder(Term).createFreeze(Term->getOperand(0)));
     }
 
+    if (!SideEffectVPBBs.empty()) {
+      // Split the latch in case it has recipes with side effects.
+      LatchVPBB = 
LatchVPBB->splitAt(LatchVPBB->getTerminator()->getIterator());
+      LatchBuilder.setInsertPoint(LatchVPBB->getTerminator());
+
+      // Branch to the latch to skip any side effects past the exiting lane.
+      for (EarlyExitInfo &Exit : Exits) {
+        VPBasicBlock *EarlyExitingVPBB = Exit.EarlyExitingVPBB;
+        VPBuilder ExitingBuilder(EarlyExitingVPBB->getTerminator());
+        VPValue *ExitMask = ExitingBuilder.createWidenIntrinsic(
+            Intrinsic::mask_beforefirst,
+            ExitingBuilder.createFreeze(ExitingBuilder.createNaryOp(
+                VPInstruction::MaskedCond, Exit.CondToExit)),
+            Type::getInt1Ty(Plan.getContext()));
+        EarlyExitingVPBB->getTerminator()->setOperand(0, ExitMask);
+        VPBlockUtils::connectBlocks(EarlyExitingVPBB, LatchVPBB);
+      }
+
+      // The latch's exit condition may be poison on lanes past the exit, 
freeze
+      // it.
+      VPRecipeBase *LatchTerm = LatchVPBB->getTerminator();
+      LatchTerm->setOperand(
+          0, VPBuilder(LatchTerm).createFreeze(LatchTerm->getOperand(0)));
+
+      // If we took the exiting branch then any uses in the latch and beyond 
are
+      // poison.
+      VPDT.recalculate(Plan);
+      for (VPBasicBlock *VPBB : {LatchVPBB, MiddleVPBB}) {
+        for (VPRecipeBase &R :
+             make_range(VPBB->getFirstNonPhi(), VPBB->end())) {
+          for (auto [I, Op] : enumerate(R.operands())) {
+            VPRecipeBase *DefR = Op->getDefiningRecipe();
+            if (!DefR || VPDT.dominates(DefR->getParent(), VPBB))
+              continue;
+            DenseMap<VPBasicBlock *, VPValue *> Defs = {
+                {HeaderVPBB, Plan.getPoison(Op->getScalarType())}};
+            Defs[DefR->getParent()] = Op;
+            R.setOperand(I, vputils::reconstructSSA(LatchVPBB, Defs));
+          }
+        }
+      }
+    }
+
     // An exit's condition is false on any path that didn't go through its
     // exiting block.
     for (EarlyExitInfo &Exit : Exits) {
@@ -3813,7 +3865,8 @@ bool VPlanTransforms::handleUncountableEarlyExits(
       ExitIRI->addIncoming(NewIncoming);
     }
 
-    EarlyExitingVPBB->getTerminator()->eraseFromParent();
+    if (EarlyExitingVPBB->getNumSuccessors() == 2)
+      EarlyExitingVPBB->getTerminator()->eraseFromParent();
     VPBlockUtils::disconnectBlocks(EarlyExitingVPBB, EarlyExitVPBB);
     VPBlockUtils::connectBlocks(VectorEarlyExitVPBB, EarlyExitVPBB);
   }
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll 
b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
index 139c9992cc38d..d4f5ef95e5da7 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
@@ -1564,7 +1564,7 @@ define void @store_in_latch_after_exit(ptr align 2 
dereferenceable(40) noalias %
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], 
i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 
500)
@@ -1577,29 +1577,13 @@ define void @store_in_latch_after_exit(ptr align 2 
dereferenceable(40) noalias %
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> 
[[TMP2]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
-; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
-; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label 
%[[VECTOR_BODY_INTERIM]]
+; CHECK:       [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP9:%.*]] = call i64 
@llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
-; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], [[TMP9]]
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[TMP10]], 20
-; CHECK-NEXT:    br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
-; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    br label %[[LOOP:.*]]
-; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[TMP10]], %[[SCALAR_PH]] ], [ 
[[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr 
[[PRED]], i64 [[IV]]
-; CHECK-NEXT:    [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
-; CHECK-NEXT:    [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT:    br i1 [[EE_COND]], label %[[EXIT]], label %[[LOOP_LATCH]]
-; CHECK:       [[LOOP_LATCH]]:
-; CHECK-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr 
[[ARRAY]], i64 [[IV]]
-; CHECK-NEXT:    [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
-; CHECK-NEXT:    [[INC:%.*]] = add nsw i16 [[DATA]], 1
-; CHECK-NEXT:    store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT:    br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]], 
!llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT:    br label %[[EXIT]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -1629,32 +1613,49 @@ exit:
 define void @conditional_store_after_two_exits(ptr align 2 dereferenceable(40) 
noalias %array, ptr align 2 dereferenceable(40) readonly %pred, ptr align 2 
dereferenceable(40) readonly %pred2) {
 ; CHECK-LABEL: define void @conditional_store_after_two_exits(
 ; CHECK-SAME: ptr noalias align 2 dereferenceable(40) [[ARRAY:%.*]], ptr 
readonly align 2 dereferenceable(40) [[PRED:%.*]], ptr readonly align 2 
dereferenceable(40) [[PRED2:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    br label %[[LOOP:.*]]
-; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], 
%[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr 
[[PRED]], i64 [[IV]]
-; CHECK-NEXT:    [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
-; CHECK-NEXT:    [[EE2_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr 
[[PRED2]], i64 [[IV]]
-; CHECK-NEXT:    [[EE2_VAL:%.*]] = load i16, ptr [[EE2_ADDR]], align 2
-; CHECK-NEXT:    [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT:    br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[LOOP_2:.*]]
-; CHECK:       [[LOOP_2]]:
-; CHECK-NEXT:    [[EE2_COND:%.*]] = icmp eq i16 [[EE2_VAL]], 0
-; CHECK-NEXT:    br i1 [[EE2_COND]], label %[[EXIT_2:.*]], label %[[LOOP_3:.*]]
-; CHECK:       [[LOOP_3]]:
-; CHECK-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr 
[[ARRAY]], i64 [[IV]]
-; CHECK-NEXT:    [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
-; CHECK-NEXT:    [[ST_COND:%.*]] = icmp slt i16 [[DATA]], 0
-; CHECK-NEXT:    br i1 [[ST_COND]], label %[[STORE:.*]], label %[[LOOP_LATCH]]
-; CHECK:       [[STORE]]:
-; CHECK-NEXT:    [[INC:%.*]] = add i16 [[DATA]], 1
-; CHECK-NEXT:    store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT:    br label %[[LOOP_LATCH]]
-; CHECK:       [[LOOP_LATCH]]:
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT:    br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], 
i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED2]], 
i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 
500)
+; CHECK-NEXT:    [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x 
i1> [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i16> [[WIDE_LOAD1]], 
zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP5]], 
<4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = freeze <4 x i1> [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x 
i1> [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP8]], 
<4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 
[[INDEX]]
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> 
@llvm.masked.load.v4i16.p0(ptr align 2 [[TMP10]], <4 x i1> [[TMP9]], <4 x i16> 
poison)
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp slt <4 x i16> [[WIDE_MASKED_LOAD]], 
zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = freeze <4 x i1> [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <4 x i1> [[TMP9]], <4 x i1> [[TMP12]], 
<4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = add <4 x i16> [[WIDE_MASKED_LOAD]], splat 
(i16 1)
+; CHECK-NEXT:    call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP14]], 
ptr align 2 [[TMP10]], <4 x i1> [[TMP13]])
+; CHECK-NEXT:    [[TMP15:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> splat (i1 
true), <4 x i1> [[TMP6]]
+; CHECK-NEXT:    [[TMP16:%.*]] = freeze <4 x i1> [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> 
[[TMP16]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label 
%[[VECTOR_BODY_INTERIM]]
+; CHECK:       [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 
@llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP16]], i1 false)
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <4 x i1> [[TMP2]], i64 
[[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br i1 [[TMP19]], label %[[VECTOR_EARLY_EXIT_0:.*]], label 
%[[VECTOR_EARLY_EXIT_1:.*]]
+; CHECK:       [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT:    br label %[[EXIT_2:.*]]
+; CHECK:       [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT:    br label %[[EXIT]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ; CHECK:       [[EXIT_2]]:
@@ -1707,7 +1708,7 @@ define i16 @store_after_exit_live_out(ptr align 2 
dereferenceable(40) noalias %a
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], 
i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 
500)
@@ -1720,34 +1721,16 @@ define i16 @store_after_exit_live_out(ptr align 2 
dereferenceable(40) noalias %a
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> 
[[TMP2]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
-; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
-; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label 
%[[VECTOR_BODY_INTERIM]]
+; CHECK:       [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i16> [[TMP5]], i64 3
-; CHECK-NEXT:    [[TMP10:%.*]] = call i64 
@llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
-; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20
-; CHECK-NEXT:    br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
-; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    br label %[[LOOP:.*]]
-; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[TMP11]], %[[SCALAR_PH]] ], [ 
[[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr 
[[PRED]], i64 [[IV]]
-; CHECK-NEXT:    [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
-; CHECK-NEXT:    [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT:    br i1 [[EE_COND]], label %[[EXIT]], label %[[LOOP_STORE:.*]]
-; CHECK:       [[LOOP_STORE]]:
-; CHECK-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr 
[[ARRAY]], i64 [[IV]]
-; CHECK-NEXT:    [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
-; CHECK-NEXT:    [[INC:%.*]] = add nsw i16 [[DATA]], 1
-; CHECK-NEXT:    store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT:    br label %[[LOOP_LATCH]]
-; CHECK:       [[LOOP_LATCH]]:
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT:    br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]], 
!llvm.loop [[LOOP23:![0-9]+]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i16> [[TMP5]], i64 3
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT:    br label %[[EXIT]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[RES:%.*]] = phi i16 [ -1, %[[LOOP]] ], [ [[INC]], 
%[[LOOP_LATCH]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[RES:%.*]] = phi i16 [ -1, %[[VECTOR_EARLY_EXIT]] ], [ 
[[TMP8]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i16 [[RES]]
 ;
 entry:
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp 
b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
index 6eed6c3053819..1f4aeaeed0af0 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
@@ -596,8 +596,9 @@ TEST_F(VPIRVerifierTest, 
BranchOnTwoCondsLatchHeaderVerification) {
   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
   // Build a plain CFG VPlan with BranchOnTwoConds as the latch terminator
   // (3 successors), without wrapping blocks in loop regions.
-  auto Plan = buildVPlan(LoopHeader, UncountableExitStyle::ReadOnly,
-                         /*CreateLoopRegions=*/false);
+  auto Plan =
+      buildVPlan(LoopHeader, 
UncountableExitStyle::MaskedHandleExitInVectorLoop,
+                 /*CreateLoopRegions=*/false);
 
   auto *MiddleVPBB =
       cast<VPBasicBlock>(Plan->getScalarPreheader()->getPredecessors()[0]);

_______________________________________________
llvm-branch-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits

Reply via email to