https://github.com/lukel97 updated 
https://github.com/llvm/llvm-project/pull/238045

>From bbd3ea0716ff7171ec3765a1b1de844cc77ab7a0 Mon Sep 17 00:00:00 2001
From: Luke Lau <[email protected]>
Date: Tue, 3 Nov 2026 01:15:40 +0800
Subject: [PATCH 1/2] Precommit tests

---
 .../LoopVectorize/early-exit-tail-folding.ll  | 365 ++++++++++--------
 1 file changed, 213 insertions(+), 152 deletions(-)

diff --git a/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll 
b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
index cff27685627005..dfad0a260c0251 100644
--- a/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py 
UTC_ARGS: --check-globals none --version 6
-; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 
-force-target-supports-masked-memory-ops 
-enable-early-exit-vectorization-with-side-effects 
-tail-folding-policy=prefer-fold-tail | FileCheck %s
+; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 
-force-target-supports-masked-memory-ops 
-enable-early-exit-vectorization-with-side-effects 
-tail-folding-policy=prefer-fold-tail | FileCheck %s 
--check-prefixes=CHECK,PREFER
+; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 
-force-target-supports-masked-memory-ops 
-enable-early-exit-vectorization-with-side-effects 
-tail-folding-policy=must-fold-tail | FileCheck %s --check-prefixes=CHECK,MUST
 
 define i32 @single_early_exit(ptr align 4 dereferenceable(4096) %p) {
 ; CHECK-LABEL: define i32 @single_early_exit(
@@ -271,199 +272,259 @@ exit:
   ret i32 %res
 }
 
-; TODO: Cannot tail fold yet.
-define i32 @countable_early_exit(ptr align 4 dereferenceable(4096) %p, i32 %n) 
{
-; CHECK-LABEL: define i32 @countable_early_exit(
-; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], i32 [[N:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.umin.i32(i32 [[N]], i32 122)
-; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i32 [[TMP0]], 1
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP1]], 4
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label 
%[[VECTOR_PH:.*]]
-; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 3
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP2]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i32 4, i32 [[TMP2]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP1]], [[TMP4]]
-; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
-; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    br label %[[SCALAR_PH]]
-; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], 
%[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    br label %[[LOOP:.*]]
-; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ 
[[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[IV]], [[N]]
-; CHECK-NEXT:    br i1 [[C]], label %[[EXIT:.*]], label %[[LATCH]]
-; CHECK:       [[LATCH]]:
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 
[[IV]]
-; CHECK-NEXT:    [[LD:%.*]] = load i32, ptr [[GEP]], align 4
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop 
[[LOOP7:![0-9]+]]
-; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[LD]], %[[LATCH]] ]
-; CHECK-NEXT:    ret i32 [[RES]]
+define void @early_exit_with_store_after_exit(ptr align 4 
dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) {
+; PREFER-LABEL: define void @early_exit_with_store_after_exit(
+; PREFER-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr 
align 4 dereferenceable(4096) [[PRED:%.*]]) {
+; PREFER-NEXT:  [[ENTRY:.*:]]
+; PREFER-NEXT:    br label %[[VECTOR_PH:.*]]
+; PREFER:       [[VECTOR_PH]]:
+; PREFER-NEXT:    br label %[[VECTOR_BODY:.*]]
+; PREFER:       [[VECTOR_BODY]]:
+; PREFER-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; PREFER-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], 
i32 [[INDEX]]
+; PREFER-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; PREFER-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 
500)
+; PREFER-NEXT:    [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; PREFER-NEXT:    [[TMP3:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 
x i1> [[TMP2]])
+; PREFER-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
+; PREFER-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> 
@llvm.masked.load.v4i32.p0(ptr align 4 [[TMP4]], <4 x i1> [[TMP3]], <4 x i32> 
poison)
+; PREFER-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat 
(i32 1)
+; PREFER-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP5]], 
ptr align 4 [[TMP4]], <4 x i1> [[TMP3]])
+; PREFER-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> 
[[TMP2]])
+; PREFER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; PREFER-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
+; PREFER-NEXT:    br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label 
%[[VECTOR_BODY_INTERIM]]
+; PREFER:       [[VECTOR_BODY_INTERIM]]:
+; PREFER-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; PREFER:       [[MIDDLE_BLOCK]]:
+; PREFER-NEXT:    br label %[[SCALAR_PH:.*]]
+; PREFER:       [[VECTOR_EARLY_EXIT]]:
+; PREFER-NEXT:    br label %[[EXIT:.*]]
+; PREFER:       [[SCALAR_PH]]:
+; PREFER-NEXT:    br label %[[LOOP:.*]]
+; PREFER:       [[LOOP]]:
+; PREFER-NEXT:    [[IV:%.*]] = phi i32 [ 120, %[[SCALAR_PH]] ], [ 
[[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; PREFER-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr 
[[PRED]], i32 [[IV]]
+; PREFER-NEXT:    [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
+; PREFER-NEXT:    [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
+; PREFER-NEXT:    br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]]
+; PREFER:       [[LATCH]]:
+; PREFER-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr 
[[DST]], i32 [[IV]]
+; PREFER-NEXT:    [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
+; PREFER-NEXT:    [[INC:%.*]] = add nsw i32 [[DATA]], 1
+; PREFER-NEXT:    store i32 [[INC]], ptr [[ST_ADDR]], align 4
+; PREFER-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; PREFER-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; PREFER-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop 
[[LOOP7:![0-9]+]]
+; PREFER:       [[EXIT]]:
+; PREFER-NEXT:    ret void
+;
+; MUST-LABEL: define void @early_exit_with_store_after_exit(
+; MUST-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 
4 dereferenceable(4096) [[PRED:%.*]]) {
+; MUST-NEXT:  [[ENTRY:.*]]:
+; MUST-NEXT:    br label %[[LOOP:.*]]
+; MUST:       [[LOOP]]:
+; MUST-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], 
%[[LATCH:.*]] ]
+; MUST-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr 
[[PRED]], i32 [[IV]]
+; MUST-NEXT:    [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
+; MUST-NEXT:    [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
+; MUST-NEXT:    br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[LATCH]]
+; MUST:       [[LATCH]]:
+; MUST-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], 
i32 [[IV]]
+; MUST-NEXT:    [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
+; MUST-NEXT:    [[INC:%.*]] = add nsw i32 [[DATA]], 1
+; MUST-NEXT:    store i32 [[INC]], ptr [[ST_ADDR]], align 4
+; MUST-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; MUST-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; MUST-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]]
+; MUST:       [[EXIT]]:
+; MUST-NEXT:    ret void
 ;
 entry:
   br label %loop
 
 loop:
   %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
-  %c = icmp eq i32 %iv, %n
-  br i1 %c, label %exit, label %latch
+  %ee.addr = getelementptr inbounds nuw i32, ptr %pred, i32 %iv
+  %ee.val = load i32, ptr %ee.addr
+  %ee.cond = icmp sgt i32 %ee.val, 500
+  br i1 %ee.cond, label %exit, label %latch
 
 latch:
-  %gep = getelementptr inbounds i32, ptr %p, i32 %iv
-  %ld = load i32, ptr %gep, align 4
+  %st.addr = getelementptr inbounds nuw i32, ptr %dst, i32 %iv
+  %data = load i32, ptr %st.addr
+  %inc = add nsw i32 %data, 1
+  store i32 %inc, ptr %st.addr
   %iv.next = add nuw nsw i32 %iv, 1
   %ec = icmp eq i32 %iv.next, 123
   br i1 %ec, label %exit, label %loop
 
 exit:
-  %res = phi i32 [ 0, %loop ], [ %ld, %latch ]
-  ret i32 %res
+  ret void
 }
 
 ; TODO: Cannot tail fold yet.
-define void @early_exit_with_store(ptr align 4 dereferenceable(4096) noalias 
%dst, ptr align 4 dereferenceable(4096) %pred) {
-; CHECK-LABEL: define void @early_exit_with_store(
-; CHECK-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 
4 dereferenceable(4096) [[PRED:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
-; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
-; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], 
i32 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 
500)
-; CHECK-NEXT:    [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
-; CHECK-NEXT:    [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> 
@llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]])
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> 
@llvm.masked.load.v4i32.p0(ptr align 4 [[TMP5]], <4 x i1> 
[[UNCOUNTABLE_EXIT_MASK]], <4 x i32> poison)
-; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat 
(i32 1)
-; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP6]], ptr 
align 4 [[TMP5]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> 
[[TMP2]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
-; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP12:%.*]] = call i64 
@llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
-; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP12]] to i32
-; CHECK-NEXT:    [[TMP10:%.*]] = add i32 [[INDEX]], [[TMP4]]
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[TMP10]], 123
-; CHECK-NEXT:    br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
-; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    br label %[[LOOP:.*]]
-; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[TMP10]], %[[SCALAR_PH]] ], [ 
[[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr 
[[DST]], i32 [[IV]]
-; CHECK-NEXT:    [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
-; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[DATA]], 1
-; CHECK-NEXT:    store i32 [[INC]], ptr [[ST_ADDR]], align 4
-; CHECK-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr 
[[PRED]], i32 [[IV]]
-; CHECK-NEXT:    [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
-; CHECK-NEXT:    [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
-; CHECK-NEXT:    br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]]
-; CHECK:       [[LATCH]]:
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop 
[[LOOP9:![0-9]+]]
-; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    ret void
+define i32 @countable_early_exit(ptr align 4 dereferenceable(4096) %p, i32 %n) 
{
+; PREFER-LABEL: define i32 @countable_early_exit(
+; PREFER-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], i32 [[N:%.*]]) {
+; PREFER-NEXT:  [[ENTRY:.*]]:
+; PREFER-NEXT:    [[TMP0:%.*]] = call i32 @llvm.umin.i32(i32 [[N]], i32 122)
+; PREFER-NEXT:    [[TMP1:%.*]] = add nuw nsw i32 [[TMP0]], 1
+; PREFER-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP1]], 4
+; PREFER-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label 
%[[VECTOR_PH:.*]]
+; PREFER:       [[VECTOR_PH]]:
+; PREFER-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 3
+; PREFER-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP2]], 0
+; PREFER-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i32 4, i32 [[TMP2]]
+; PREFER-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP1]], [[TMP4]]
+; PREFER-NEXT:    br label %[[VECTOR_BODY:.*]]
+; PREFER:       [[VECTOR_BODY]]:
+; PREFER-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; PREFER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; PREFER-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; PREFER-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; PREFER:       [[MIDDLE_BLOCK]]:
+; PREFER-NEXT:    br label %[[SCALAR_PH]]
+; PREFER:       [[SCALAR_PH]]:
+; PREFER-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], 
%[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; PREFER-NEXT:    br label %[[LOOP:.*]]
+; PREFER:       [[LOOP]]:
+; PREFER-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], 
[ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; PREFER-NEXT:    [[C:%.*]] = icmp eq i32 [[IV]], [[N]]
+; PREFER-NEXT:    br i1 [[C]], label %[[EXIT:.*]], label %[[LATCH]]
+; PREFER:       [[LATCH]]:
+; PREFER-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 
[[IV]]
+; PREFER-NEXT:    [[LD:%.*]] = load i32, ptr [[GEP]], align 4
+; PREFER-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; PREFER-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; PREFER-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop 
[[LOOP9:![0-9]+]]
+; PREFER:       [[EXIT]]:
+; PREFER-NEXT:    [[RES:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[LD]], %[[LATCH]] 
]
+; PREFER-NEXT:    ret i32 [[RES]]
+;
+; MUST-LABEL: define i32 @countable_early_exit(
+; MUST-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], i32 [[N:%.*]]) {
+; MUST-NEXT:  [[ENTRY:.*]]:
+; MUST-NEXT:    br label %[[LOOP:.*]]
+; MUST:       [[LOOP]]:
+; MUST-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], 
%[[LATCH:.*]] ]
+; MUST-NEXT:    [[C:%.*]] = icmp eq i32 [[IV]], [[N]]
+; MUST-NEXT:    br i1 [[C]], label %[[EXIT:.*]], label %[[LATCH]]
+; MUST:       [[LATCH]]:
+; MUST-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]]
+; MUST-NEXT:    [[LD:%.*]] = load i32, ptr [[GEP]], align 4
+; MUST-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; MUST-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; MUST-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]]
+; MUST:       [[EXIT]]:
+; MUST-NEXT:    [[RES:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[LD]], %[[LATCH]] ]
+; MUST-NEXT:    ret i32 [[RES]]
 ;
 entry:
   br label %loop
 
 loop:
   %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
-  %st.addr = getelementptr inbounds nuw i32, ptr %dst, i32 %iv
-  %data = load i32, ptr %st.addr, align 4
-  %inc = add nsw i32 %data, 1
-  store i32 %inc, ptr %st.addr, align 4
-  %ee.addr = getelementptr inbounds nuw i32, ptr %pred, i32 %iv
-  %ee.val = load i32, ptr %ee.addr, align 4
-  %ee.cond = icmp sgt i32 %ee.val, 500
-  br i1 %ee.cond, label %exit, label %latch
+  %c = icmp eq i32 %iv, %n
+  br i1 %c, label %exit, label %latch
 
 latch:
+  %gep = getelementptr inbounds i32, ptr %p, i32 %iv
+  %ld = load i32, ptr %gep, align 4
   %iv.next = add nuw nsw i32 %iv, 1
   %ec = icmp eq i32 %iv.next, 123
   br i1 %ec, label %exit, label %loop
 
 exit:
-  ret void
+  %res = phi i32 [ 0, %loop ], [ %ld, %latch ]
+  ret i32 %res
 }
 
-define void @early_exit_with_store_after_exit(ptr align 4 
dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) {
-; CHECK-LABEL: define void @early_exit_with_store_after_exit(
-; CHECK-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 
4 dereferenceable(4096) [[PRED:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
-; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
-; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], 
i32 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 
500)
-; CHECK-NEXT:    [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x 
i1> [[TMP2]])
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> 
@llvm.masked.load.v4i32.p0(ptr align 4 [[TMP4]], <4 x i1> [[TMP3]], <4 x i32> 
poison)
-; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat 
(i32 1)
-; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP5]], ptr 
align 4 [[TMP4]], <4 x i1> [[TMP3]])
-; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> 
[[TMP2]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label 
%[[VECTOR_BODY_INTERIM]]
-; CHECK:       [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    br label %[[SCALAR_PH:.*]]
-; CHECK:       [[VECTOR_EARLY_EXIT]]:
-; CHECK-NEXT:    br label %[[EXIT:.*]]
-; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    br label %[[LOOP:.*]]
-; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 120, %[[SCALAR_PH]] ], [ 
[[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr 
[[PRED]], i32 [[IV]]
-; CHECK-NEXT:    [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
-; CHECK-NEXT:    [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
-; CHECK-NEXT:    br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]]
-; CHECK:       [[LATCH]]:
-; CHECK-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr 
[[DST]], i32 [[IV]]
-; CHECK-NEXT:    [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
-; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[DATA]], 1
-; CHECK-NEXT:    store i32 [[INC]], ptr [[ST_ADDR]], align 4
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop 
[[LOOP11:![0-9]+]]
-; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    ret void
+; TODO: Cannot tail fold yet.
+define void @early_exit_with_store(ptr align 4 dereferenceable(4096) noalias 
%dst, ptr align 4 dereferenceable(4096) %pred) {
+; PREFER-LABEL: define void @early_exit_with_store(
+; PREFER-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr 
align 4 dereferenceable(4096) [[PRED:%.*]]) {
+; PREFER-NEXT:  [[ENTRY:.*:]]
+; PREFER-NEXT:    br label %[[VECTOR_PH:.*]]
+; PREFER:       [[VECTOR_PH]]:
+; PREFER-NEXT:    br label %[[VECTOR_BODY:.*]]
+; PREFER:       [[VECTOR_BODY]]:
+; PREFER-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; PREFER-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], 
i32 [[INDEX]]
+; PREFER-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; PREFER-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 
500)
+; PREFER-NEXT:    [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; PREFER-NEXT:    [[TMP3:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 
x i1> [[TMP2]])
+; PREFER-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
+; PREFER-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> 
@llvm.masked.load.v4i32.p0(ptr align 4 [[TMP4]], <4 x i1> [[TMP3]], <4 x i32> 
poison)
+; PREFER-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat 
(i32 1)
+; PREFER-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP5]], 
ptr align 4 [[TMP4]], <4 x i1> [[TMP3]])
+; PREFER-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> 
[[TMP2]])
+; PREFER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; PREFER-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
+; PREFER-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
+; PREFER-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; PREFER:       [[MIDDLE_BLOCK]]:
+; PREFER-NEXT:    [[TMP9:%.*]] = call i64 
@llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; PREFER-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
+; PREFER-NEXT:    [[TMP11:%.*]] = add i32 [[INDEX]], [[TMP10]]
+; PREFER-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[TMP11]], 123
+; PREFER-NEXT:    br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
+; PREFER:       [[SCALAR_PH]]:
+; PREFER-NEXT:    br label %[[LOOP:.*]]
+; PREFER:       [[LOOP]]:
+; PREFER-NEXT:    [[IV:%.*]] = phi i32 [ [[TMP11]], %[[SCALAR_PH]] ], [ 
[[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; PREFER-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr 
[[DST]], i32 [[IV]]
+; PREFER-NEXT:    [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
+; PREFER-NEXT:    [[INC:%.*]] = add nsw i32 [[DATA]], 1
+; PREFER-NEXT:    store i32 [[INC]], ptr [[ST_ADDR]], align 4
+; PREFER-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr 
[[PRED]], i32 [[IV]]
+; PREFER-NEXT:    [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
+; PREFER-NEXT:    [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
+; PREFER-NEXT:    br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]]
+; PREFER:       [[LATCH]]:
+; PREFER-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; PREFER-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; PREFER-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop 
[[LOOP11:![0-9]+]]
+; PREFER:       [[EXIT]]:
+; PREFER-NEXT:    ret void
+;
+; MUST-LABEL: define void @early_exit_with_store(
+; MUST-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 
4 dereferenceable(4096) [[PRED:%.*]]) {
+; MUST-NEXT:  [[ENTRY:.*]]:
+; MUST-NEXT:    br label %[[LOOP:.*]]
+; MUST:       [[LOOP]]:
+; MUST-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], 
%[[LATCH:.*]] ]
+; MUST-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], 
i32 [[IV]]
+; MUST-NEXT:    [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
+; MUST-NEXT:    [[INC:%.*]] = add nsw i32 [[DATA]], 1
+; MUST-NEXT:    store i32 [[INC]], ptr [[ST_ADDR]], align 4
+; MUST-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr 
[[PRED]], i32 [[IV]]
+; MUST-NEXT:    [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
+; MUST-NEXT:    [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
+; MUST-NEXT:    br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[LATCH]]
+; MUST:       [[LATCH]]:
+; MUST-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; MUST-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
+; MUST-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]]
+; MUST:       [[EXIT]]:
+; MUST-NEXT:    ret void
 ;
 entry:
   br label %loop
 
 loop:
   %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+  %st.addr = getelementptr inbounds nuw i32, ptr %dst, i32 %iv
+  %data = load i32, ptr %st.addr, align 4
+  %inc = add nsw i32 %data, 1
+  store i32 %inc, ptr %st.addr, align 4
   %ee.addr = getelementptr inbounds nuw i32, ptr %pred, i32 %iv
-  %ee.val = load i32, ptr %ee.addr
+  %ee.val = load i32, ptr %ee.addr, align 4
   %ee.cond = icmp sgt i32 %ee.val, 500
   br i1 %ee.cond, label %exit, label %latch
 
 latch:
-  %st.addr = getelementptr inbounds nuw i32, ptr %dst, i32 %iv
-  %data = load i32, ptr %st.addr
-  %inc = add nsw i32 %data, 1
-  store i32 %inc, ptr %st.addr
   %iv.next = add nuw nsw i32 %iv, 1
   %ec = icmp eq i32 %iv.next, 123
   br i1 %ec, label %exit, label %loop

>From 96ca70dfb312cd8592fa4aecd06129a9440463af Mon Sep 17 00:00:00 2001
From: Luke Lau <[email protected]>
Date: Sat, 31 Oct 2026 02:56:48 +0800
Subject: [PATCH 2/2] [VPlan] Allow tail folding early exits with side effects

Continues the work in #179595.

This allows us to tail fold e.g. the following loop:

```c
extern char a[1024], b[1024], c[1024], y[1024];
void two_guards(void) {
  for (int i = 0; i < 1000; i++) {
    if (a[i] == 0)
      break;
    if (b[i] > 3) {
      if (c[i] < 9)
        y[i] = 1;
    }
  }
}
```

Into this on RISC-V:

```asm
.LBB0_1:                                # %vector.body
                                        # =>This Inner Loop Header: Depth=1
        vsetvli a6, a1, e8, m2, ta, ma
        add     a7, a2, a0
        vle8.v  v10, (a7)
        vmseq.vi        v12, v10, 0
        vmsbf.m v0, v12
        add     a7, a3, a0
        vle8.v  v10, (a7), v0.t
        vmsgtu.vi       v13, v10, 3
        vmand.mm        v0, v0, v13
        add     a7, a4, a0
        vle8.v  v10, (a7), v0.t
        vmsleu.vi       v13, v10, 8
        vmand.mm        v0, v0, v13
        vcpop.m a7, v12
        add     t0, a5, a0
        vse8.v  v8, (t0), v0.t
        bnez    a7, .LBB0_3
                                        #   in Loop: Header=BB0_1 Depth=1
        sub     a1, a1, a6
        add     a0, a0, a6
        bnez    a1, .LBB0_1
```

For early exit loops with side effects, we may be able to handle the exits by 
masking the vector body, or we may have to handle them in the scalar loop. If 
it's the latter we can't tail fold the plan just yet, we need to fixup a lot of 
assumptions around the middle block etc. For now just disable tail folding in 
the cost model. It's safe to change the tail folding here because 
foldTailByMasking hasn't been called yet and there hasn't been any costing done.
---
 .../Vectorize/LoopVectorizationLegality.cpp   |   7 --
 .../Transforms/Vectorize/LoopVectorize.cpp    |  24 ++++
 .../Transforms/Vectorize/VPlanTransforms.cpp  |   2 +-
 .../Transforms/Vectorize/VPlanTransforms.h    |   3 +-
 .../LoopVectorize/RISCV/early-exit.ll         |  53 +++------
 .../RISCV/early_exit_with_stores.ll           |  65 ++++-------
 .../LoopVectorize/early-exit-tail-folding.ll  | 110 +++++++-----------
 .../early_exit_store_legality.ll              |   4 +-
 8 files changed, 111 insertions(+), 157 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp 
b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 0cd4b7ec2ffb45..1271fe6de706a8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -2115,13 +2115,6 @@ bool LoopVectorizationLegality::canFoldTailByMasking() 
const {
     return false;
   }
 
-  // TODO: Handle early exits with side effects without a scalar tail.
-  if (hasUncountableExitWithSideEffects()) {
-    LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking. Uncountable exit "
-                         "with side effects requires a scalar epilogue\n");
-    return false;
-  }
-
   LLVM_DEBUG(dbgs() << "LV: checking if tail can be folded by masking.\n");
 
   // No accesses are known to be safe in inactive lanes of the folded tail.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp 
b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e38995f42ff5b4..440b632cce6b04 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -748,6 +748,20 @@ class LoopVectorizationCostModel {
                        : "due to non-interleaving reasons.\n"));
   }
 
+  /// Disable all tail folding styles. Returns false if an epilogue isn't
+  /// allowed. Must be called before the plan is tail folded.
+  bool tryToDisableTailFolding() {
+    assert(foldTailByMasking() && "Expected tail folding to be enabled!");
+    if (EpilogueLoweringStatus != CM_EpilogueNotNeededFoldTail)
+      return false;
+    LLVM_DEBUG(dbgs() << "LV: Disabling tail folding: vectorize with an "
+                         "epilogue instead.\n");
+    ChosenTailFoldingStyle = TailFoldingStyle::None;
+    EpilogueLoweringStatus = CM_EpilogueAllowed;
+    PartialAliasMaskingStatus = AliasMaskingStatus::Disabled;
+    return true;
+  }
+
   /// Returns true if all loop blocks should be masked to fold tail loop.
   bool foldTailByMasking() const {
     return getTailFoldingStyle() != TailFoldingStyle::None;
@@ -6282,6 +6296,16 @@ 
LoopVectorizationPlanner::tryToBuildVPlan1(LoopVectorizationCostModel &PlanCM) {
                         EEStyle)) {
       return nullptr;
     }
+
+    // Exiting in the scalar loop requires a scalar epilogue for now.
+    if (EEStyle == UncountableExitStyle::MaskedHandleExitInScalarLoop &&
+        PlanCM.foldTailByMasking() && !PlanCM.tryToDisableTailFolding()) {
+      reportVectorizationFailure(
+          "Cannot fold tail by masking when early exits are handled in the "
+          "scalar loop",
+          "EarlyExitInScalarLoopCannotFoldTail", ORE, OrigLoop);
+      return nullptr;
+    }
   } else {
     RUN_VPLAN_PASS(VPlanTransforms::handleCountableEarlyExits, *VPlan0);
   }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp 
b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 280346698099e1..d9ab64a024e7ff 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3573,7 +3573,7 @@ static SmallVector<VPBasicBlock *> 
blocksWithStores(VPBasicBlock *HeaderVPBB) {
 bool VPlanTransforms::handleUncountableEarlyExits(
     VPlan &Plan, OptimizationRemarkEmitter *ORE, Loop *TheLoop,
     PredicatedScalarEvolution &PSE, DominatorTree &DT, AssumptionCache *AC,
-    UncountableExitStyle Style) {
+    UncountableExitStyle &Style) {
   VPDominatorTree VPDT(Plan);
 
   auto *MiddleVPBB = VPBlockUtils::getPlainCFGMiddleBlock(Plan);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h 
b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 9e467171d57e74..35dafb85c073c9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -403,11 +403,12 @@ struct VPlanTransforms {
   /// appropriate branching logic in the latch that handles early exits and the
   /// latch exit condition. Multiple exits are handled with a dispatch block
   /// that determines which exit to take based on lane-by-lane semantics.
+  /// \p Style is the preferred style, and is updated to the style used.
   LLVM_ABI_FOR_TEST static bool
   handleUncountableEarlyExits(VPlan &Plan, OptimizationRemarkEmitter *ORE,
                               Loop *TheLoop, PredicatedScalarEvolution &PSE,
                               DominatorTree &DT, AssumptionCache *AC,
-                              UncountableExitStyle Style);
+                              UncountableExitStyle &Style);
 
   /// Disconnect countable early exits from the loop.
   LLVM_ABI_FOR_TEST static void handleCountableEarlyExits(VPlan &Plan);
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll 
b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll
index f8563ff31a417f..06afcc1d4e1c8b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll
@@ -405,54 +405,37 @@ exit:
 define void @early_exit_with_store_after_exit(ptr align 4 
dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) {
 ; CHECK-LABEL: define void @early_exit_with_store_after_exit(
 ; CHECK-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 
4 dereferenceable(4096) [[PRED:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 123, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label 
%[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 123, [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 123, [[N_MOD_VF]]
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ 
[[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i32 [ 123, %[[VECTOR_PH]] ], [ 
[[AVL_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 
@llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], 
i32 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], 
align 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> 
@llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> splat (i1 
true), i32 [[TMP0]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt <vscale x 4 x i32> [[WIDE_LOAD]], 
splat (i32 500)
-; CHECK-NEXT:    [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <vscale x 4 x i1> 
@llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> 
[[TMP3]], <vscale x 4 x i1> zeroinitializer, i32 [[TMP0]])
+; CHECK-NEXT:    [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP10]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 4 x i1> 
@llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> 
@llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[TMP5]], 
<vscale x 4 x i32> poison)
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> 
@llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[TMP5]], i32 
[[TMP0]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = add nsw <vscale x 4 x i32> 
[[WIDE_MASKED_LOAD]], splat (i32 1)
-; CHECK-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> 
[[TMP7]], ptr align 4 [[TMP6]], <vscale x 4 x i1> [[TMP5]])
-; CHECK-NEXT:    [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale 
x 4 x i1> [[TMP4]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> 
[[TMP7]], ptr align 4 [[TMP6]], <vscale x 4 x i1> [[TMP5]], i32 [[TMP0]])
+; CHECK-NEXT:    [[TMP12:%.*]] = freeze <vscale x 4 x i1> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call i1 @llvm.vp.reduce.or.nxv4i1(i1 false, 
<vscale x 4 x i1> [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP9]], false
+; CHECK-NEXT:    [[CURRENT_ITERATION_NEXT]] = add i32 [[TMP0]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP0]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
 ; CHECK-NEXT:    br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT:.*]], label 
%[[VECTOR_BODY_INTERIM]]
 ; CHECK:       [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 123, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[VECTOR_EARLY_EXIT]]:
 ; CHECK-NEXT:    br label %[[EXIT]]
-; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], 
%[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    br label %[[LOOP:.*]]
-; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ 
[[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr 
[[PRED]], i32 [[IV]]
-; CHECK-NEXT:    [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
-; CHECK-NEXT:    [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
-; CHECK-NEXT:    br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]]
-; CHECK:       [[LATCH]]:
-; CHECK-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr 
[[DST]], i32 [[IV]]
-; CHECK-NEXT:    [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
-; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[DATA]], 1
-; CHECK-NEXT:    store i32 [[INC]], ptr [[ST_ADDR]], align 4
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop 
[[LOOP9:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll 
b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
index 8ba6d5f81679e5..a566df5db77019 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
@@ -1267,57 +1267,40 @@ exit.uncountable:
 define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 
dereferenceable(40) %src) {
 ; CHECK-LABEL: define void @interleaved_stores_after_early_exit(
 ; CHECK-SAME: ptr noalias [[DST:%.*]], ptr align 2 dereferenceable(40) 
[[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label 
%[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[MIDDLE_BLOCK:.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ 
[[CURRENT_ITERATION_NEXT:%.*]], %[[MIDDLE_BLOCK:.*]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 20, %[[VECTOR_PH]] ], [ 
[[AVL_NEXT:%.*]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 
@llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP2]], 
align 2
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 8 x i16> 
@llvm.vp.load.nxv8i16.p0(ptr align 2 [[TMP2]], <vscale x 8 x i1> splat (i1 
true), i32 [[TMP0]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], 
splat (i16 500)
-; CHECK-NEXT:    [[TMP4:%.*]] = freeze <vscale x 8 x i1> [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 8 x i1> 
@llvm.vp.merge.nxv8i1(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x i1> 
[[TMP3]], <vscale x 8 x i1> zeroinitializer, i32 [[TMP0]])
+; CHECK-NEXT:    [[TMP4:%.*]] = freeze <vscale x 8 x i1> [[TMP5]]
 ; CHECK-NEXT:    [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> 
@llvm.mask.beforefirst.nxv8i1(<vscale x 8 x i1> [[TMP4]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP6]]
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 16 x i16> 
@llvm.vector.interleave2.nxv16i16(<vscale x 8 x i16> splat (i16 1), <vscale x 8 
x i16> splat (i16 2))
+; CHECK-NEXT:    [[INTERLEAVE_EVL:%.*]] = mul nuw nsw i32 [[TMP0]], 2
 ; CHECK-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 16 x i1> 
@llvm.vector.interleave2.nxv16i1(<vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], 
<vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT:    call void @llvm.masked.store.nxv16i16.p0(<vscale x 16 x i16> 
[[INTERLEAVED_VEC]], ptr align 2 [[TMP7]], <vscale x 16 x i1> 
[[INTERLEAVED_MASK]])
-; CHECK-NEXT:    [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale 
x 8 x i1> [[TMP4]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 16 x i16> 
@llvm.vector.interleave2.nxv16i16(<vscale x 8 x i16> splat (i16 1), <vscale x 8 
x i16> splat (i16 2))
+; CHECK-NEXT:    call void @llvm.vp.store.nxv16i16.p0(<vscale x 16 x i16> 
[[INTERLEAVED_VEC]], ptr align 2 [[TMP7]], <vscale x 16 x i1> 
[[INTERLEAVED_MASK]], i32 [[INTERLEAVE_EVL]])
+; CHECK-NEXT:    [[TMP10:%.*]] = freeze <vscale x 8 x i1> [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call i1 @llvm.vp.reduce.or.nxv8i1(i1 false, 
<vscale x 8 x i1> [[TMP10]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP9]], false
+; CHECK-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP11]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
 ; CHECK-NEXT:    br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT:.*]], label 
%[[MIDDLE_BLOCK]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK1:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK1:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK1]]:
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 20, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[VECTOR_EARLY_EXIT]]:
 ; CHECK-NEXT:    br label %[[EXIT]]
-; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], 
%[[MIDDLE_BLOCK1]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
-; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ 
[[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP_SRC]], align 2
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i16 [[L]], 500
-; CHECK-NEXT:    br i1 [[C]], label %[[EXIT]], label %[[LOOP_LATCH]]
-; CHECK:       [[LOOP_LATCH]]:
-; CHECK-NEXT:    [[OFF_0:%.*]] = shl i64 [[IV]], 1
-; CHECK-NEXT:    [[GEP_DST_0:%.*]] = getelementptr i16, ptr [[DST]], i64 
[[OFF_0]]
-; CHECK-NEXT:    store i16 1, ptr [[GEP_DST_0]], align 2
-; CHECK-NEXT:    [[OFF_1:%.*]] = or disjoint i64 [[OFF_0]], 1
-; CHECK-NEXT:    [[GEP_DST_1:%.*]] = getelementptr i16, ptr [[DST]], i64 
[[OFF_1]]
-; CHECK-NEXT:    store i16 2, ptr [[GEP_DST_1]], align 2
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], 
!llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -1374,7 +1357,7 @@ define void @interleaved_stores_before_early_exit(ptr 
noalias %dst, ptr align 2
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
-; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 
@llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP4]], i1 false)
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP5]]
@@ -1398,7 +1381,7 @@ define void @interleaved_stores_before_early_exit(ptr 
noalias %dst, ptr align 2
 ; CHECK:       [[LOOP_LATCH]]:
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], 
!llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], 
!llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -1458,7 +1441,7 @@ define void 
@reverse_interleaved_stores_before_early_exit(ptr noalias %dst, ptr
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = or i1 [[TMP12]], [[TMP13]]
-; CHECK-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 
@llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> [[TMP7]], i1 false)
 ; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], [[TMP8]]
@@ -1483,7 +1466,7 @@ define void 
@reverse_interleaved_stores_before_early_exit(ptr noalias %dst, ptr
 ; CHECK:       [[LOOP_LATCH]]:
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], 
!llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], 
!llvm.loop [[LOOP15:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll 
b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
index dfad0a260c0251..74cd834636c7c3 100644
--- a/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll
@@ -273,72 +273,42 @@ exit:
 }
 
 define void @early_exit_with_store_after_exit(ptr align 4 
dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) {
-; PREFER-LABEL: define void @early_exit_with_store_after_exit(
-; PREFER-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr 
align 4 dereferenceable(4096) [[PRED:%.*]]) {
-; PREFER-NEXT:  [[ENTRY:.*:]]
-; PREFER-NEXT:    br label %[[VECTOR_PH:.*]]
-; PREFER:       [[VECTOR_PH]]:
-; PREFER-NEXT:    br label %[[VECTOR_BODY:.*]]
-; PREFER:       [[VECTOR_BODY]]:
-; PREFER-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
-; PREFER-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], 
i32 [[INDEX]]
-; PREFER-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; PREFER-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 
500)
-; PREFER-NEXT:    [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
-; PREFER-NEXT:    [[TMP3:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 
x i1> [[TMP2]])
-; PREFER-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
-; PREFER-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> 
@llvm.masked.load.v4i32.p0(ptr align 4 [[TMP4]], <4 x i1> [[TMP3]], <4 x i32> 
poison)
-; PREFER-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat 
(i32 1)
-; PREFER-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP5]], 
ptr align 4 [[TMP4]], <4 x i1> [[TMP3]])
-; PREFER-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> 
[[TMP2]])
-; PREFER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; PREFER-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
-; PREFER-NEXT:    br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label 
%[[VECTOR_BODY_INTERIM]]
-; PREFER:       [[VECTOR_BODY_INTERIM]]:
-; PREFER-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; PREFER:       [[MIDDLE_BLOCK]]:
-; PREFER-NEXT:    br label %[[SCALAR_PH:.*]]
-; PREFER:       [[VECTOR_EARLY_EXIT]]:
-; PREFER-NEXT:    br label %[[EXIT:.*]]
-; PREFER:       [[SCALAR_PH]]:
-; PREFER-NEXT:    br label %[[LOOP:.*]]
-; PREFER:       [[LOOP]]:
-; PREFER-NEXT:    [[IV:%.*]] = phi i32 [ 120, %[[SCALAR_PH]] ], [ 
[[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; PREFER-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr 
[[PRED]], i32 [[IV]]
-; PREFER-NEXT:    [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
-; PREFER-NEXT:    [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
-; PREFER-NEXT:    br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]]
-; PREFER:       [[LATCH]]:
-; PREFER-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr 
[[DST]], i32 [[IV]]
-; PREFER-NEXT:    [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
-; PREFER-NEXT:    [[INC:%.*]] = add nsw i32 [[DATA]], 1
-; PREFER-NEXT:    store i32 [[INC]], ptr [[ST_ADDR]], align 4
-; PREFER-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; PREFER-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; PREFER-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop 
[[LOOP7:![0-9]+]]
-; PREFER:       [[EXIT]]:
-; PREFER-NEXT:    ret void
-;
-; MUST-LABEL: define void @early_exit_with_store_after_exit(
-; MUST-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 
4 dereferenceable(4096) [[PRED:%.*]]) {
-; MUST-NEXT:  [[ENTRY:.*]]:
-; MUST-NEXT:    br label %[[LOOP:.*]]
-; MUST:       [[LOOP]]:
-; MUST-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], 
%[[LATCH:.*]] ]
-; MUST-NEXT:    [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr 
[[PRED]], i32 [[IV]]
-; MUST-NEXT:    [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4
-; MUST-NEXT:    [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500
-; MUST-NEXT:    br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[LATCH]]
-; MUST:       [[LATCH]]:
-; MUST-NEXT:    [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], 
i32 [[IV]]
-; MUST-NEXT:    [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4
-; MUST-NEXT:    [[INC:%.*]] = add nsw i32 [[DATA]], 1
-; MUST-NEXT:    store i32 [[INC]], ptr [[ST_ADDR]], align 4
-; MUST-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; MUST-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; MUST-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]]
-; MUST:       [[EXIT]]:
-; MUST-NEXT:    ret void
+; CHECK-LABEL: define void @early_exit_with_store_after_exit(
+; CHECK-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 
4 dereferenceable(4096) [[PRED:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, 
%[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 122)
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], 
i32 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> 
@llvm.masked.load.v4i32.p0(ptr align 4 [[TMP1]], <4 x i1> [[TMP0]], <4 x i32> 
poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt <4 x i32> [[WIDE_MASKED_LOAD]], splat 
(i32 500)
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], 
<4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = freeze <4 x i1> [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x 
i1> [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP5]], 
<4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i32> 
@llvm.masked.load.v4i32.p0(ptr align 4 [[TMP7]], <4 x i1> [[TMP6]], <4 x i32> 
poison)
+; CHECK-NEXT:    [[TMP8:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD1]], splat 
(i32 1)
+; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP8]], ptr 
align 4 [[TMP7]], <4 x i1> [[TMP6]])
+; CHECK-NEXT:    [[TMP9:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP9]], 
<4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> 
[[TMP10]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], 124
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4)
+; CHECK-NEXT:    br i1 [[TMP11]], label %[[VECTOR_EARLY_EXIT:.*]], label 
%[[VECTOR_BODY_INTERIM]]
+; CHECK:       [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -382,7 +352,7 @@ define i32 @countable_early_exit(ptr align 4 
dereferenceable(4096) %p, i32 %n) {
 ; PREFER-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ 
[[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; PREFER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; PREFER-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; PREFER-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; PREFER-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; PREFER:       [[MIDDLE_BLOCK]]:
 ; PREFER-NEXT:    br label %[[SCALAR_PH]]
 ; PREFER:       [[SCALAR_PH]]:
@@ -397,7 +367,7 @@ define i32 @countable_early_exit(ptr align 4 
dereferenceable(4096) %p, i32 %n) {
 ; PREFER-NEXT:    [[LD:%.*]] = load i32, ptr [[GEP]], align 4
 ; PREFER-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
 ; PREFER-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; PREFER-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop 
[[LOOP9:![0-9]+]]
+; PREFER-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop 
[[LOOP8:![0-9]+]]
 ; PREFER:       [[EXIT]]:
 ; PREFER-NEXT:    [[RES:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[LD]], %[[LATCH]] 
]
 ; PREFER-NEXT:    ret i32 [[RES]]
@@ -463,7 +433,7 @@ define void @early_exit_with_store(ptr align 4 
dereferenceable(4096) noalias %ds
 ; PREFER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; PREFER-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120
 ; PREFER-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
-; PREFER-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; PREFER-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label 
%[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; PREFER:       [[MIDDLE_BLOCK]]:
 ; PREFER-NEXT:    [[TMP9:%.*]] = call i64 
@llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
 ; PREFER-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
@@ -485,7 +455,7 @@ define void @early_exit_with_store(ptr align 4 
dereferenceable(4096) noalias %ds
 ; PREFER:       [[LATCH]]:
 ; PREFER-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
 ; PREFER-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123
-; PREFER-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop 
[[LOOP11:![0-9]+]]
+; PREFER-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop 
[[LOOP10:![0-9]+]]
 ; PREFER:       [[EXIT]]:
 ; PREFER-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll 
b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
index 40e869e94857ef..67ba1b29406561 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
@@ -777,8 +777,8 @@ exit:
 define void @uncountable_exit_with_constant_nonunit_stride(ptr 
dereferenceable(4000) noalias %array, ptr align 2 dereferenceable(4000) 
readonly %pred) !dbg !60 {
 ; CHECK-DEBUG-LABEL: LV: Checking a loop in 
'uncountable_exit_with_constant_nonunit_stride'
 ; CHECK-DEBUG:       LV: Loop passed LoopVectorizationLegality checks!
-; CHECK-DEBUG:       LV: Not vectorizing: unable to calculate the loop count 
due to complex control flow.
-; CHECK-REMARK:      foo.c:260:3: loop not vectorized: unable to calculate the 
loop count due to complex control flow
+; CHECK-DEBUG:       LV: Not vectorizing: Early exit loop with side effects 
contains load used by the exit condition with an unsupported memory access 
pattern.
+; CHECK-REMARK:      foo.c:260:3: loop not vectorized: Early exit loop with 
side effects contains load used by the exit condition with an unsupported 
memory access pattern
 entry:
   br label %for.body, !dbg !61
 

_______________________________________________
llvm-branch-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits

Reply via email to