https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/238045
>From 517ed7f37103b5cb5d8b1220d058fe86e840efd7 Mon Sep 17 00:00:00 2001 From: Luke Lau <[email protected]> Date: Tue, 3 Nov 2026 01:15:40 +0800 Subject: [PATCH 1/2] Precommit tests --- .../LoopVectorize/early-exit-tail-folding.ll | 365 ++++++++++-------- 1 file changed, 213 insertions(+), 152 deletions(-) diff --git a/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll index cff27685627005..dfad0a260c0251 100644 --- a/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll +++ b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6 -; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 -force-target-supports-masked-memory-ops -enable-early-exit-vectorization-with-side-effects -tail-folding-policy=prefer-fold-tail | FileCheck %s +; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 -force-target-supports-masked-memory-ops -enable-early-exit-vectorization-with-side-effects -tail-folding-policy=prefer-fold-tail | FileCheck %s --check-prefixes=CHECK,PREFER +; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 -force-target-supports-masked-memory-ops -enable-early-exit-vectorization-with-side-effects -tail-folding-policy=must-fold-tail | FileCheck %s --check-prefixes=CHECK,MUST define i32 @single_early_exit(ptr align 4 dereferenceable(4096) %p) { ; CHECK-LABEL: define i32 @single_early_exit( @@ -271,199 +272,259 @@ exit: ret i32 %res } -; TODO: Cannot tail fold yet. -define i32 @countable_early_exit(ptr align 4 dereferenceable(4096) %p, i32 %n) { -; CHECK-LABEL: define i32 @countable_early_exit( -; CHECK-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], i32 [[N:%.*]]) { -; CHECK-NEXT: [[ENTRY:.*]]: -; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.umin.i32(i32 [[N]], i32 122) -; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i32 [[TMP0]], 1 -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP1]], 4 -; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] -; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[TMP2:%.*]] = and i32 [[TMP1]], 3 -; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[TMP2]], 0 -; CHECK-NEXT: [[TMP4:%.*]] = select i1 [[TMP3]], i32 4, i32 [[TMP2]] -; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[TMP4]] -; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] -; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 -; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] -; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: br label %[[SCALAR_PH]] -; CHECK: [[SCALAR_PH]]: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] -; CHECK-NEXT: br label %[[LOOP:.*]] -; CHECK: [[LOOP]]: -; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ] -; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[IV]], [[N]] -; CHECK-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LATCH]] -; CHECK: [[LATCH]]: -; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]] -; CHECK-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4 -; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 -; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123 -; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]] -; CHECK: [[EXIT]]: -; CHECK-NEXT: [[RES:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[LD]], %[[LATCH]] ] -; CHECK-NEXT: ret i32 [[RES]] +define void @early_exit_with_store_after_exit(ptr align 4 dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) { +; PREFER-LABEL: define void @early_exit_with_store_after_exit( +; PREFER-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) { +; PREFER-NEXT: [[ENTRY:.*:]] +; PREFER-NEXT: br label %[[VECTOR_PH:.*]] +; PREFER: [[VECTOR_PH]]: +; PREFER-NEXT: br label %[[VECTOR_BODY:.*]] +; PREFER: [[VECTOR_BODY]]: +; PREFER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ] +; PREFER-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]] +; PREFER-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4 +; PREFER-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 500) +; PREFER-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]] +; PREFER-NEXT: [[TMP3:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]]) +; PREFER-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]] +; PREFER-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP4]], <4 x i1> [[TMP3]], <4 x i32> poison) +; PREFER-NEXT: [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1) +; PREFER-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP5]], ptr align 4 [[TMP4]], <4 x i1> [[TMP3]]) +; PREFER-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) +; PREFER-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 +; PREFER-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120 +; PREFER-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]] +; PREFER: [[VECTOR_BODY_INTERIM]]: +; PREFER-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] +; PREFER: [[MIDDLE_BLOCK]]: +; PREFER-NEXT: br label %[[SCALAR_PH:.*]] +; PREFER: [[VECTOR_EARLY_EXIT]]: +; PREFER-NEXT: br label %[[EXIT:.*]] +; PREFER: [[SCALAR_PH]]: +; PREFER-NEXT: br label %[[LOOP:.*]] +; PREFER: [[LOOP]]: +; PREFER-NEXT: [[IV:%.*]] = phi i32 [ 120, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ] +; PREFER-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]] +; PREFER-NEXT: [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4 +; PREFER-NEXT: [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500 +; PREFER-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]] +; PREFER: [[LATCH]]: +; PREFER-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]] +; PREFER-NEXT: [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4 +; PREFER-NEXT: [[INC:%.*]] = add nsw i32 [[DATA]], 1 +; PREFER-NEXT: store i32 [[INC]], ptr [[ST_ADDR]], align 4 +; PREFER-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 +; PREFER-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123 +; PREFER-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]] +; PREFER: [[EXIT]]: +; PREFER-NEXT: ret void +; +; MUST-LABEL: define void @early_exit_with_store_after_exit( +; MUST-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) { +; MUST-NEXT: [[ENTRY:.*]]: +; MUST-NEXT: br label %[[LOOP:.*]] +; MUST: [[LOOP]]: +; MUST-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ] +; MUST-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]] +; MUST-NEXT: [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4 +; MUST-NEXT: [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500 +; MUST-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[LATCH]] +; MUST: [[LATCH]]: +; MUST-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]] +; MUST-NEXT: [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4 +; MUST-NEXT: [[INC:%.*]] = add nsw i32 [[DATA]], 1 +; MUST-NEXT: store i32 [[INC]], ptr [[ST_ADDR]], align 4 +; MUST-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 +; MUST-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123 +; MUST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]] +; MUST: [[EXIT]]: +; MUST-NEXT: ret void ; entry: br label %loop loop: %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ] - %c = icmp eq i32 %iv, %n - br i1 %c, label %exit, label %latch + %ee.addr = getelementptr inbounds nuw i32, ptr %pred, i32 %iv + %ee.val = load i32, ptr %ee.addr + %ee.cond = icmp sgt i32 %ee.val, 500 + br i1 %ee.cond, label %exit, label %latch latch: - %gep = getelementptr inbounds i32, ptr %p, i32 %iv - %ld = load i32, ptr %gep, align 4 + %st.addr = getelementptr inbounds nuw i32, ptr %dst, i32 %iv + %data = load i32, ptr %st.addr + %inc = add nsw i32 %data, 1 + store i32 %inc, ptr %st.addr %iv.next = add nuw nsw i32 %iv, 1 %ec = icmp eq i32 %iv.next, 123 br i1 %ec, label %exit, label %loop exit: - %res = phi i32 [ 0, %loop ], [ %ld, %latch ] - ret i32 %res + ret void } ; TODO: Cannot tail fold yet. -define void @early_exit_with_store(ptr align 4 dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) { -; CHECK-LABEL: define void @early_exit_with_store( -; CHECK-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) { -; CHECK-NEXT: [[ENTRY:.*:]] -; CHECK-NEXT: br label %[[VECTOR_PH:.*]] -; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] -; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4 -; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 500) -; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]] -; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]]) -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]] -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP5]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i32> poison) -; CHECK-NEXT: [[TMP6:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1) -; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP6]], ptr align 4 [[TMP5]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]]) -; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 -; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120 -; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]] -; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] -; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false) -; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP12]] to i32 -; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[INDEX]], [[TMP4]] -; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[TMP10]], 123 -; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]] -; CHECK: [[SCALAR_PH]]: -; CHECK-NEXT: br label %[[LOOP:.*]] -; CHECK: [[LOOP]]: -; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[TMP10]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ] -; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]] -; CHECK-NEXT: [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4 -; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[DATA]], 1 -; CHECK-NEXT: store i32 [[INC]], ptr [[ST_ADDR]], align 4 -; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]] -; CHECK-NEXT: [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4 -; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500 -; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]] -; CHECK: [[LATCH]]: -; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 -; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123 -; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]] -; CHECK: [[EXIT]]: -; CHECK-NEXT: ret void +define i32 @countable_early_exit(ptr align 4 dereferenceable(4096) %p, i32 %n) { +; PREFER-LABEL: define i32 @countable_early_exit( +; PREFER-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], i32 [[N:%.*]]) { +; PREFER-NEXT: [[ENTRY:.*]]: +; PREFER-NEXT: [[TMP0:%.*]] = call i32 @llvm.umin.i32(i32 [[N]], i32 122) +; PREFER-NEXT: [[TMP1:%.*]] = add nuw nsw i32 [[TMP0]], 1 +; PREFER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP1]], 4 +; PREFER-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; PREFER: [[VECTOR_PH]]: +; PREFER-NEXT: [[TMP2:%.*]] = and i32 [[TMP1]], 3 +; PREFER-NEXT: [[TMP3:%.*]] = icmp eq i32 [[TMP2]], 0 +; PREFER-NEXT: [[TMP4:%.*]] = select i1 [[TMP3]], i32 4, i32 [[TMP2]] +; PREFER-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[TMP4]] +; PREFER-NEXT: br label %[[VECTOR_BODY:.*]] +; PREFER: [[VECTOR_BODY]]: +; PREFER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; PREFER-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 +; PREFER-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] +; PREFER-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] +; PREFER: [[MIDDLE_BLOCK]]: +; PREFER-NEXT: br label %[[SCALAR_PH]] +; PREFER: [[SCALAR_PH]]: +; PREFER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; PREFER-NEXT: br label %[[LOOP:.*]] +; PREFER: [[LOOP]]: +; PREFER-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ] +; PREFER-NEXT: [[C:%.*]] = icmp eq i32 [[IV]], [[N]] +; PREFER-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LATCH]] +; PREFER: [[LATCH]]: +; PREFER-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]] +; PREFER-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4 +; PREFER-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 +; PREFER-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123 +; PREFER-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]] +; PREFER: [[EXIT]]: +; PREFER-NEXT: [[RES:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[LD]], %[[LATCH]] ] +; PREFER-NEXT: ret i32 [[RES]] +; +; MUST-LABEL: define i32 @countable_early_exit( +; MUST-SAME: ptr align 4 dereferenceable(4096) [[P:%.*]], i32 [[N:%.*]]) { +; MUST-NEXT: [[ENTRY:.*]]: +; MUST-NEXT: br label %[[LOOP:.*]] +; MUST: [[LOOP]]: +; MUST-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ] +; MUST-NEXT: [[C:%.*]] = icmp eq i32 [[IV]], [[N]] +; MUST-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LATCH]] +; MUST: [[LATCH]]: +; MUST-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[IV]] +; MUST-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4 +; MUST-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 +; MUST-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123 +; MUST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]] +; MUST: [[EXIT]]: +; MUST-NEXT: [[RES:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[LD]], %[[LATCH]] ] +; MUST-NEXT: ret i32 [[RES]] ; entry: br label %loop loop: %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ] - %st.addr = getelementptr inbounds nuw i32, ptr %dst, i32 %iv - %data = load i32, ptr %st.addr, align 4 - %inc = add nsw i32 %data, 1 - store i32 %inc, ptr %st.addr, align 4 - %ee.addr = getelementptr inbounds nuw i32, ptr %pred, i32 %iv - %ee.val = load i32, ptr %ee.addr, align 4 - %ee.cond = icmp sgt i32 %ee.val, 500 - br i1 %ee.cond, label %exit, label %latch + %c = icmp eq i32 %iv, %n + br i1 %c, label %exit, label %latch latch: + %gep = getelementptr inbounds i32, ptr %p, i32 %iv + %ld = load i32, ptr %gep, align 4 %iv.next = add nuw nsw i32 %iv, 1 %ec = icmp eq i32 %iv.next, 123 br i1 %ec, label %exit, label %loop exit: - ret void + %res = phi i32 [ 0, %loop ], [ %ld, %latch ] + ret i32 %res } -define void @early_exit_with_store_after_exit(ptr align 4 dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) { -; CHECK-LABEL: define void @early_exit_with_store_after_exit( -; CHECK-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) { -; CHECK-NEXT: [[ENTRY:.*:]] -; CHECK-NEXT: br label %[[VECTOR_PH:.*]] -; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] -; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ] -; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4 -; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 500) -; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]] -; CHECK-NEXT: [[TMP3:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]]) -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]] -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP4]], <4 x i1> [[TMP3]], <4 x i32> poison) -; CHECK-NEXT: [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1) -; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP5]], ptr align 4 [[TMP4]], <4 x i1> [[TMP3]]) -; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 -; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120 -; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]] -; CHECK: [[VECTOR_BODY_INTERIM]]: -; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] -; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: br label %[[SCALAR_PH:.*]] -; CHECK: [[VECTOR_EARLY_EXIT]]: -; CHECK-NEXT: br label %[[EXIT:.*]] -; CHECK: [[SCALAR_PH]]: -; CHECK-NEXT: br label %[[LOOP:.*]] -; CHECK: [[LOOP]]: -; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 120, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ] -; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]] -; CHECK-NEXT: [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4 -; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500 -; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]] -; CHECK: [[LATCH]]: -; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]] -; CHECK-NEXT: [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4 -; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[DATA]], 1 -; CHECK-NEXT: store i32 [[INC]], ptr [[ST_ADDR]], align 4 -; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 -; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123 -; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]] -; CHECK: [[EXIT]]: -; CHECK-NEXT: ret void +; TODO: Cannot tail fold yet. +define void @early_exit_with_store(ptr align 4 dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) { +; PREFER-LABEL: define void @early_exit_with_store( +; PREFER-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) { +; PREFER-NEXT: [[ENTRY:.*:]] +; PREFER-NEXT: br label %[[VECTOR_PH:.*]] +; PREFER: [[VECTOR_PH]]: +; PREFER-NEXT: br label %[[VECTOR_BODY:.*]] +; PREFER: [[VECTOR_BODY]]: +; PREFER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; PREFER-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]] +; PREFER-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4 +; PREFER-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 500) +; PREFER-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]] +; PREFER-NEXT: [[TMP3:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]]) +; PREFER-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]] +; PREFER-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP4]], <4 x i1> [[TMP3]], <4 x i32> poison) +; PREFER-NEXT: [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1) +; PREFER-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP5]], ptr align 4 [[TMP4]], <4 x i1> [[TMP3]]) +; PREFER-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) +; PREFER-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 +; PREFER-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120 +; PREFER-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]] +; PREFER-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] +; PREFER: [[MIDDLE_BLOCK]]: +; PREFER-NEXT: [[TMP9:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false) +; PREFER-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; PREFER-NEXT: [[TMP11:%.*]] = add i32 [[INDEX]], [[TMP10]] +; PREFER-NEXT: [[TMP12:%.*]] = icmp eq i32 [[TMP11]], 123 +; PREFER-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]] +; PREFER: [[SCALAR_PH]]: +; PREFER-NEXT: br label %[[LOOP:.*]] +; PREFER: [[LOOP]]: +; PREFER-NEXT: [[IV:%.*]] = phi i32 [ [[TMP11]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ] +; PREFER-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]] +; PREFER-NEXT: [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4 +; PREFER-NEXT: [[INC:%.*]] = add nsw i32 [[DATA]], 1 +; PREFER-NEXT: store i32 [[INC]], ptr [[ST_ADDR]], align 4 +; PREFER-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]] +; PREFER-NEXT: [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4 +; PREFER-NEXT: [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500 +; PREFER-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]] +; PREFER: [[LATCH]]: +; PREFER-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 +; PREFER-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123 +; PREFER-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]] +; PREFER: [[EXIT]]: +; PREFER-NEXT: ret void +; +; MUST-LABEL: define void @early_exit_with_store( +; MUST-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) { +; MUST-NEXT: [[ENTRY:.*]]: +; MUST-NEXT: br label %[[LOOP:.*]] +; MUST: [[LOOP]]: +; MUST-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ] +; MUST-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]] +; MUST-NEXT: [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4 +; MUST-NEXT: [[INC:%.*]] = add nsw i32 [[DATA]], 1 +; MUST-NEXT: store i32 [[INC]], ptr [[ST_ADDR]], align 4 +; MUST-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]] +; MUST-NEXT: [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4 +; MUST-NEXT: [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500 +; MUST-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[LATCH]] +; MUST: [[LATCH]]: +; MUST-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 +; MUST-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123 +; MUST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]] +; MUST: [[EXIT]]: +; MUST-NEXT: ret void ; entry: br label %loop loop: %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ] + %st.addr = getelementptr inbounds nuw i32, ptr %dst, i32 %iv + %data = load i32, ptr %st.addr, align 4 + %inc = add nsw i32 %data, 1 + store i32 %inc, ptr %st.addr, align 4 %ee.addr = getelementptr inbounds nuw i32, ptr %pred, i32 %iv - %ee.val = load i32, ptr %ee.addr + %ee.val = load i32, ptr %ee.addr, align 4 %ee.cond = icmp sgt i32 %ee.val, 500 br i1 %ee.cond, label %exit, label %latch latch: - %st.addr = getelementptr inbounds nuw i32, ptr %dst, i32 %iv - %data = load i32, ptr %st.addr - %inc = add nsw i32 %data, 1 - store i32 %inc, ptr %st.addr %iv.next = add nuw nsw i32 %iv, 1 %ec = icmp eq i32 %iv.next, 123 br i1 %ec, label %exit, label %loop >From 8bf28763b549e0a4b6347fc5fb7434ec10ab76f9 Mon Sep 17 00:00:00 2001 From: Luke Lau <[email protected]> Date: Sat, 31 Oct 2026 02:56:48 +0800 Subject: [PATCH 2/2] [VPlan] Allow tail folding early exits with side effects Continues the work in #179595. This allows us to tail fold e.g. the following loop: ```c extern char a[1024], b[1024], c[1024], y[1024]; void two_guards(void) { for (int i = 0; i < 1000; i++) { if (a[i] == 0) break; if (b[i] > 3) { if (c[i] < 9) y[i] = 1; } } } ``` Into this on RISC-V: ```asm .LBB0_1: # %vector.body # =>This Inner Loop Header: Depth=1 vsetvli a6, a1, e8, m2, ta, ma add a7, a2, a0 vle8.v v10, (a7) vmseq.vi v12, v10, 0 vmsbf.m v0, v12 add a7, a3, a0 vle8.v v10, (a7), v0.t vmsgtu.vi v13, v10, 3 vmand.mm v0, v0, v13 add a7, a4, a0 vle8.v v10, (a7), v0.t vmsleu.vi v13, v10, 8 vmand.mm v0, v0, v13 vcpop.m a7, v12 add t0, a5, a0 vse8.v v8, (t0), v0.t bnez a7, .LBB0_3 # in Loop: Header=BB0_1 Depth=1 sub a1, a1, a6 add a0, a0, a6 bnez a1, .LBB0_1 ``` For early exit loops with side effects, we may be able to handle the exits by masking the vector body, or we may have to handle them in the scalar loop. If it's the latter we can't tail fold the plan just yet, we need to fixup a lot of assumptions around the middle block etc. For now just disable tail folding in the cost model. It's safe to change the tail folding here because foldTailByMasking hasn't been called yet and there hasn't been any costing done. --- .../Vectorize/LoopVectorizationLegality.cpp | 7 -- .../Transforms/Vectorize/LoopVectorize.cpp | 24 ++++ .../Transforms/Vectorize/VPlanTransforms.cpp | 2 +- .../Transforms/Vectorize/VPlanTransforms.h | 3 +- .../LoopVectorize/RISCV/early-exit.ll | 53 +++------ .../RISCV/early_exit_with_stores.ll | 65 ++++------- .../LoopVectorize/early-exit-tail-folding.ll | 110 +++++++----------- .../early_exit_store_legality.ll | 4 +- 8 files changed, 111 insertions(+), 157 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp index 0cd4b7ec2ffb45..1271fe6de706a8 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp @@ -2115,13 +2115,6 @@ bool LoopVectorizationLegality::canFoldTailByMasking() const { return false; } - // TODO: Handle early exits with side effects without a scalar tail. - if (hasUncountableExitWithSideEffects()) { - LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking. Uncountable exit " - "with side effects requires a scalar epilogue\n"); - return false; - } - LLVM_DEBUG(dbgs() << "LV: checking if tail can be folded by masking.\n"); // No accesses are known to be safe in inactive lanes of the folded tail. diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 70a11792f76fd9..b1faf533f1d263 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -748,6 +748,20 @@ class LoopVectorizationCostModel { : "due to non-interleaving reasons.\n")); } + /// Disable all tail folding styles. Returns false if an epilogue isn't + /// allowed. Must be called before the plan is tail folded. + bool tryToDisableTailFolding() { + assert(foldTailByMasking() && "Expected tail folding to be enabled!"); + if (EpilogueLoweringStatus != CM_EpilogueNotNeededFoldTail) + return false; + LLVM_DEBUG(dbgs() << "LV: Disabling tail folding: vectorize with an " + "epilogue instead.\n"); + ChosenTailFoldingStyle = TailFoldingStyle::None; + EpilogueLoweringStatus = CM_EpilogueAllowed; + PartialAliasMaskingStatus = AliasMaskingStatus::Disabled; + return true; + } + /// Returns true if all loop blocks should be masked to fold tail loop. bool foldTailByMasking() const { return getTailFoldingStyle() != TailFoldingStyle::None; @@ -6283,6 +6297,16 @@ LoopVectorizationPlanner::tryToBuildVPlan1(LoopVectorizationCostModel &PlanCM) { EEStyle)) { return nullptr; } + + // Exiting in the scalar loop requires a scalar epilogue for now. + if (EEStyle == UncountableExitStyle::MaskedHandleExitInScalarLoop && + PlanCM.foldTailByMasking() && !PlanCM.tryToDisableTailFolding()) { + reportVectorizationFailure( + "Cannot fold tail by masking when early exits are handled in the " + "scalar loop", + "EarlyExitInScalarLoopCannotFoldTail", ORE, OrigLoop); + return nullptr; + } } else { RUN_VPLAN_PASS(VPlanTransforms::handleCountableEarlyExits, *VPlan0); } diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 548ddc68f4f398..201987855142a0 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -3578,7 +3578,7 @@ static SmallVector<VPBasicBlock *> blocksWithStores(VPBasicBlock *HeaderVPBB) { bool VPlanTransforms::handleUncountableEarlyExits( VPlan &Plan, OptimizationRemarkEmitter *ORE, Loop *TheLoop, PredicatedScalarEvolution &PSE, DominatorTree &DT, AssumptionCache *AC, - UncountableExitStyle Style) { + UncountableExitStyle &Style) { VPDominatorTree VPDT(Plan); auto *MiddleVPBB = VPBlockUtils::getPlainCFGMiddleBlock(Plan); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index f38fa15d0a16c0..93b6eb90fba25e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -409,11 +409,12 @@ struct VPlanTransforms { /// appropriate branching logic in the latch that handles early exits and the /// latch exit condition. Multiple exits are handled with a dispatch block /// that determines which exit to take based on lane-by-lane semantics. + /// \p Style is the preferred style, and is updated to the style used. LLVM_ABI_FOR_TEST static bool handleUncountableEarlyExits(VPlan &Plan, OptimizationRemarkEmitter *ORE, Loop *TheLoop, PredicatedScalarEvolution &PSE, DominatorTree &DT, AssumptionCache *AC, - UncountableExitStyle Style); + UncountableExitStyle &Style); /// Disconnect countable early exits from the loop. LLVM_ABI_FOR_TEST static void handleCountableEarlyExits(VPlan &Plan); diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll index f8563ff31a417f..06afcc1d4e1c8b 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/early-exit.ll @@ -405,54 +405,37 @@ exit: define void @early_exit_with_store_after_exit(ptr align 4 dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) { ; CHECK-LABEL: define void @early_exit_with_store_after_exit( ; CHECK-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[ENTRY:.*]]: -; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2 -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 123, [[TMP1]] -; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 123, [[TMP1]] -; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 123, [[N_MOD_VF]] ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ] +; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ] +; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 123, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ] +; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true) ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP2]], align 4 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) ; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 500) -; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP3]] +; CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP3]], <vscale x 4 x i1> zeroinitializer, i32 [[TMP0]]) +; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 4 x i1> [[TMP10]] ; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i1> @llvm.mask.beforefirst.nxv4i1(<vscale x 4 x i1> [[TMP4]]) ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]] -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[TMP5]], <vscale x 4 x i32> poison) +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> [[TMP5]], i32 [[TMP0]]) ; CHECK-NEXT: [[TMP7:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1) -; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP7]], ptr align 4 [[TMP6]], <vscale x 4 x i1> [[TMP5]]) -; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP4]]) -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]] -; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP7]], ptr align 4 [[TMP6]], <vscale x 4 x i1> [[TMP5]], i32 [[TMP0]]) +; CHECK-NEXT: [[TMP12:%.*]] = freeze <vscale x 4 x i1> [[TMP3]] +; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vp.reduce.or.nxv4i1(i1 false, <vscale x 4 x i1> [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP9]], false +; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i32 [[TMP0]], [[INDEX]] +; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP0]] +; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[AVL_NEXT]], 0 ; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]] ; CHECK: [[VECTOR_BODY_INTERIM]]: -; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 123, [[N_VEC]] -; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] +; CHECK-NEXT: br label %[[EXIT:.*]] ; CHECK: [[VECTOR_EARLY_EXIT]]: ; CHECK-NEXT: br label %[[EXIT]] -; CHECK: [[SCALAR_PH]]: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] -; CHECK-NEXT: br label %[[LOOP:.*]] -; CHECK: [[LOOP]]: -; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ] -; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]] -; CHECK-NEXT: [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4 -; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500 -; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]] -; CHECK: [[LATCH]]: -; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]] -; CHECK-NEXT: [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4 -; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[DATA]], 1 -; CHECK-NEXT: store i32 [[INC]], ptr [[ST_ADDR]], align 4 -; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 -; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123 -; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]] ; CHECK: [[EXIT]]: ; CHECK-NEXT: ret void ; diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll index 8ba6d5f81679e5..a566df5db77019 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll @@ -1267,57 +1267,40 @@ exit.uncountable: define void @interleaved_stores_after_early_exit(ptr noalias %dst, ptr align 2 dereferenceable(40) %src) { ; CHECK-LABEL: define void @interleaved_stores_after_early_exit( ; CHECK-SAME: ptr noalias [[DST:%.*]], ptr align 2 dereferenceable(40) [[SRC:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[ENTRY:.*]]: -; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3 -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]] -; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]] -; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]] ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[MIDDLE_BLOCK:.*]] ] +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[MIDDLE_BLOCK:.*]] ] +; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 20, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[MIDDLE_BLOCK]] ] +; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true) ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP2]], align 2 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 2 [[TMP2]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]]) ; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500) -; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 8 x i1> [[TMP3]] +; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 8 x i1> @llvm.vp.merge.nxv8i1(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x i1> [[TMP3]], <vscale x 8 x i1> zeroinitializer, i32 [[TMP0]]) +; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 8 x i1> [[TMP5]] ; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.mask.beforefirst.nxv8i1(<vscale x 8 x i1> [[TMP4]]) ; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[INDEX]], 1 ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP6]] -; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 16 x i16> @llvm.vector.interleave2.nxv16i16(<vscale x 8 x i16> splat (i16 1), <vscale x 8 x i16> splat (i16 2)) +; CHECK-NEXT: [[INTERLEAVE_EVL:%.*]] = mul nuw nsw i32 [[TMP0]], 2 ; CHECK-NEXT: [[INTERLEAVED_MASK:%.*]] = call <vscale x 16 x i1> @llvm.vector.interleave2.nxv16i1(<vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]]) -; CHECK-NEXT: call void @llvm.masked.store.nxv16i16.p0(<vscale x 16 x i16> [[INTERLEAVED_VEC]], ptr align 2 [[TMP7]], <vscale x 16 x i1> [[INTERLEAVED_MASK]]) -; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP4]]) -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]] -; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 16 x i16> @llvm.vector.interleave2.nxv16i16(<vscale x 8 x i16> splat (i16 1), <vscale x 8 x i16> splat (i16 2)) +; CHECK-NEXT: call void @llvm.vp.store.nxv16i16.p0(<vscale x 16 x i16> [[INTERLEAVED_VEC]], ptr align 2 [[TMP7]], <vscale x 16 x i1> [[INTERLEAVED_MASK]], i32 [[INTERLEAVE_EVL]]) +; CHECK-NEXT: [[TMP10:%.*]] = freeze <vscale x 8 x i1> [[TMP3]] +; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vp.reduce.or.nxv8i1(i1 false, <vscale x 8 x i1> [[TMP10]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP9]], false +; CHECK-NEXT: [[TMP11:%.*]] = zext i32 [[TMP0]] to i64 +; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP11]], [[INDEX]] +; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]] +; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 ; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK]] ; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK1]]: -; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 20, [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]] +; CHECK-NEXT: br label %[[EXIT:.*]] ; CHECK: [[VECTOR_EARLY_EXIT]]: ; CHECK-NEXT: br label %[[EXIT]] -; CHECK: [[SCALAR_PH]]: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK1]] ], [ 0, %[[ENTRY]] ] -; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] -; CHECK: [[LOOP_HEADER]]: -; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] -; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]] -; CHECK-NEXT: [[L:%.*]] = load i16, ptr [[GEP_SRC]], align 2 -; CHECK-NEXT: [[C:%.*]] = icmp sgt i16 [[L]], 500 -; CHECK-NEXT: br i1 [[C]], label %[[EXIT]], label %[[LOOP_LATCH]] -; CHECK: [[LOOP_LATCH]]: -; CHECK-NEXT: [[OFF_0:%.*]] = shl i64 [[IV]], 1 -; CHECK-NEXT: [[GEP_DST_0:%.*]] = getelementptr i16, ptr [[DST]], i64 [[OFF_0]] -; CHECK-NEXT: store i16 1, ptr [[GEP_DST_0]], align 2 -; CHECK-NEXT: [[OFF_1:%.*]] = or disjoint i64 [[OFF_0]], 1 -; CHECK-NEXT: [[GEP_DST_1:%.*]] = getelementptr i16, ptr [[DST]], i64 [[OFF_1]] -; CHECK-NEXT: store i16 2, ptr [[GEP_DST_1]], align 2 -; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 -; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 20 -; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP12:![0-9]+]] ; CHECK: [[EXIT]]: ; CHECK-NEXT: ret void ; @@ -1374,7 +1357,7 @@ define void @interleaved_stores_before_early_exit(ptr noalias %dst, ptr align 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]] ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]] -; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP4]], i1 false) ; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP5]] @@ -1398,7 +1381,7 @@ define void @interleaved_stores_before_early_exit(ptr noalias %dst, ptr align 2 ; CHECK: [[LOOP_LATCH]]: ; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 20 -; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP14:![0-9]+]] +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP13:![0-9]+]] ; CHECK: [[EXIT]]: ; CHECK-NEXT: ret void ; @@ -1458,7 +1441,7 @@ define void @reverse_interleaved_stores_before_early_exit(ptr noalias %dst, ptr ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]] ; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: [[TMP14:%.*]] = or i1 [[TMP12]], [[TMP13]] -; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: ; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> [[TMP7]], i1 false) ; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], [[TMP8]] @@ -1483,7 +1466,7 @@ define void @reverse_interleaved_stores_before_early_exit(ptr noalias %dst, ptr ; CHECK: [[LOOP_LATCH]]: ; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 20 -; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP16:![0-9]+]] +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP15:![0-9]+]] ; CHECK: [[EXIT]]: ; CHECK-NEXT: ret void ; diff --git a/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll index dfad0a260c0251..74cd834636c7c3 100644 --- a/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll +++ b/llvm/test/Transforms/LoopVectorize/early-exit-tail-folding.ll @@ -273,72 +273,42 @@ exit: } define void @early_exit_with_store_after_exit(ptr align 4 dereferenceable(4096) noalias %dst, ptr align 4 dereferenceable(4096) %pred) { -; PREFER-LABEL: define void @early_exit_with_store_after_exit( -; PREFER-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) { -; PREFER-NEXT: [[ENTRY:.*:]] -; PREFER-NEXT: br label %[[VECTOR_PH:.*]] -; PREFER: [[VECTOR_PH]]: -; PREFER-NEXT: br label %[[VECTOR_BODY:.*]] -; PREFER: [[VECTOR_BODY]]: -; PREFER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ] -; PREFER-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]] -; PREFER-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4 -; PREFER-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 500) -; PREFER-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]] -; PREFER-NEXT: [[TMP3:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP2]]) -; PREFER-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]] -; PREFER-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP4]], <4 x i1> [[TMP3]], <4 x i32> poison) -; PREFER-NEXT: [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 1) -; PREFER-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP5]], ptr align 4 [[TMP4]], <4 x i1> [[TMP3]]) -; PREFER-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) -; PREFER-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 -; PREFER-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120 -; PREFER-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]] -; PREFER: [[VECTOR_BODY_INTERIM]]: -; PREFER-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] -; PREFER: [[MIDDLE_BLOCK]]: -; PREFER-NEXT: br label %[[SCALAR_PH:.*]] -; PREFER: [[VECTOR_EARLY_EXIT]]: -; PREFER-NEXT: br label %[[EXIT:.*]] -; PREFER: [[SCALAR_PH]]: -; PREFER-NEXT: br label %[[LOOP:.*]] -; PREFER: [[LOOP]]: -; PREFER-NEXT: [[IV:%.*]] = phi i32 [ 120, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ] -; PREFER-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]] -; PREFER-NEXT: [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4 -; PREFER-NEXT: [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500 -; PREFER-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LATCH]] -; PREFER: [[LATCH]]: -; PREFER-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]] -; PREFER-NEXT: [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4 -; PREFER-NEXT: [[INC:%.*]] = add nsw i32 [[DATA]], 1 -; PREFER-NEXT: store i32 [[INC]], ptr [[ST_ADDR]], align 4 -; PREFER-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 -; PREFER-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123 -; PREFER-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]] -; PREFER: [[EXIT]]: -; PREFER-NEXT: ret void -; -; MUST-LABEL: define void @early_exit_with_store_after_exit( -; MUST-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) { -; MUST-NEXT: [[ENTRY:.*]]: -; MUST-NEXT: br label %[[LOOP:.*]] -; MUST: [[LOOP]]: -; MUST-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ] -; MUST-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[IV]] -; MUST-NEXT: [[EE_VAL:%.*]] = load i32, ptr [[EE_ADDR]], align 4 -; MUST-NEXT: [[EE_COND:%.*]] = icmp sgt i32 [[EE_VAL]], 500 -; MUST-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[LATCH]] -; MUST: [[LATCH]]: -; MUST-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]] -; MUST-NEXT: [[DATA:%.*]] = load i32, ptr [[ST_ADDR]], align 4 -; MUST-NEXT: [[INC:%.*]] = add nsw i32 [[DATA]], 1 -; MUST-NEXT: store i32 [[INC]], ptr [[ST_ADDR]], align 4 -; MUST-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 -; MUST-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123 -; MUST-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]] -; MUST: [[EXIT]]: -; MUST-NEXT: ret void +; CHECK-LABEL: define void @early_exit_with_store_after_exit( +; CHECK-SAME: ptr noalias align 4 dereferenceable(4096) [[DST:%.*]], ptr align 4 dereferenceable(4096) [[PRED:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY_INTERIM]] ] +; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 122) +; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i32, ptr [[PRED]], i32 [[INDEX]] +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP1]], <4 x i1> [[TMP0]], <4 x i32> poison) +; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 500) +; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP3]] +; CHECK-NEXT: [[TMP5:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP4]]) +; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP5]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[DST]], i32 [[INDEX]] +; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP7]], <4 x i1> [[TMP6]], <4 x i32> poison) +; CHECK-NEXT: [[TMP8:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD1]], splat (i32 1) +; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP8]], ptr align 4 [[TMP7]], <4 x i1> [[TMP6]]) +; CHECK-NEXT: [[TMP9:%.*]] = freeze <4 x i1> [[TMP2]] +; CHECK-NEXT: [[TMP10:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP9]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 +; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], 124 +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4) +; CHECK-NEXT: br i1 [[TMP11]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]] +; CHECK: [[VECTOR_BODY_INTERIM]]: +; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[VECTOR_EARLY_EXIT]]: +; CHECK-NEXT: br label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void ; entry: br label %loop @@ -382,7 +352,7 @@ define i32 @countable_early_exit(ptr align 4 dereferenceable(4096) %p, i32 %n) { ; PREFER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] ; PREFER-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 ; PREFER-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; PREFER-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] +; PREFER-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] ; PREFER: [[MIDDLE_BLOCK]]: ; PREFER-NEXT: br label %[[SCALAR_PH]] ; PREFER: [[SCALAR_PH]]: @@ -397,7 +367,7 @@ define i32 @countable_early_exit(ptr align 4 dereferenceable(4096) %p, i32 %n) { ; PREFER-NEXT: [[LD:%.*]] = load i32, ptr [[GEP]], align 4 ; PREFER-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 ; PREFER-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123 -; PREFER-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]] +; PREFER-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]] ; PREFER: [[EXIT]]: ; PREFER-NEXT: [[RES:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[LD]], %[[LATCH]] ] ; PREFER-NEXT: ret i32 [[RES]] @@ -463,7 +433,7 @@ define void @early_exit_with_store(ptr align 4 dereferenceable(4096) noalias %ds ; PREFER-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 ; PREFER-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 120 ; PREFER-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]] -; PREFER-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] +; PREFER-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] ; PREFER: [[MIDDLE_BLOCK]]: ; PREFER-NEXT: [[TMP9:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false) ; PREFER-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 @@ -485,7 +455,7 @@ define void @early_exit_with_store(ptr align 4 dereferenceable(4096) noalias %ds ; PREFER: [[LATCH]]: ; PREFER-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 ; PREFER-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 123 -; PREFER-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]] +; PREFER-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP10:![0-9]+]] ; PREFER: [[EXIT]]: ; PREFER-NEXT: ret void ; diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll index 40e869e94857ef..67ba1b29406561 100644 --- a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll +++ b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll @@ -777,8 +777,8 @@ exit: define void @uncountable_exit_with_constant_nonunit_stride(ptr dereferenceable(4000) noalias %array, ptr align 2 dereferenceable(4000) readonly %pred) !dbg !60 { ; CHECK-DEBUG-LABEL: LV: Checking a loop in 'uncountable_exit_with_constant_nonunit_stride' ; CHECK-DEBUG: LV: Loop passed LoopVectorizationLegality checks! -; CHECK-DEBUG: LV: Not vectorizing: unable to calculate the loop count due to complex control flow. -; CHECK-REMARK: foo.c:260:3: loop not vectorized: unable to calculate the loop count due to complex control flow +; CHECK-DEBUG: LV: Not vectorizing: Early exit loop with side effects contains load used by the exit condition with an unsupported memory access pattern. +; CHECK-REMARK: foo.c:260:3: loop not vectorized: Early exit loop with side effects contains load used by the exit condition with an unsupported memory access pattern entry: br label %for.body, !dbg !61 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
