https://github.com/eas updated https://github.com/llvm/llvm-project/pull/217484
>From e6cebbcea4047f857a53310b172879fd5a04df37 Mon Sep 17 00:00:00 2001 From: Andrei Elovikov <[email protected]> Date: Fri, 14 Aug 2026 12:20:20 -0700 Subject: [PATCH 1/3] Predicator tests for uniform control flow preservation --- .../LoopVectorize/VPlan/predicator.ll | 168 ++++++++++++++++++ 1 file changed, 168 insertions(+) diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll index 429dbb5b7fffd..07161159a8d48 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll @@ -949,3 +949,171 @@ latch: exit: ret void } + +; @outermost_uniform_branch +; bb0 (u) +; / \ +; bb1 (v) \ +; / | | +; bb2 | | +; \ | | +; bb3 [phi] | +; \ / +; bb4 [phi] +; +; bb0's uniform branch can be easily preserved. + +; @outermost_uniform_branch_more_blocks +; bb0 (u) +; / \ +; bb2 (v) bb1 +; / \ | +; bb4 bb3 | +; \ / | +; bb5 [phi] | +; \ / +; bb6 [phi] +; +; Similar to above, but with extra blocks on some edges. Probably doesn't +; require any extra handling but nice to test explicitly. + +; @uniform_branch_after_varying_branch +; bb0 (v) +; / \ +; bb4 bb1 (u) +; | / | +; | bb2 | +; | \ | +; | bb3 [phi] +; \ / +; bb5 [phi] + +; @uniform_branch_after_varying_branch_more_blocks +; bb0 (v) +; / \ +; bb5 bb1 (u) +; | / \ +; | bb3 bb2 +; | \ / +; | bb4 [phi] +; \ / +; bb6 [phi] +; +; Similar to above, but with extra blocks on some edges. Probably doesn't +; require any extra handling but nice to test explicitly. + +; @uniform_branch_after_varying_branch_more_blocks_mirrored +; bb0 (v) +; / \ +; bb2 (u) bb1 +; / \ | +; bb4 bb3 | +; \ / | +; bb5 [phi] | +; \ / +; bb6 [phi] +; +; Mirror of the above test so that "(u)" block would be processed before/after +; the other "(v)" destination by the RPOT. + +; @uniform_branch_on_masked_def +; bb0 (v) +; / \ +; | bb1 (br i1 (load i1 %uni.ptr)) +; | / \ +; | bb3 bb2 +; | \ / +; | bb4 +; \ / +; bb5 [phi] +; +; bb1's condition is uniform, but its value is loaded only along one path of +; bb0's varying branch. It cannot be preserved trivially, because some the load +; may never be executed. + + +; All @uniform_branch_unstructured_merge[0-4] below have the same structure of uniform +; control flow merging into the middle of the varying diamond, but "covering" +; different potential RPOT traversals. + +; @uniform_branch_unstructured_merge1 +; bb0 (u) +; / \ +; bb2 (v) bb1 +; / \ / +; bb4 bb3 [phi] +; \ / +; bb5 [phi] +; + +; @uniform_branch_unstructured_merge2 +; bb0 (u) +; / \ +; bb2 (v) bb1 +; / \ / +; | +-\---+ +; | / \ +; bb4 [phi] bb3 +; \ / +; bb5 [phi] +; + +; @uniform_branch_unstructured_merge3 +; bb0 (u) +; / \ +; bb3 bb1 (v) +; \ / \ +; bb4 [phi] bb2 +; \ / +; bb5 [phi] +; + +; @uniform_branch_unstructured_merge4 +; bb0 (u) +; / \ +; bb3 bb1 (v) +; \ / \ +; +-/--+ \ +; / \ \ +; bb4 bb2 [phi] +; \ / +; bb5 [phi] + + + +; @uniform_branch_shared_join_with_varying +; bb0 (u) +; / \ +; bb2 (v) bb1 +; / | | +; bb3 | / +; \ | / +; bb4 [phi] +; +; Theoretically can be done, but would require a combination of a blend and a phi. + +; @unstructured_uniform_only +; bb0 (u) +; / \ +; bb2 (u) bb1 +; / \ / +; bb4 bb3 [phi] +; \ / +; bb5 [phi] +; +; Triviallly preservable, but detection might not be easy. + +; @unstructured_uniform_only_sese_region +; bb0 (v) +; / \ +; bb6 bb1 (u) +; | / \ +; | bb3 (u) bb2 +; | / \ / +; | bb5 bb4 [phi] +; | \ / +; | bb7 [phi] +; \ / +; \ / +; bb8 [phi] +; >From ae18eb28cc7bd7ce823be487afb033bf0a8a985c Mon Sep 17 00:00:00 2001 From: Andrei Elovikov <[email protected]> Date: Fri, 14 Aug 2026 14:23:21 -0700 Subject: [PATCH 2/3] Create actual test functions (AI-assisted) --- .../LoopVectorize/VPlan/predicator.ll | 503 +++++++++++++++++- 1 file changed, 478 insertions(+), 25 deletions(-) diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll index 07161159a8d48..1c95c9381ab73 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll @@ -950,7 +950,11 @@ exit: ret void } -; @outermost_uniform_branch +define void @outermost_uniform_branch(ptr %a, i1 %u0) { +entry: + br label %bb0 + +bb0: ; bb0 (u) ; / \ ; bb1 (v) \ @@ -960,10 +964,41 @@ exit: ; bb3 [phi] | ; \ / ; bb4 [phi] -; ; bb0's uniform branch can be easily preserved. + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ] + %add0 = add i64 %iv, 0 + br i1 %u0, label %bb1, label %bb4 + +bb1: + %add1 = add i64 %iv, 1 + %v1 = icmp sle i64 %iv, 1 + br i1 %v1, label %bb2, label %bb3 + +bb2: + %add2 = add i64 %iv, 2 + br label %bb3 + +bb3: + %phi3 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ] + %add3 = add i64 %phi3, 3 + br label %bb4 + +bb4: + %phi4 = phi i64 [ %add3, %bb3 ], [ %add0, %bb0 ] + store i64 %phi4, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +define void @outermost_uniform_branch_more_blocks(ptr %a, i1 %u0) { +entry: + br label %bb0 -; @outermost_uniform_branch_more_blocks +bb0: ; bb0 (u) ; / \ ; bb2 (v) bb1 @@ -973,11 +1008,48 @@ exit: ; bb5 [phi] | ; \ / ; bb6 [phi] -; ; Similar to above, but with extra blocks on some edges. Probably doesn't ; require any extra handling but nice to test explicitly. + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb6 ] + br i1 %u0, label %bb2, label %bb1 + +bb1: + %add1 = add i64 %iv, 1 + br label %bb6 + +bb2: + %v2 = icmp sle i64 %iv, 2 + br i1 %v2, label %bb4, label %bb3 + +bb3: + %add3 = add i64 %iv, 3 + br label %bb5 + +bb4: + %add4 = add i64 %iv, 4 + br label %bb5 -; @uniform_branch_after_varying_branch +bb5: + %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ] + %add5 = add i64 %phi5, 5 + br label %bb6 + +bb6: + %phi6 = phi i64 [ %add1, %bb1 ], [ %add5, %bb5 ] + store i64 %phi6, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) { +entry: + br label %bb0 + +bb0: ; bb0 (v) ; / \ ; bb4 bb1 (u) @@ -987,8 +1059,43 @@ exit: ; | bb3 [phi] ; \ / ; bb5 [phi] + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ] + %v0 = icmp sle i64 %iv, 0 + br i1 %v0, label %bb4, label %bb1 + +bb1: + %add1 = add i64 %iv, 1 + br i1 %u1, label %bb2, label %bb3 + +bb2: + %add2 = add i64 %iv, 2 + br label %bb3 -; @uniform_branch_after_varying_branch_more_blocks +bb3: + %phi3 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ] + %add3 = add i64 %phi3, 3 + br label %bb5 + +bb4: + %add4 = add i64 %iv, 4 + br label %bb5 + +bb5: + %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ] + store i64 %phi5, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +define void @uniform_branch_after_varying_branch_more_blocks(ptr %a, i1 %u1) { +entry: + br label %bb0 + +bb0: ; bb0 (v) ; / \ ; bb5 bb1 (u) @@ -998,11 +1105,48 @@ exit: ; | bb4 [phi] ; \ / ; bb6 [phi] -; ; Similar to above, but with extra blocks on some edges. Probably doesn't ; require any extra handling but nice to test explicitly. + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb6 ] + %v0 = icmp sle i64 %iv, 0 + br i1 %v0, label %bb5, label %bb1 + +bb1: + br i1 %u1, label %bb3, label %bb2 + +bb2: + %add2 = add i64 %iv, 2 + br label %bb4 + +bb3: + %add3 = add i64 %iv, 3 + br label %bb4 + +bb4: + %phi4 = phi i64 [ %add2, %bb2 ], [ %add3, %bb3 ] + %add4 = add i64 %phi4, 4 + br label %bb6 + +bb5: + %add5 = add i64 %iv, 5 + br label %bb6 + +bb6: + %phi6 = phi i64 [ %add4, %bb4 ], [ %add5, %bb5 ] + store i64 %phi6, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +define void @uniform_branch_after_varying_branch_more_blocks_mirrored(ptr %a, i1 %u2) { +entry: + br label %bb0 -; @uniform_branch_after_varying_branch_more_blocks_mirrored +bb0: ; bb0 (v) ; / \ ; bb2 (u) bb1 @@ -1012,11 +1156,48 @@ exit: ; bb5 [phi] | ; \ / ; bb6 [phi] -; ; Mirror of the above test so that "(u)" block would be processed before/after ; the other "(v)" destination by the RPOT. + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb6 ] + %v0 = icmp sle i64 %iv, 0 + br i1 %v0, label %bb2, label %bb1 + +bb1: + %add1 = add i64 %iv, 1 + br label %bb6 + +bb2: + br i1 %u2, label %bb4, label %bb3 + +bb3: + %add3 = add i64 %iv, 3 + br label %bb5 + +bb4: + %add4 = add i64 %iv, 4 + br label %bb5 + +bb5: + %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ] + %add5 = add i64 %phi5, 5 + br label %bb6 -; @uniform_branch_on_masked_def +bb6: + %phi6 = phi i64 [ %add1, %bb1 ], [ %add5, %bb5 ] + store i64 %phi6, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +define void @uniform_branch_on_masked_def(ptr %a, ptr %uni.ptr1) { +entry: + br label %bb0 + +bb0: ; bb0 (v) ; / \ ; | bb1 (br i1 (load i1 %uni.ptr)) @@ -1026,17 +1207,50 @@ exit: ; | bb4 ; \ / ; bb5 [phi] -; ; bb1's condition is uniform, but its value is loaded only along one path of ; bb0's varying branch. It cannot be preserved trivially, because some the load ; may never be executed. + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ] + %add0 = add i64 %iv, 0 + %v0 = icmp sle i64 %iv, 0 + br i1 %v0, label %bb5, label %bb1 +bb1: + %cu1 = load i1, ptr %uni.ptr1 + br i1 %cu1, label %bb3, label %bb2 + +bb2: + %add2 = add i64 %iv, 2 + br label %bb4 + +bb3: + %add3 = add i64 %iv, 3 + br label %bb4 +bb4: + %phi4 = phi i64 [ %add2, %bb2 ], [ %add3, %bb3 ] + %add4 = add i64 %phi4, 4 + br label %bb5 + +bb5: + %phi5 = phi i64 [ %add0, %bb0 ], [ %add4, %bb4 ] + store i64 %phi5, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +define void @uniform_branch_unstructured_merge1(ptr %a, i1 %u0) { +entry: + br label %bb0 + +bb0: ; All @uniform_branch_unstructured_merge[0-4] below have the same structure of uniform ; control flow merging into the middle of the varying diamond, but "covering" ; different potential RPOT traversals. - -; @uniform_branch_unstructured_merge1 ; bb0 (u) ; / \ ; bb2 (v) bb1 @@ -1044,9 +1258,43 @@ exit: ; bb4 bb3 [phi] ; \ / ; bb5 [phi] -; + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ] + br i1 %u0, label %bb2, label %bb1 + +bb1: + %add1 = add i64 %iv, 1 + br label %bb3 + +bb2: + %add2 = add i64 %iv, 2 + %v2 = icmp sle i64 %iv, 2 + br i1 %v2, label %bb4, label %bb3 -; @uniform_branch_unstructured_merge2 +bb3: + %phi3 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ] + %add3 = add i64 %phi3, 3 + br label %bb5 + +bb4: + %add4 = add i64 %iv, 4 + br label %bb5 + +bb5: + %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ] + store i64 %phi5, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +define void @uniform_branch_unstructured_merge2(ptr %a, i1 %u0) { +entry: + br label %bb0 + +bb0: ; bb0 (u) ; / \ ; bb2 (v) bb1 @@ -1056,9 +1304,43 @@ exit: ; bb4 [phi] bb3 ; \ / ; bb5 [phi] -; + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ] + br i1 %u0, label %bb2, label %bb1 + +bb1: + %add1 = add i64 %iv, 1 + br label %bb4 + +bb2: + %add2 = add i64 %iv, 2 + %v2 = icmp sle i64 %iv, 2 + br i1 %v2, label %bb4, label %bb3 + +bb3: + %add3 = add i64 %iv, 3 + br label %bb5 + +bb4: + %phi4 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ] + %add4 = add i64 %phi4, 4 + br label %bb5 + +bb5: + %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ] + store i64 %phi5, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 -; @uniform_branch_unstructured_merge3 +exit: + ret void +} + +define void @uniform_branch_unstructured_merge3(ptr %a, i1 %u0) { +entry: + br label %bb0 + +bb0: ; bb0 (u) ; / \ ; bb3 bb1 (v) @@ -1066,9 +1348,43 @@ exit: ; bb4 [phi] bb2 ; \ / ; bb5 [phi] -; + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ] + br i1 %u0, label %bb3, label %bb1 + +bb1: + %add1 = add i64 %iv, 1 + %v1 = icmp sle i64 %iv, 1 + br i1 %v1, label %bb4, label %bb2 + +bb2: + %add2 = add i64 %iv, 2 + br label %bb5 + +bb3: + %add3 = add i64 %iv, 3 + br label %bb4 + +bb4: + %phi4 = phi i64 [ %add1, %bb1 ], [ %add3, %bb3 ] + %add4 = add i64 %phi4, 4 + br label %bb5 -; @uniform_branch_unstructured_merge4 +bb5: + %phi5 = phi i64 [ %add2, %bb2 ], [ %add4, %bb4 ] + store i64 %phi5, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +define void @uniform_branch_unstructured_merge4(ptr %a, i1 %u0) { +entry: + br label %bb0 + +bb0: ; bb0 (u) ; / \ ; bb3 bb1 (v) @@ -1078,10 +1394,43 @@ exit: ; bb4 bb2 [phi] ; \ / ; bb5 [phi] + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ] + br i1 %u0, label %bb3, label %bb1 + +bb1: + %add1 = add i64 %iv, 1 + %v1 = icmp sle i64 %iv, 1 + br i1 %v1, label %bb4, label %bb2 + +bb2: + %phi2 = phi i64 [ %add1, %bb1 ], [ %add3, %bb3 ] + %add2 = add i64 %phi2, 2 + br label %bb5 + +bb3: + %add3 = add i64 %iv, 3 + br label %bb2 + +bb4: + %add4 = add i64 %iv, 4 + br label %bb5 + +bb5: + %phi5 = phi i64 [ %add2, %bb2 ], [ %add4, %bb4 ] + store i64 %phi5, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 +exit: + ret void +} +define void @uniform_branch_shared_join_with_varying(ptr %a, i1 %u0) { +entry: + br label %bb0 -; @uniform_branch_shared_join_with_varying +bb0: ; bb0 (u) ; / \ ; bb2 (v) bb1 @@ -1089,10 +1438,39 @@ exit: ; bb3 | / ; \ | / ; bb4 [phi] -; ; Theoretically can be done, but would require a combination of a blend and a phi. + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ] + br i1 %u0, label %bb2, label %bb1 + +bb1: + %add1 = add i64 %iv, 1 + br label %bb4 + +bb2: + %add2 = add i64 %iv, 2 + %v2 = icmp sle i64 %iv, 2 + br i1 %v2, label %bb3, label %bb4 + +bb3: + %add3 = add i64 %iv, 3 + br label %bb4 + +bb4: + %phi4 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ], [ %add3, %bb3 ] + store i64 %phi4, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +define void @unstructured_uniform_only(ptr %a, i1 %u0, i1 %u2) { +entry: + br label %bb0 -; @unstructured_uniform_only +bb0: ; bb0 (u) ; / \ ; bb2 (u) bb1 @@ -1100,10 +1478,43 @@ exit: ; bb4 bb3 [phi] ; \ / ; bb5 [phi] -; ; Triviallly preservable, but detection might not be easy. + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ] + br i1 %u0, label %bb2, label %bb1 + +bb1: + %add1 = add i64 %iv, 1 + br label %bb3 + +bb2: + %add2 = add i64 %iv, 2 + br i1 %u2, label %bb4, label %bb3 + +bb3: + %phi3 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ] + %add3 = add i64 %phi3, 3 + br label %bb5 -; @unstructured_uniform_only_sese_region +bb4: + %add4 = add i64 %iv, 4 + br label %bb5 + +bb5: + %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ] + store i64 %phi5, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +define void @unstructured_uniform_only_sese_region(ptr %a, i1 %u1, i1 %u3) { +entry: + br label %bb0 + +bb0: ; bb0 (v) ; / \ ; bb6 bb1 (u) @@ -1116,4 +1527,46 @@ exit: ; \ / ; \ / ; bb8 [phi] -; + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb8 ] + %v0 = icmp sle i64 %iv, 0 + br i1 %v0, label %bb6, label %bb1 + +bb1: + br i1 %u1, label %bb3, label %bb2 + +bb2: + %add2 = add i64 %iv, 2 + br label %bb4 + +bb3: + %add3 = add i64 %iv, 3 + br i1 %u3, label %bb5, label %bb4 + +bb4: + %phi4 = phi i64 [ %add2, %bb2 ], [ %add3, %bb3 ] + %add4 = add i64 %phi4, 4 + br label %bb7 + +bb5: + %add5 = add i64 %iv, 5 + br label %bb7 + +bb6: + %add6 = add i64 %iv, 6 + br label %bb8 + +bb7: + %phi7 = phi i64 [ %add4, %bb4 ], [ %add5, %bb5 ] + %add7 = add i64 %phi7, 7 + br label %bb8 + +bb8: + %phi8 = phi i64 [ %add6, %bb6 ], [ %add7, %bb7 ] + store i64 %phi8, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} >From cdfc6cb7ac5143de79a0754549ba9fa892346cf3 Mon Sep 17 00:00:00 2001 From: Andrei Elovikov <[email protected]> Date: Wed, 26 Aug 2026 12:16:07 -0700 Subject: [PATCH 3/3] Copy to LoopVectorize/predicator.ll and generate CHECKs in both --- .../LoopVectorize/VPlan/predicator.ll | 546 +++++++++ .../Transforms/LoopVectorize/predicator.ll | 1074 +++++++++++++++++ 2 files changed, 1620 insertions(+) diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll index 1c95c9381ab73..74607c8c4c256 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll @@ -951,6 +951,40 @@ exit: } define void @outermost_uniform_branch(ptr %a, i1 %u0) { +; CHECK-LABEL: VPlan for loop in 'outermost_uniform_branch' +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> +; CHECK-NEXT: Successor(s): bb1 +; CHECK-EMPTY: +; CHECK-NEXT: bb1: +; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, ir<%u0> (!vplan.execution.frequency 9223372036854775807 (50%, estimated)) +; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1>, ir<%u0> (!vplan.execution.frequency 9223372036854775807 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb2 +; CHECK-EMPTY: +; CHECK-NEXT: bb2: +; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = logical-and ir<%u0>, ir<%v1> +; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP4]]> (!vplan.execution.frequency 4611686018427387903 (25%, estimated)) +; CHECK-NEXT: Successor(s): bb3 +; CHECK-EMPTY: +; CHECK-NEXT: bb3: +; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/ir<true> ir<%add2>/ir<%v1> +; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>, ir<%u0> (!vplan.execution.frequency 9223372036854775807 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb4 +; CHECK-EMPTY: +; CHECK-NEXT: bb4: +; CHECK-NEXT: BLEND ir<%phi4> = ir<%iv>/ir<true> ir<%add3>/ir<%u0> +; CHECK-NEXT: EMIT store ir<%phi4>, ir<%a> +; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1> +; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; entry: br label %bb0 @@ -995,6 +1029,50 @@ exit: } define void @outermost_uniform_branch_more_blocks(ptr %a, i1 %u0) { +; CHECK-LABEL: VPlan for loop in 'outermost_uniform_branch_more_blocks' +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> +; CHECK-NEXT: Successor(s): bb1 +; CHECK-EMPTY: +; CHECK-NEXT: bb1: +; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0> +; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb2 +; CHECK-EMPTY: +; CHECK-NEXT: bb2: +; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>, ir<%u0> (!vplan.execution.frequency 9223372036854775808 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb3 +; CHECK-EMPTY: +; CHECK-NEXT: bb3: +; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%v2> +; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%u0>, vp<[[VP5]]> +; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP6]]> (!vplan.execution.frequency 4611686018427387904 (25%, estimated)) +; CHECK-NEXT: Successor(s): bb4 +; CHECK-EMPTY: +; CHECK-NEXT: bb4: +; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%u0>, ir<%v2> +; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP7]]> (!vplan.execution.frequency 4611686018427387904 (25%, estimated)) +; CHECK-NEXT: Successor(s): bb5 +; CHECK-EMPTY: +; CHECK-NEXT: bb5: +; CHECK-NEXT: BLEND ir<%phi5> = ir<%add3>/vp<[[VP5]]> ir<%add4>/ir<%v2> +; CHECK-NEXT: EMIT ir<%add5> = add ir<%phi5>, ir<5>, ir<%u0> (!vplan.execution.frequency 9223372036854775808 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb6 +; CHECK-EMPTY: +; CHECK-NEXT: bb6: +; CHECK-NEXT: BLEND ir<%phi6> = ir<%add1>/vp<[[VP4]]> ir<%add5>/ir<%u0> +; CHECK-NEXT: EMIT store ir<%phi6>, ir<%a> +; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1> +; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; entry: br label %bb0 @@ -1046,6 +1124,45 @@ exit: } define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) { +; CHECK-LABEL: VPlan for loop in 'uniform_branch_after_varying_branch' +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> +; CHECK-NEXT: EMIT ir<%v0> = icmp sle ir<%iv>, ir<0> +; CHECK-NEXT: Successor(s): bb1 +; CHECK-EMPTY: +; CHECK-NEXT: bb1: +; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v0> +; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb2 +; CHECK-EMPTY: +; CHECK-NEXT: bb2: +; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%u1> +; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP5]]> (!vplan.execution.frequency 4611686018427387903 (25%, estimated)) +; CHECK-NEXT: Successor(s): bb3 +; CHECK-EMPTY: +; CHECK-NEXT: bb3: +; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/ir<true> ir<%add2>/ir<%u1> +; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>, vp<[[VP4]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb4 +; CHECK-EMPTY: +; CHECK-NEXT: bb4: +; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, ir<%v0> (!vplan.execution.frequency 9223372036854775808 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb5 +; CHECK-EMPTY: +; CHECK-NEXT: bb5: +; CHECK-NEXT: BLEND ir<%phi5> = ir<%add3>/vp<[[VP4]]> ir<%add4>/ir<%v0> +; CHECK-NEXT: EMIT store ir<%phi5>, ir<%a> +; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1> +; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; entry: br label %bb0 @@ -1092,6 +1209,50 @@ exit: } define void @uniform_branch_after_varying_branch_more_blocks(ptr %a, i1 %u1) { +; CHECK-LABEL: VPlan for loop in 'uniform_branch_after_varying_branch_more_blocks' +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> +; CHECK-NEXT: EMIT ir<%v0> = icmp sle ir<%iv>, ir<0> +; CHECK-NEXT: Successor(s): bb1 +; CHECK-EMPTY: +; CHECK-NEXT: bb1: +; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v0> +; CHECK-NEXT: Successor(s): bb2 +; CHECK-EMPTY: +; CHECK-NEXT: bb2: +; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%u1> +; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]> +; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP6]]> (!vplan.execution.frequency 4611686018427387903 (25%, estimated)) +; CHECK-NEXT: Successor(s): bb3 +; CHECK-EMPTY: +; CHECK-NEXT: bb3: +; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%u1> +; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP7]]> (!vplan.execution.frequency 4611686018427387904 (25%, estimated)) +; CHECK-NEXT: Successor(s): bb4 +; CHECK-EMPTY: +; CHECK-NEXT: bb4: +; CHECK-NEXT: BLEND ir<%phi4> = ir<%add2>/vp<[[VP5]]> ir<%add3>/ir<%u1> +; CHECK-NEXT: EMIT ir<%add4> = add ir<%phi4>, ir<4>, vp<[[VP4]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb5 +; CHECK-EMPTY: +; CHECK-NEXT: bb5: +; CHECK-NEXT: EMIT ir<%add5> = add ir<%iv>, ir<5>, ir<%v0> (!vplan.execution.frequency 9223372036854775808 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb6 +; CHECK-EMPTY: +; CHECK-NEXT: bb6: +; CHECK-NEXT: BLEND ir<%phi6> = ir<%add4>/vp<[[VP4]]> ir<%add5>/ir<%v0> +; CHECK-NEXT: EMIT store ir<%phi6>, ir<%a> +; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1> +; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; entry: br label %bb0 @@ -1143,6 +1304,50 @@ exit: } define void @uniform_branch_after_varying_branch_more_blocks_mirrored(ptr %a, i1 %u2) { +; CHECK-LABEL: VPlan for loop in 'uniform_branch_after_varying_branch_more_blocks_mirrored' +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> +; CHECK-NEXT: EMIT ir<%v0> = icmp sle ir<%iv>, ir<0> +; CHECK-NEXT: Successor(s): bb1 +; CHECK-EMPTY: +; CHECK-NEXT: bb1: +; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v0> +; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb2 +; CHECK-EMPTY: +; CHECK-NEXT: bb2: +; CHECK-NEXT: Successor(s): bb3 +; CHECK-EMPTY: +; CHECK-NEXT: bb3: +; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%u2> +; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%v0>, vp<[[VP5]]> +; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP6]]> (!vplan.execution.frequency 4611686018427387904 (25%, estimated)) +; CHECK-NEXT: Successor(s): bb4 +; CHECK-EMPTY: +; CHECK-NEXT: bb4: +; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%v0>, ir<%u2> +; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP7]]> (!vplan.execution.frequency 4611686018427387904 (25%, estimated)) +; CHECK-NEXT: Successor(s): bb5 +; CHECK-EMPTY: +; CHECK-NEXT: bb5: +; CHECK-NEXT: BLEND ir<%phi5> = ir<%add3>/vp<[[VP5]]> ir<%add4>/ir<%u2> +; CHECK-NEXT: EMIT ir<%add5> = add ir<%phi5>, ir<5>, ir<%v0> (!vplan.execution.frequency 9223372036854775808 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb6 +; CHECK-EMPTY: +; CHECK-NEXT: bb6: +; CHECK-NEXT: BLEND ir<%phi6> = ir<%add1>/vp<[[VP4]]> ir<%add5>/ir<%v0> +; CHECK-NEXT: EMIT store ir<%phi6>, ir<%a> +; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1> +; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; entry: br label %bb0 @@ -1194,6 +1399,47 @@ exit: } define void @uniform_branch_on_masked_def(ptr %a, ptr %uni.ptr1) { +; CHECK-LABEL: VPlan for loop in 'uniform_branch_on_masked_def' +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> +; CHECK-NEXT: EMIT ir<%v0> = icmp sle ir<%iv>, ir<0> +; CHECK-NEXT: Successor(s): bb1 +; CHECK-EMPTY: +; CHECK-NEXT: bb1: +; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v0> +; CHECK-NEXT: EMIT-SCALAR ir<%cu1> = load ir<%uni.ptr1>, vp<[[VP4]]> (!alias.scope !17, !vplan.execution.frequency 9223372036854775807 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb2 +; CHECK-EMPTY: +; CHECK-NEXT: bb2: +; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%cu1> +; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]> +; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP6]]> (!vplan.execution.frequency 4611686018427387903 (25%, estimated)) +; CHECK-NEXT: Successor(s): bb3 +; CHECK-EMPTY: +; CHECK-NEXT: bb3: +; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%cu1> +; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP7]]> (!vplan.execution.frequency 4611686018427387904 (25%, estimated)) +; CHECK-NEXT: Successor(s): bb4 +; CHECK-EMPTY: +; CHECK-NEXT: bb4: +; CHECK-NEXT: BLEND ir<%phi4> = ir<%add2>/vp<[[VP5]]> ir<%add3>/ir<%cu1> +; CHECK-NEXT: EMIT ir<%add4> = add ir<%phi4>, ir<4>, vp<[[VP4]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb5 +; CHECK-EMPTY: +; CHECK-NEXT: bb5: +; CHECK-NEXT: BLEND ir<%phi5> = ir<%iv>/ir<true> ir<%add4>/vp<[[VP4]]> +; CHECK-NEXT: EMIT store ir<%phi5>, ir<%a> (!alias.scope !18, !noalias !17) +; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1> +; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; entry: br label %bb0 @@ -1244,6 +1490,48 @@ exit: } define void @uniform_branch_unstructured_merge1(ptr %a, i1 %u0) { +; CHECK-LABEL: VPlan for loop in 'uniform_branch_unstructured_merge1' +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> +; CHECK-NEXT: Successor(s): bb1 +; CHECK-EMPTY: +; CHECK-NEXT: bb1: +; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0> +; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb2 +; CHECK-EMPTY: +; CHECK-NEXT: bb2: +; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%u0> (!vplan.execution.frequency 9223372036854775808 (50%, estimated)) +; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>, ir<%u0> (!vplan.execution.frequency 9223372036854775808 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb4 +; CHECK-EMPTY: +; CHECK-NEXT: bb4: +; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%u0>, ir<%v2> +; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP5]]> (!vplan.execution.frequency 4611686018427387904 (25%, estimated)) +; CHECK-NEXT: Successor(s): bb3 +; CHECK-EMPTY: +; CHECK-NEXT: bb3: +; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = not ir<%v2> +; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%u0>, vp<[[VP6]]> +; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = or vp<[[VP7]]>, vp<[[VP4]]> +; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/vp<[[VP4]]> ir<%add2>/ir<%u0> +; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>, vp<[[VP8]]> (!vplan.execution.frequency 13835058055282163711 (75%, estimated)) +; CHECK-NEXT: Successor(s): bb5 +; CHECK-EMPTY: +; CHECK-NEXT: bb5: +; CHECK-NEXT: BLEND ir<%phi5> = ir<%add4>/vp<[[VP5]]> ir<%add3>/vp<[[VP8]]> +; CHECK-NEXT: EMIT store ir<%phi5>, ir<%a> +; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1> +; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; entry: br label %bb0 @@ -1291,6 +1579,48 @@ exit: } define void @uniform_branch_unstructured_merge2(ptr %a, i1 %u0) { +; CHECK-LABEL: VPlan for loop in 'uniform_branch_unstructured_merge2' +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> +; CHECK-NEXT: Successor(s): bb1 +; CHECK-EMPTY: +; CHECK-NEXT: bb1: +; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0> +; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb2 +; CHECK-EMPTY: +; CHECK-NEXT: bb2: +; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%u0> (!vplan.execution.frequency 9223372036854775808 (50%, estimated)) +; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>, ir<%u0> (!vplan.execution.frequency 9223372036854775808 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb3 +; CHECK-EMPTY: +; CHECK-NEXT: bb3: +; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%v2> +; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%u0>, vp<[[VP5]]> +; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP6]]> (!vplan.execution.frequency 4611686018427387904 (25%, estimated)) +; CHECK-NEXT: Successor(s): bb4 +; CHECK-EMPTY: +; CHECK-NEXT: bb4: +; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%u0>, ir<%v2> +; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = or vp<[[VP7]]>, vp<[[VP4]]> +; CHECK-NEXT: BLEND ir<%phi4> = ir<%add1>/vp<[[VP4]]> ir<%add2>/ir<%u0> +; CHECK-NEXT: EMIT ir<%add4> = add ir<%phi4>, ir<4>, vp<[[VP8]]> (!vplan.execution.frequency 13835058055282163711 (75%, estimated)) +; CHECK-NEXT: Successor(s): bb5 +; CHECK-EMPTY: +; CHECK-NEXT: bb5: +; CHECK-NEXT: BLEND ir<%phi5> = ir<%add3>/vp<[[VP6]]> ir<%add4>/vp<[[VP8]]> +; CHECK-NEXT: EMIT store ir<%phi5>, ir<%a> +; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1> +; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; entry: br label %bb0 @@ -1337,6 +1667,48 @@ exit: } define void @uniform_branch_unstructured_merge3(ptr %a, i1 %u0) { +; CHECK-LABEL: VPlan for loop in 'uniform_branch_unstructured_merge3' +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> +; CHECK-NEXT: Successor(s): bb1 +; CHECK-EMPTY: +; CHECK-NEXT: bb1: +; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0> +; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated)) +; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1>, vp<[[VP4]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb2 +; CHECK-EMPTY: +; CHECK-NEXT: bb2: +; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%v1> +; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]> +; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP6]]> (!vplan.execution.frequency 4611686018427387903 (25%, estimated)) +; CHECK-NEXT: Successor(s): bb3 +; CHECK-EMPTY: +; CHECK-NEXT: bb3: +; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, ir<%u0> (!vplan.execution.frequency 9223372036854775808 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb4 +; CHECK-EMPTY: +; CHECK-NEXT: bb4: +; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%v1> +; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = or ir<%u0>, vp<[[VP7]]> +; CHECK-NEXT: BLEND ir<%phi4> = ir<%add1>/vp<[[VP4]]> ir<%add3>/ir<%u0> +; CHECK-NEXT: EMIT ir<%add4> = add ir<%phi4>, ir<4>, vp<[[VP8]]> (!vplan.execution.frequency 13835058055282163712 (75%, estimated)) +; CHECK-NEXT: Successor(s): bb5 +; CHECK-EMPTY: +; CHECK-NEXT: bb5: +; CHECK-NEXT: BLEND ir<%phi5> = ir<%add2>/vp<[[VP6]]> ir<%add4>/vp<[[VP8]]> +; CHECK-NEXT: EMIT store ir<%phi5>, ir<%a> +; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1> +; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; entry: br label %bb0 @@ -1381,6 +1753,48 @@ exit: } define void @uniform_branch_unstructured_merge4(ptr %a, i1 %u0) { +; CHECK-LABEL: VPlan for loop in 'uniform_branch_unstructured_merge4' +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> +; CHECK-NEXT: Successor(s): bb1 +; CHECK-EMPTY: +; CHECK-NEXT: bb1: +; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0> +; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated)) +; CHECK-NEXT: EMIT ir<%v1> = icmp sle ir<%iv>, ir<1>, vp<[[VP4]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb4 +; CHECK-EMPTY: +; CHECK-NEXT: bb4: +; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%v1> +; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP5]]> (!vplan.execution.frequency 4611686018427387903 (25%, estimated)) +; CHECK-NEXT: Successor(s): bb3 +; CHECK-EMPTY: +; CHECK-NEXT: bb3: +; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, ir<%u0> (!vplan.execution.frequency 9223372036854775808 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb2 +; CHECK-EMPTY: +; CHECK-NEXT: bb2: +; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = not ir<%v1> +; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP6]]> +; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = or ir<%u0>, vp<[[VP7]]> +; CHECK-NEXT: BLEND ir<%phi2> = ir<%add1>/vp<[[VP4]]> ir<%add3>/ir<%u0> +; CHECK-NEXT: EMIT ir<%add2> = add ir<%phi2>, ir<2>, vp<[[VP8]]> (!vplan.execution.frequency 13835058055282163712 (75%, estimated)) +; CHECK-NEXT: Successor(s): bb5 +; CHECK-EMPTY: +; CHECK-NEXT: bb5: +; CHECK-NEXT: BLEND ir<%phi5> = ir<%add4>/vp<[[VP5]]> ir<%add2>/vp<[[VP8]]> +; CHECK-NEXT: EMIT store ir<%phi5>, ir<%a> +; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1> +; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; entry: br label %bb0 @@ -1427,6 +1841,40 @@ exit: } define void @uniform_branch_shared_join_with_varying(ptr %a, i1 %u0) { +; CHECK-LABEL: VPlan for loop in 'uniform_branch_shared_join_with_varying' +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> +; CHECK-NEXT: Successor(s): bb1 +; CHECK-EMPTY: +; CHECK-NEXT: bb1: +; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0> +; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb2 +; CHECK-EMPTY: +; CHECK-NEXT: bb2: +; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%u0> (!vplan.execution.frequency 9223372036854775808 (50%, estimated)) +; CHECK-NEXT: EMIT ir<%v2> = icmp sle ir<%iv>, ir<2>, ir<%u0> (!vplan.execution.frequency 9223372036854775808 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb3 +; CHECK-EMPTY: +; CHECK-NEXT: bb3: +; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%u0>, ir<%v2> +; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP5]]> (!vplan.execution.frequency 4611686018427387904 (25%, estimated)) +; CHECK-NEXT: Successor(s): bb4 +; CHECK-EMPTY: +; CHECK-NEXT: bb4: +; CHECK-NEXT: BLEND ir<%phi4> = ir<%add1>/vp<[[VP4]]> ir<%add2>/ir<%u0> ir<%add3>/vp<[[VP5]]> +; CHECK-NEXT: EMIT store ir<%phi4>, ir<%a> +; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1> +; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; entry: br label %bb0 @@ -1467,6 +1915,47 @@ exit: } define void @unstructured_uniform_only(ptr %a, i1 %u0, i1 %u2) { +; CHECK-LABEL: VPlan for loop in 'unstructured_uniform_only' +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> +; CHECK-NEXT: Successor(s): bb1 +; CHECK-EMPTY: +; CHECK-NEXT: bb1: +; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%u0> +; CHECK-NEXT: EMIT ir<%add1> = add ir<%iv>, ir<1>, vp<[[VP4]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb2 +; CHECK-EMPTY: +; CHECK-NEXT: bb2: +; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, ir<%u0> (!vplan.execution.frequency 9223372036854775808 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb4 +; CHECK-EMPTY: +; CHECK-NEXT: bb4: +; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and ir<%u0>, ir<%u2> +; CHECK-NEXT: EMIT ir<%add4> = add ir<%iv>, ir<4>, vp<[[VP5]]> (!vplan.execution.frequency 4611686018427387904 (25%, estimated)) +; CHECK-NEXT: Successor(s): bb3 +; CHECK-EMPTY: +; CHECK-NEXT: bb3: +; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = not ir<%u2> +; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and ir<%u0>, vp<[[VP6]]> +; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = or vp<[[VP7]]>, vp<[[VP4]]> +; CHECK-NEXT: BLEND ir<%phi3> = ir<%add1>/vp<[[VP4]]> ir<%add2>/ir<%u0> +; CHECK-NEXT: EMIT ir<%add3> = add ir<%phi3>, ir<3>, vp<[[VP8]]> (!vplan.execution.frequency 13835058055282163711 (75%, estimated)) +; CHECK-NEXT: Successor(s): bb5 +; CHECK-EMPTY: +; CHECK-NEXT: bb5: +; CHECK-NEXT: BLEND ir<%phi5> = ir<%add4>/vp<[[VP5]]> ir<%add3>/vp<[[VP8]]> +; CHECK-NEXT: EMIT store ir<%phi5>, ir<%a> +; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1> +; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; entry: br label %bb0 @@ -1511,6 +2000,63 @@ exit: } define void @unstructured_uniform_only_sese_region(ptr %a, i1 %u1, i1 %u3) { +; CHECK-LABEL: VPlan for loop in 'unstructured_uniform_only_sese_region' +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> +; CHECK-NEXT: EMIT ir<%v0> = icmp sle ir<%iv>, ir<0> +; CHECK-NEXT: Successor(s): bb1 +; CHECK-EMPTY: +; CHECK-NEXT: bb1: +; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%v0> +; CHECK-NEXT: Successor(s): bb2 +; CHECK-EMPTY: +; CHECK-NEXT: bb2: +; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%u1> +; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]> +; CHECK-NEXT: EMIT ir<%add2> = add ir<%iv>, ir<2>, vp<[[VP6]]> (!vplan.execution.frequency 4611686018427387903 (25%, estimated)) +; CHECK-NEXT: Successor(s): bb3 +; CHECK-EMPTY: +; CHECK-NEXT: bb3: +; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%u1> +; CHECK-NEXT: EMIT ir<%add3> = add ir<%iv>, ir<3>, vp<[[VP7]]> (!vplan.execution.frequency 4611686018427387904 (25%, estimated)) +; CHECK-NEXT: Successor(s): bb5 +; CHECK-EMPTY: +; CHECK-NEXT: bb5: +; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP7]]>, ir<%u3> +; CHECK-NEXT: EMIT ir<%add5> = add ir<%iv>, ir<5>, vp<[[VP8]]> (!vplan.execution.frequency 2305843009213693952 (12.5%, estimated)) +; CHECK-NEXT: Successor(s): bb4 +; CHECK-EMPTY: +; CHECK-NEXT: bb4: +; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = not ir<%u3> +; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = logical-and vp<[[VP7]]>, vp<[[VP9]]> +; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = or vp<[[VP10]]>, vp<[[VP6]]> +; CHECK-NEXT: BLEND ir<%phi4> = ir<%add2>/vp<[[VP5]]> ir<%add3>/ir<%u1> +; CHECK-NEXT: EMIT ir<%add4> = add ir<%phi4>, ir<4>, vp<[[VP11]]> (!vplan.execution.frequency 6917529027641081855 (37.5%, estimated)) +; CHECK-NEXT: Successor(s): bb7 +; CHECK-EMPTY: +; CHECK-NEXT: bb7: +; CHECK-NEXT: BLEND ir<%phi7> = ir<%add5>/vp<[[VP8]]> ir<%add4>/vp<[[VP11]]> +; CHECK-NEXT: EMIT ir<%add7> = add ir<%phi7>, ir<7>, vp<[[VP4]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb6 +; CHECK-EMPTY: +; CHECK-NEXT: bb6: +; CHECK-NEXT: EMIT ir<%add6> = add ir<%iv>, ir<6>, ir<%v0> (!vplan.execution.frequency 9223372036854775808 (50%, estimated)) +; CHECK-NEXT: Successor(s): bb8 +; CHECK-EMPTY: +; CHECK-NEXT: bb8: +; CHECK-NEXT: BLEND ir<%phi8> = ir<%add7>/vp<[[VP4]]> ir<%add6>/ir<%v0> +; CHECK-NEXT: EMIT store ir<%phi8>, ir<%a> +; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1> +; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; entry: br label %bb0 diff --git a/llvm/test/Transforms/LoopVectorize/predicator.ll b/llvm/test/Transforms/LoopVectorize/predicator.ll index e3c9dcdf40cc9..8fc53dbda481d 100644 --- a/llvm/test/Transforms/LoopVectorize/predicator.ll +++ b/llvm/test/Transforms/LoopVectorize/predicator.ll @@ -294,3 +294,1077 @@ latch: exit: ret void } + +define void @outermost_uniform_branch(ptr %a, i1 %u0) { +; CHECK-LABEL: define void @outermost_uniform_branch( +; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128 +; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1) +; CHECK-NEXT: [[TMP2:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 1) +; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2) +; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> [[TMP3]], <4 x i64> [[TMP1]] +; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 3) +; CHECK-NEXT: [[PREDPHI1:%.*]] = select i1 [[U0]], <4 x i64> [[TMP4]], <4 x i64> [[VEC_IND]] +; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3 +; CHECK-NEXT: store i64 [[TMP5]], ptr [[A]], align 4 +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %bb0 + +bb0: +; bb0 (u) +; / \ +; bb1 (v) \ +; / | | +; bb2 | | +; \ | | +; bb3 [phi] | +; \ / +; bb4 [phi] +; bb0's uniform branch can be easily preserved. + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ] + %add0 = add i64 %iv, 0 + br i1 %u0, label %bb1, label %bb4 + +bb1: + %add1 = add i64 %iv, 1 + %v1 = icmp sle i64 %iv, 1 + br i1 %v1, label %bb2, label %bb3 + +bb2: + %add2 = add i64 %iv, 2 + br label %bb3 + +bb3: + %phi3 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ] + %add3 = add i64 %phi3, 3 + br label %bb4 + +bb4: + %phi4 = phi i64 [ %add3, %bb3 ], [ %add0, %bb0 ] + store i64 %phi4, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +define void @outermost_uniform_branch_more_blocks(ptr %a, i1 %u0) { +; CHECK-LABEL: define void @outermost_uniform_branch_more_blocks( +; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128 +; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1) +; CHECK-NEXT: [[TMP2:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 2) +; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3) +; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> [[TMP4]], <4 x i64> [[TMP3]] +; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 5) +; CHECK-NEXT: [[PREDPHI1:%.*]] = select i1 [[U0]], <4 x i64> [[TMP5]], <4 x i64> [[TMP1]] +; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3 +; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4 +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %bb0 + +bb0: +; bb0 (u) +; / \ +; bb2 (v) bb1 +; / \ | +; bb4 bb3 | +; \ / | +; bb5 [phi] | +; \ / +; bb6 [phi] +; Similar to above, but with extra blocks on some edges. Probably doesn't +; require any extra handling but nice to test explicitly. + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb6 ] + br i1 %u0, label %bb2, label %bb1 + +bb1: + %add1 = add i64 %iv, 1 + br label %bb6 + +bb2: + %v2 = icmp sle i64 %iv, 2 + br i1 %v2, label %bb4, label %bb3 + +bb3: + %add3 = add i64 %iv, 3 + br label %bb5 + +bb4: + %add4 = add i64 %iv, 4 + br label %bb5 + +bb5: + %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ] + %add5 = add i64 %phi5, 5 + br label %bb6 + +bb6: + %phi6 = phi i64 [ %add1, %bb1 ], [ %add5, %bb5 ] + store i64 %phi6, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +define void @uniform_branch_after_varying_branch(ptr %a, i1 %u1) { +; CHECK-LABEL: define void @uniform_branch_after_varying_branch( +; CHECK-SAME: ptr [[A:%.*]], i1 [[U1:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128 +; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP1:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1) +; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2) +; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U1]], <4 x i64> [[TMP3]], <4 x i64> [[TMP2]] +; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 3) +; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP5]], <4 x i64> [[TMP4]] +; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3 +; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4 +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %bb0 + +bb0: +; bb0 (v) +; / \ +; bb4 bb1 (u) +; | / | +; | bb2 | +; | \ | +; | bb3 [phi] +; \ / +; bb5 [phi] + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ] + %v0 = icmp sle i64 %iv, 0 + br i1 %v0, label %bb4, label %bb1 + +bb1: + %add1 = add i64 %iv, 1 + br i1 %u1, label %bb2, label %bb3 + +bb2: + %add2 = add i64 %iv, 2 + br label %bb3 + +bb3: + %phi3 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ] + %add3 = add i64 %phi3, 3 + br label %bb5 + +bb4: + %add4 = add i64 %iv, 4 + br label %bb5 + +bb5: + %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ] + store i64 %phi5, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +define void @uniform_branch_after_varying_branch_more_blocks(ptr %a, i1 %u1) { +; CHECK-LABEL: define void @uniform_branch_after_varying_branch_more_blocks( +; CHECK-SAME: ptr [[A:%.*]], i1 [[U1:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128 +; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP1:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2) +; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3) +; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U1]], <4 x i64> [[TMP3]], <4 x i64> [[TMP2]] +; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 4) +; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 5) +; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP5]], <4 x i64> [[TMP4]] +; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3 +; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4 +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %bb0 + +bb0: +; bb0 (v) +; / \ +; bb5 bb1 (u) +; | / \ +; | bb3 bb2 +; | \ / +; | bb4 [phi] +; \ / +; bb6 [phi] +; Similar to above, but with extra blocks on some edges. Probably doesn't +; require any extra handling but nice to test explicitly. + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb6 ] + %v0 = icmp sle i64 %iv, 0 + br i1 %v0, label %bb5, label %bb1 + +bb1: + br i1 %u1, label %bb3, label %bb2 + +bb2: + %add2 = add i64 %iv, 2 + br label %bb4 + +bb3: + %add3 = add i64 %iv, 3 + br label %bb4 + +bb4: + %phi4 = phi i64 [ %add2, %bb2 ], [ %add3, %bb3 ] + %add4 = add i64 %phi4, 4 + br label %bb6 + +bb5: + %add5 = add i64 %iv, 5 + br label %bb6 + +bb6: + %phi6 = phi i64 [ %add4, %bb4 ], [ %add5, %bb5 ] + store i64 %phi6, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +define void @uniform_branch_after_varying_branch_more_blocks_mirrored(ptr %a, i1 %u2) { +; CHECK-LABEL: define void @uniform_branch_after_varying_branch_more_blocks_mirrored( +; CHECK-SAME: ptr [[A:%.*]], i1 [[U2:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128 +; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP1:%.*]] = icmp sle <4 x i64> [[VEC_IND]], zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1) +; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3) +; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U2]], <4 x i64> [[TMP4]], <4 x i64> [[TMP3]] +; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 5) +; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP5]], <4 x i64> [[TMP2]] +; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3 +; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4 +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %bb0 + +bb0: +; bb0 (v) +; / \ +; bb2 (u) bb1 +; / \ | +; bb4 bb3 | +; \ / | +; bb5 [phi] | +; \ / +; bb6 [phi] +; Mirror of the above test so that "(u)" block would be processed before/after +; the other "(v)" destination by the RPOT. + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb6 ] + %v0 = icmp sle i64 %iv, 0 + br i1 %v0, label %bb2, label %bb1 + +bb1: + %add1 = add i64 %iv, 1 + br label %bb6 + +bb2: + br i1 %u2, label %bb4, label %bb3 + +bb3: + %add3 = add i64 %iv, 3 + br label %bb5 + +bb4: + %add4 = add i64 %iv, 4 + br label %bb5 + +bb5: + %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ] + %add5 = add i64 %phi5, 5 + br label %bb6 + +bb6: + %phi6 = phi i64 [ %add1, %bb1 ], [ %add5, %bb5 ] + store i64 %phi6, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +define void @uniform_branch_on_masked_def(ptr %a, ptr %uni.ptr1) { +; CHECK-LABEL: define void @uniform_branch_on_masked_def( +; CHECK-SAME: ptr [[A:%.*]], ptr [[UNI_PTR1:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]] +; CHECK: [[VECTOR_MEMCHECK]]: +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 8 +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[UNI_PTR1]], i64 1 +; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]] +; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[UNI_PTR1]], [[SCEVGEP]] +; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] +; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE7:.*]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE7]] ] +; CHECK-NEXT: [[TMP0:%.*]] = icmp sgt <4 x i64> [[VEC_IND]], zeroinitializer +; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0 +; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]] +; CHECK: [[PRED_LOAD_IF]]: +; CHECK-NEXT: [[TMP2:%.*]] = load i1, ptr [[UNI_PTR1]], align 1, !alias.scope [[META11:![0-9]+]] +; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i1> poison, i1 [[TMP2]], i64 0 +; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]] +; CHECK: [[PRED_LOAD_CONTINUE]]: +; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i1> [ poison, %[[VECTOR_BODY]] ], [ [[TMP3]], %[[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP0]], i64 1 +; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_LOAD_IF2:.*]], label %[[PRED_LOAD_CONTINUE3:.*]] +; CHECK: [[PRED_LOAD_IF2]]: +; CHECK-NEXT: [[TMP6:%.*]] = load i1, ptr [[UNI_PTR1]], align 1, !alias.scope [[META11]] +; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i1> [[TMP4]], i1 [[TMP6]], i64 1 +; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE3]] +; CHECK: [[PRED_LOAD_CONTINUE3]]: +; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i1> [ [[TMP4]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP7]], %[[PRED_LOAD_IF2]] ] +; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP0]], i64 2 +; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_LOAD_IF4:.*]], label %[[PRED_LOAD_CONTINUE5:.*]] +; CHECK: [[PRED_LOAD_IF4]]: +; CHECK-NEXT: [[TMP10:%.*]] = load i1, ptr [[UNI_PTR1]], align 1, !alias.scope [[META11]] +; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i1> [[TMP8]], i1 [[TMP10]], i64 2 +; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE5]] +; CHECK: [[PRED_LOAD_CONTINUE5]]: +; CHECK-NEXT: [[TMP12:%.*]] = phi <4 x i1> [ [[TMP8]], %[[PRED_LOAD_CONTINUE3]] ], [ [[TMP11]], %[[PRED_LOAD_IF4]] ] +; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP0]], i64 3 +; CHECK-NEXT: br i1 [[TMP13]], label %[[PRED_LOAD_IF6:.*]], label %[[PRED_LOAD_CONTINUE7]] +; CHECK: [[PRED_LOAD_IF6]]: +; CHECK-NEXT: [[TMP14:%.*]] = load i1, ptr [[UNI_PTR1]], align 1, !alias.scope [[META11]] +; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i1> [[TMP12]], i1 [[TMP14]], i64 3 +; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE7]] +; CHECK: [[PRED_LOAD_CONTINUE7]]: +; CHECK-NEXT: [[TMP16:%.*]] = phi <4 x i1> [ [[TMP12]], %[[PRED_LOAD_CONTINUE5]] ], [ [[TMP15]], %[[PRED_LOAD_IF6]] ] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128 +; CHECK-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP18:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2) +; CHECK-NEXT: [[TMP19:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3) +; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP16]], <4 x i64> [[TMP19]], <4 x i64> [[TMP18]] +; CHECK-NEXT: [[TMP20:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 4) +; CHECK-NEXT: [[PREDPHI8:%.*]] = select <4 x i1> [[TMP0]], <4 x i64> [[TMP20]], <4 x i64> [[VEC_IND]] +; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i64> [[PREDPHI8]], i64 3 +; CHECK-NEXT: store i64 [[TMP21]], ptr [[A]], align 4, !alias.scope [[META15:![0-9]+]], !noalias [[META11]] +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: br label %[[BB0:.*]] +; CHECK: [[BB0]]: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[BB5:.*]] ] +; CHECK-NEXT: [[ADD0:%.*]] = add i64 [[IV]], 0 +; CHECK-NEXT: [[V0:%.*]] = icmp sle i64 [[IV]], 0 +; CHECK-NEXT: br i1 [[V0]], label %[[BB5]], label %[[BB1:.*]] +; CHECK: [[BB1]]: +; CHECK-NEXT: [[CU1:%.*]] = load i1, ptr [[UNI_PTR1]], align 1 +; CHECK-NEXT: br i1 [[CU1]], label %[[BB3:.*]], label %[[BB2:.*]] +; CHECK: [[BB2]]: +; CHECK-NEXT: [[ADD2:%.*]] = add i64 [[IV]], 2 +; CHECK-NEXT: br label %[[BB4:.*]] +; CHECK: [[BB3]]: +; CHECK-NEXT: [[ADD3:%.*]] = add i64 [[IV]], 3 +; CHECK-NEXT: br label %[[BB4]] +; CHECK: [[BB4]]: +; CHECK-NEXT: [[PHI4:%.*]] = phi i64 [ [[ADD2]], %[[BB2]] ], [ [[ADD3]], %[[BB3]] ] +; CHECK-NEXT: [[ADD4:%.*]] = add i64 [[PHI4]], 4 +; CHECK-NEXT: br label %[[BB5]] +; CHECK: [[BB5]]: +; CHECK-NEXT: [[PHI5:%.*]] = phi i64 [ [[ADD0]], %[[BB0]] ], [ [[ADD4]], %[[BB4]] ] +; CHECK-NEXT: store i64 [[PHI5]], ptr [[A]], align 4 +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 128 +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[BB0]], !llvm.loop [[LOOP17:![0-9]+]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %bb0 + +bb0: +; bb0 (v) +; / \ +; | bb1 (br i1 (load i1 %uni.ptr)) +; | / \ +; | bb3 bb2 +; | \ / +; | bb4 +; \ / +; bb5 [phi] +; bb1's condition is uniform, but its value is loaded only along one path of +; bb0's varying branch. It cannot be preserved trivially, because some the load +; may never be executed. + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ] + %add0 = add i64 %iv, 0 + %v0 = icmp sle i64 %iv, 0 + br i1 %v0, label %bb5, label %bb1 + +bb1: + %cu1 = load i1, ptr %uni.ptr1 + br i1 %cu1, label %bb3, label %bb2 + +bb2: + %add2 = add i64 %iv, 2 + br label %bb4 + +bb3: + %add3 = add i64 %iv, 3 + br label %bb4 + +bb4: + %phi4 = phi i64 [ %add2, %bb2 ], [ %add3, %bb3 ] + %add4 = add i64 %phi4, 4 + br label %bb5 + +bb5: + %phi5 = phi i64 [ %add0, %bb0 ], [ %add4, %bb4 ] + store i64 %phi5, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +define void @uniform_branch_unstructured_merge1(ptr %a, i1 %u0) { +; CHECK-LABEL: define void @uniform_branch_unstructured_merge1( +; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true) +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128 +; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1) +; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2) +; CHECK-NEXT: [[TMP4:%.*]] = icmp sgt <4 x i64> [[VEC_IND]], splat (i64 2) +; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> splat (i1 true), <4 x i1> [[TMP4]] +; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U0]], <4 x i64> [[TMP3]], <4 x i64> [[TMP2]] +; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 3) +; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP7]], <4 x i64> [[TMP5]] +; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3 +; CHECK-NEXT: store i64 [[TMP8]], ptr [[A]], align 4 +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %bb0 + +bb0: +; All @uniform_branch_unstructured_merge[0-4] below have the same structure of uniform +; control flow merging into the middle of the varying diamond, but "covering" +; different potential RPOT traversals. +; bb0 (u) +; / \ +; bb2 (v) bb1 +; / \ / +; bb4 bb3 [phi] +; \ / +; bb5 [phi] + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ] + br i1 %u0, label %bb2, label %bb1 + +bb1: + %add1 = add i64 %iv, 1 + br label %bb3 + +bb2: + %add2 = add i64 %iv, 2 + %v2 = icmp sle i64 %iv, 2 + br i1 %v2, label %bb4, label %bb3 + +bb3: + %phi3 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ] + %add3 = add i64 %phi3, 3 + br label %bb5 + +bb4: + %add4 = add i64 %iv, 4 + br label %bb5 + +bb5: + %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ] + store i64 %phi5, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +define void @uniform_branch_unstructured_merge2(ptr %a, i1 %u0) { +; CHECK-LABEL: define void @uniform_branch_unstructured_merge2( +; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true) +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128 +; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1) +; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2) +; CHECK-NEXT: [[TMP4:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 2) +; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3) +; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> splat (i1 true), <4 x i1> [[TMP4]] +; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U0]], <4 x i64> [[TMP3]], <4 x i64> [[TMP2]] +; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 4) +; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP6]], <4 x i64> [[TMP7]], <4 x i64> [[TMP5]] +; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3 +; CHECK-NEXT: store i64 [[TMP8]], ptr [[A]], align 4 +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %bb0 + +bb0: +; bb0 (u) +; / \ +; bb2 (v) bb1 +; / \ / +; | +-\---+ +; | / \ +; bb4 [phi] bb3 +; \ / +; bb5 [phi] + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ] + br i1 %u0, label %bb2, label %bb1 + +bb1: + %add1 = add i64 %iv, 1 + br label %bb4 + +bb2: + %add2 = add i64 %iv, 2 + %v2 = icmp sle i64 %iv, 2 + br i1 %v2, label %bb4, label %bb3 + +bb3: + %add3 = add i64 %iv, 3 + br label %bb5 + +bb4: + %phi4 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ] + %add4 = add i64 %phi4, 4 + br label %bb5 + +bb5: + %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ] + store i64 %phi5, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +define void @uniform_branch_unstructured_merge3(ptr %a, i1 %u0) { +; CHECK-LABEL: define void @uniform_branch_unstructured_merge3( +; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true) +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128 +; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1) +; CHECK-NEXT: [[TMP3:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 1) +; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2) +; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3) +; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP7:%.*]] = or <4 x i1> [[BROADCAST_SPLAT]], [[TMP6]] +; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U0]], <4 x i64> [[TMP5]], <4 x i64> [[TMP2]] +; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 4) +; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP7]], <4 x i64> [[TMP8]], <4 x i64> [[TMP4]] +; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3 +; CHECK-NEXT: store i64 [[TMP9]], ptr [[A]], align 4 +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %bb0 + +bb0: +; bb0 (u) +; / \ +; bb3 bb1 (v) +; \ / \ +; bb4 [phi] bb2 +; \ / +; bb5 [phi] + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ] + br i1 %u0, label %bb3, label %bb1 + +bb1: + %add1 = add i64 %iv, 1 + %v1 = icmp sle i64 %iv, 1 + br i1 %v1, label %bb4, label %bb2 + +bb2: + %add2 = add i64 %iv, 2 + br label %bb5 + +bb3: + %add3 = add i64 %iv, 3 + br label %bb4 + +bb4: + %phi4 = phi i64 [ %add1, %bb1 ], [ %add3, %bb3 ] + %add4 = add i64 %phi4, 4 + br label %bb5 + +bb5: + %phi5 = phi i64 [ %add2, %bb2 ], [ %add4, %bb4 ] + store i64 %phi5, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +define void @uniform_branch_unstructured_merge4(ptr %a, i1 %u0) { +; CHECK-LABEL: define void @uniform_branch_unstructured_merge4( +; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true) +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128 +; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1) +; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <4 x i64> [[VEC_IND]], splat (i64 1) +; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3) +; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP7:%.*]] = or <4 x i1> [[BROADCAST_SPLAT]], [[TMP6]] +; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U0]], <4 x i64> [[TMP5]], <4 x i64> [[TMP2]] +; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 2) +; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP7]], <4 x i64> [[TMP8]], <4 x i64> [[TMP4]] +; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3 +; CHECK-NEXT: store i64 [[TMP9]], ptr [[A]], align 4 +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %bb0 + +bb0: +; bb0 (u) +; / \ +; bb3 bb1 (v) +; \ / \ +; +-/--+ \ +; / \ \ +; bb4 bb2 [phi] +; \ / +; bb5 [phi] + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ] + br i1 %u0, label %bb3, label %bb1 + +bb1: + %add1 = add i64 %iv, 1 + %v1 = icmp sle i64 %iv, 1 + br i1 %v1, label %bb4, label %bb2 + +bb2: + %phi2 = phi i64 [ %add1, %bb1 ], [ %add3, %bb3 ] + %add2 = add i64 %phi2, 2 + br label %bb5 + +bb3: + %add3 = add i64 %iv, 3 + br label %bb2 + +bb4: + %add4 = add i64 %iv, 4 + br label %bb5 + +bb5: + %phi5 = phi i64 [ %add2, %bb2 ], [ %add4, %bb4 ] + store i64 %phi5, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +define void @uniform_branch_shared_join_with_varying(ptr %a, i1 %u0) { +; CHECK-LABEL: define void @uniform_branch_shared_join_with_varying( +; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128 +; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1) +; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2) +; CHECK-NEXT: [[TMP3:%.*]] = icmp sle <4 x i64> [[VEC_IND]], splat (i64 2) +; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3) +; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U0]], <4 x i64> [[TMP2]], <4 x i64> [[TMP1]] +; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP4]], <4 x i64> [[TMP5]], <4 x i64> [[PREDPHI]] +; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[PREDPHI1]], i64 3 +; CHECK-NEXT: store i64 [[TMP6]], ptr [[A]], align 4 +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %bb0 + +bb0: +; bb0 (u) +; / \ +; bb2 (v) bb1 +; / | | +; bb3 | / +; \ | / +; bb4 [phi] +; Theoretically can be done, but would require a combination of a blend and a phi. + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb4 ] + br i1 %u0, label %bb2, label %bb1 + +bb1: + %add1 = add i64 %iv, 1 + br label %bb4 + +bb2: + %add2 = add i64 %iv, 2 + %v2 = icmp sle i64 %iv, 2 + br i1 %v2, label %bb3, label %bb4 + +bb3: + %add3 = add i64 %iv, 3 + br label %bb4 + +bb4: + %phi4 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ], [ %add3, %bb3 ] + store i64 %phi4, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +define void @unstructured_uniform_only(ptr %a, i1 %u0, i1 %u2) { +; CHECK-LABEL: define void @unstructured_uniform_only( +; CHECK-SAME: ptr [[A:%.*]], i1 [[U0:%.*]], i1 [[U2:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U2]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i1> poison, i1 [[U0]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT2]], splat (i1 true) +; CHECK-NEXT: [[TMP1:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true) +; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> splat (i1 true), <4 x i1> [[TMP1]] +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128 +; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1) +; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2) +; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U0]], <4 x i64> [[TMP5]], <4 x i64> [[TMP4]] +; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 3) +; CHECK-NEXT: [[PREDPHI3:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> [[TMP7]], <4 x i64> [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[PREDPHI3]], i64 3 +; CHECK-NEXT: store i64 [[TMP8]], ptr [[A]], align 4 +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %bb0 + +bb0: +; bb0 (u) +; / \ +; bb2 (u) bb1 +; / \ / +; bb4 bb3 [phi] +; \ / +; bb5 [phi] +; Triviallly preservable, but detection might not be easy. + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb5 ] + br i1 %u0, label %bb2, label %bb1 + +bb1: + %add1 = add i64 %iv, 1 + br label %bb3 + +bb2: + %add2 = add i64 %iv, 2 + br i1 %u2, label %bb4, label %bb3 + +bb3: + %phi3 = phi i64 [ %add1, %bb1 ], [ %add2, %bb2 ] + %add3 = add i64 %phi3, 3 + br label %bb5 + +bb4: + %add4 = add i64 %iv, 4 + br label %bb5 + +bb5: + %phi5 = phi i64 [ %add3, %bb3 ], [ %add4, %bb4 ] + store i64 %phi5, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} + +define void @unstructured_uniform_only_sese_region(ptr %a, i1 %u1, i1 %u3) { +; CHECK-LABEL: define void @unstructured_uniform_only_sese_region( +; CHECK-SAME: ptr [[A:%.*]], i1 [[U1:%.*]], i1 [[U3:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[U3]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i1> poison, i1 [[U1]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT2]], splat (i1 true) +; CHECK-NEXT: [[TMP1:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true) +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128 +; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <4 x i64> [[VEC_IND]], zeroinitializer +; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[TMP0]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 2) +; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[BROADCAST_SPLAT2]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 3) +; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 5) +; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> [[TMP1]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP10:%.*]] = or <4 x i1> [[TMP9]], [[TMP4]] +; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[U1]], <4 x i64> [[TMP7]], <4 x i64> [[TMP5]] +; CHECK-NEXT: [[TMP11:%.*]] = add <4 x i64> [[PREDPHI]], splat (i64 4) +; CHECK-NEXT: [[PREDPHI3:%.*]] = select <4 x i1> [[TMP10]], <4 x i64> [[TMP11]], <4 x i64> [[TMP8]] +; CHECK-NEXT: [[TMP12:%.*]] = add <4 x i64> [[PREDPHI3]], splat (i64 7) +; CHECK-NEXT: [[TMP13:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 6) +; CHECK-NEXT: [[PREDPHI4:%.*]] = select <4 x i1> [[TMP3]], <4 x i64> [[TMP12]], <4 x i64> [[TMP13]] +; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i64> [[PREDPHI4]], i64 3 +; CHECK-NEXT: store i64 [[TMP14]], ptr [[A]], align 4 +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %bb0 + +bb0: +; bb0 (v) +; / \ +; bb6 bb1 (u) +; | / \ +; | bb3 (u) bb2 +; | / \ / +; | bb5 bb4 [phi] +; | \ / +; | bb7 [phi] +; \ / +; \ / +; bb8 [phi] + %iv = phi i64 [ 0, %entry ], [ %iv.next, %bb8 ] + %v0 = icmp sle i64 %iv, 0 + br i1 %v0, label %bb6, label %bb1 + +bb1: + br i1 %u1, label %bb3, label %bb2 + +bb2: + %add2 = add i64 %iv, 2 + br label %bb4 + +bb3: + %add3 = add i64 %iv, 3 + br i1 %u3, label %bb5, label %bb4 + +bb4: + %phi4 = phi i64 [ %add2, %bb2 ], [ %add3, %bb3 ] + %add4 = add i64 %phi4, 4 + br label %bb7 + +bb5: + %add5 = add i64 %iv, 5 + br label %bb7 + +bb6: + %add6 = add i64 %iv, 6 + br label %bb8 + +bb7: + %phi7 = phi i64 [ %add4, %bb4 ], [ %add5, %bb5 ] + %add7 = add i64 %phi7, 7 + br label %bb8 + +bb8: + %phi8 = phi i64 [ %add6, %bb6 ], [ %add7, %bb7 ] + store i64 %phi8, ptr %a + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 128 + br i1 %ec, label %exit, label %bb0 + +exit: + ret void +} _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
