https://github.com/eas updated https://github.com/llvm/llvm-project/pull/219061
>From 3a3724bbf03add227d455b22dff83eebff020ce6 Mon Sep 17 00:00:00 2001 From: Andrei Elovikov <[email protected]> Date: Wed, 26 Aug 2026 08:53:10 -0700 Subject: [PATCH 1/3] Copy tests to predicator.ll --- .../LoopVectorize/VPlan/predicator.ll | 343 ++++++++++++++++++ 1 file changed, 343 insertions(+) diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll index 483ab8ad94b50..16f340818345f 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll @@ -959,3 +959,346 @@ latch: exit: ret void } + +; RUN: llvm-extract --func const_tc_with_predicated_store < %s | opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -vplan-verify-each -disable-output -vplan-print-after=introduce 2>&1 | FileCheck %s --check-prefix TMP1 +define void @const_tc_with_predicated_store(i1 %c1, i1 %c2, i1 %c3, ptr %dst) #1 { +; CHECK-LABEL: VPlan for loop in 'const_tc_with_predicated_store' +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> +; CHECK-NEXT: Successor(s): if.then +; CHECK-EMPTY: +; CHECK-NEXT: if.then: +; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c1> +; CHECK-NEXT: Successor(s): if.else1 +; CHECK-EMPTY: +; CHECK-NEXT: if.else1: +; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c2> +; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]> +; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP6]]>, ir<%c1> +; CHECK-NEXT: BLEND ir<%phi1> = ir<0.000000e+00>/vp<[[VP6]]> ir<1.000000e+00>/ir<%c1> +; CHECK-NEXT: Successor(s): if.else2 +; CHECK-EMPTY: +; CHECK-NEXT: if.else2: +; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = not ir<%c3> +; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and vp<[[VP7]]>, vp<[[VP8]]> +; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c2> +; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = or vp<[[VP9]]>, vp<[[VP10]]> +; CHECK-NEXT: Successor(s): latch +; CHECK-EMPTY: +; CHECK-NEXT: latch: +; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = logical-and vp<[[VP7]]>, ir<%c3> +; CHECK-NEXT: BLEND ir<%phi> = ir<2.000000e+00>/vp<[[VP11]]> ir<%phi1>/vp<[[VP12]]> +; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%dst>, ir<%iv> +; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep> +; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1> +; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv>, ir<56> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; +; TMP1-LABEL: VPlan for loop in 'const_tc_with_predicated_store' +; TMP1: VPlan ' for UF>=1' { +; TMP1-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF +; TMP1-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF +; TMP1-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count +; TMP1-NEXT: Live-in ir<57> = original trip-count +; TMP1-EMPTY: +; TMP1-NEXT: ir-bb<entry>: +; TMP1-NEXT: Successor(s): scalar.ph, vector.ph +; TMP1-EMPTY: +; TMP1-NEXT: vector.ph: +; TMP1-NEXT: Successor(s): vector loop +; TMP1-EMPTY: +; TMP1-NEXT: <x1> vector loop: { +; TMP1-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; TMP1-NEXT: vp<[[VP4:%[0-9]+]]> = HEADER-MASK +; TMP1-EMPTY: +; TMP1-NEXT: vector.body: +; TMP1-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]> +; TMP1-NEXT: Successor(s): vector.body.split +; TMP1-EMPTY: +; TMP1-NEXT: vector.body.split: +; TMP1-NEXT: Successor(s): if.then +; TMP1-EMPTY: +; TMP1-NEXT: if.then: +; TMP1-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1> +; TMP1-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]> +; TMP1-NEXT: Successor(s): if.else1 +; TMP1-EMPTY: +; TMP1-NEXT: if.else1: +; TMP1-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%c2> +; TMP1-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP6]]>, vp<[[VP7]]> +; TMP1-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c1> +; TMP1-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = or vp<[[VP8]]>, vp<[[VP9]]> +; TMP1-NEXT: BLEND ir<%phi1> = ir<0.000000e+00>/vp<[[VP8]]> ir<1.000000e+00>/vp<[[VP9]]> +; TMP1-NEXT: Successor(s): if.else2 +; TMP1-EMPTY: +; TMP1-NEXT: if.else2: +; TMP1-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = not ir<%c3> +; TMP1-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = logical-and vp<[[VP10]]>, vp<[[VP11]]> +; TMP1-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = logical-and vp<[[VP6]]>, ir<%c2> +; TMP1-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = or vp<[[VP12]]>, vp<[[VP13]]> +; TMP1-NEXT: Successor(s): latch +; TMP1-EMPTY: +; TMP1-NEXT: latch: +; TMP1-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = logical-and vp<[[VP10]]>, ir<%c3> +; TMP1-NEXT: BLEND ir<%phi> = ir<2.000000e+00>/vp<[[VP14]]> ir<%phi1>/vp<[[VP15]]> +; TMP1-NEXT: EMIT ir<%gep> = getelementptr ir<%dst>, ir<%iv> +; TMP1-NEXT: EMIT store ir<%phi>, ir<%gep>, vp<[[VP4]]> +; TMP1-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>, vp<[[VP4]]> +; TMP1-NEXT: EMIT ir<%ec> = icmp eq ir<%iv>, ir<56>, vp<[[VP4]]> +; TMP1-NEXT: Successor(s): vector.latch +; TMP1-EMPTY: +; TMP1-NEXT: vector.latch: +; TMP1-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]> +; TMP1-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]> +; TMP1-NEXT: No successors +; TMP1-NEXT: } +; TMP1-NEXT: Successor(s): middle.block +; TMP1-EMPTY: +; TMP1-NEXT: middle.block: +; TMP1-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = exiting-iv-value ir<%iv> +; TMP1-NEXT: EMIT vp<%cmp.n> = icmp eq ir<57>, vp<[[VP2]]> +; TMP1-NEXT: EMIT branch-on-cond ir<true> +; TMP1-NEXT: Successor(s): ir-bb<exit>, scalar.ph +; TMP1-EMPTY: +; TMP1-NEXT: ir-bb<exit>: +; TMP1-NEXT: No successors +; TMP1-EMPTY: +; TMP1-NEXT: scalar.ph: +; TMP1-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP17]]>, middle.block ], [ ir<0>, ir-bb<entry> ] +; TMP1-NEXT: Successor(s): ir-bb<header> +; TMP1-EMPTY: +; TMP1-NEXT: ir-bb<header>: +; TMP1-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ] (extra operand: vp<%bc.resume.val> from scalar.ph) +; TMP1-NEXT: No successors +; TMP1-NEXT: } +; +entry: + br label %header + +header: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ] + br i1 %c1, label %if.else1, label %if.then + +if.then: + br i1 %c2, label %if.else2, label %if.else1 + +if.else1: + %phi1 = phi float [ 0.0, %if.then ], [ 1.0, %header ] + br i1 %c3, label %latch, label %if.else2 + +if.else2: + br label %latch + +latch: + %phi = phi float [ %phi1, %if.else1 ], [ 2.0, %if.else2 ] + %gep = getelementptr float, ptr %dst, i64 %iv + store float %phi, ptr %gep, align 4 + %iv.next = add i64 %iv, 1 + %ec = icmp eq i64 %iv, 56 + br i1 %ec, label %exit, label %header + +exit: + ret void +} + +; RUN: llvm-extract --func test < %s | opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -disable-output -vplan-print-after=introduce 2>&1 | FileCheck %s --check-prefix=TMP2 +define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src2, ptr noalias %dst, i1 %c1, i1 %c2, i1 %c3) { +; CHECK-LABEL: VPlan for loop in 'test' +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> +; CHECK-NEXT: Successor(s): check.cond1 +; CHECK-EMPTY: +; CHECK-NEXT: check.cond1: +; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c1> +; CHECK-NEXT: EMIT ir<%not.c2> = xor ir<%c2>, ir<true>, vp<[[VP4]]> +; CHECK-NEXT: EMIT ir<%cond1> = or ir<%c1>, ir<%not.c2>, vp<[[VP4]]> +; CHECK-NEXT: Successor(s): load.v0 +; CHECK-EMPTY: +; CHECK-NEXT: load.v0: +; CHECK-NEXT: EMIT ir<%gep0> = getelementptr inbounds ir<%src0>, ir<%iv> +; CHECK-NEXT: EMIT-SCALAR ir<%v0> = load ir<%gep0>, ir<%c1> +; CHECK-NEXT: Successor(s): load.v1 +; CHECK-EMPTY: +; CHECK-NEXT: load.v1: +; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%cond1> +; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = or ir<%c1>, vp<[[VP5]]> +; CHECK-NEXT: BLEND ir<%val0> = ir<%v0>/ir<%c1> ir<0>/vp<[[VP5]]> +; CHECK-NEXT: EMIT ir<%gep1> = getelementptr inbounds ir<%src1>, ir<%iv> +; CHECK-NEXT: EMIT-SCALAR ir<%v1> = load ir<%gep1>, vp<[[VP6]]> +; CHECK-NEXT: EMIT ir<%val1> = add ir<%v1>, ir<%val0>, vp<[[VP6]]> +; CHECK-NEXT: Successor(s): load.v2.check +; CHECK-EMPTY: +; CHECK-NEXT: load.v2.check: +; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%cond1> +; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP7]]> +; CHECK-NEXT: BLEND ir<%val2> = ir<%val1>/vp<[[VP6]]> ir<0>/vp<[[VP8]]> +; CHECK-NEXT: Successor(s): load.v2 +; CHECK-EMPTY: +; CHECK-NEXT: load.v2: +; CHECK-NEXT: EMIT ir<%gep2> = getelementptr inbounds ir<%src2>, ir<%iv> +; CHECK-NEXT: EMIT-SCALAR ir<%v2> = load ir<%gep2>, ir<%c3> +; CHECK-NEXT: EMIT ir<%val3> = add ir<%v2>, ir<%val2>, ir<%c3> +; CHECK-NEXT: Successor(s): latch +; CHECK-EMPTY: +; CHECK-NEXT: latch: +; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = not ir<%c3> +; CHECK-NEXT: BLEND ir<%result> = ir<%val3>/ir<%c3> ir<%val2>/vp<[[VP9]]> +; CHECK-NEXT: EMIT ir<%out> = getelementptr inbounds ir<%dst>, ir<%iv> +; CHECK-NEXT: EMIT store ir<%result>, ir<%out> +; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1> +; CHECK-NEXT: EMIT ir<%exitcond> = icmp eq ir<%iv.next>, ir<%n> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; +; TMP2-LABEL: VPlan for loop in 'test' +; TMP2: VPlan ' for UF>=1' { +; TMP2-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF +; TMP2-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF +; TMP2-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count +; TMP2-NEXT: Live-in ir<%n> = original trip-count +; TMP2-EMPTY: +; TMP2-NEXT: ir-bb<entry>: +; TMP2-NEXT: Successor(s): scalar.ph, vector.ph +; TMP2-EMPTY: +; TMP2-NEXT: vector.ph: +; TMP2-NEXT: Successor(s): vector loop +; TMP2-EMPTY: +; TMP2-NEXT: <x1> vector loop: { +; TMP2-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; TMP2-NEXT: vp<[[VP4:%[0-9]+]]> = HEADER-MASK +; TMP2-EMPTY: +; TMP2-NEXT: vector.body: +; TMP2-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]> +; TMP2-NEXT: Successor(s): vector.body.split +; TMP2-EMPTY: +; TMP2-NEXT: vector.body.split: +; TMP2-NEXT: Successor(s): check.cond1 +; TMP2-EMPTY: +; TMP2-NEXT: check.cond1: +; TMP2-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1> +; TMP2-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]> +; TMP2-NEXT: EMIT ir<%not.c2> = xor ir<%c2>, ir<true>, vp<[[VP6]]> +; TMP2-NEXT: EMIT ir<%cond1> = or ir<%c1>, ir<%not.c2>, vp<[[VP6]]> +; TMP2-NEXT: Successor(s): load.v0 +; TMP2-EMPTY: +; TMP2-NEXT: load.v0: +; TMP2-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c1> +; TMP2-NEXT: EMIT ir<%gep0> = getelementptr inbounds ir<%src0>, ir<%iv> +; TMP2-NEXT: EMIT-SCALAR ir<%v0> = load ir<%gep0>, vp<[[VP7]]> +; TMP2-NEXT: Successor(s): load.v1 +; TMP2-EMPTY: +; TMP2-NEXT: load.v1: +; TMP2-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP6]]>, ir<%cond1> +; TMP2-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = or vp<[[VP7]]>, vp<[[VP8]]> +; TMP2-NEXT: BLEND ir<%val0> = ir<%v0>/vp<[[VP7]]> ir<0>/vp<[[VP8]]> +; TMP2-NEXT: EMIT ir<%gep1> = getelementptr inbounds ir<%src1>, ir<%iv> +; TMP2-NEXT: EMIT-SCALAR ir<%v1> = load ir<%gep1>, vp<[[VP9]]> +; TMP2-NEXT: EMIT ir<%val1> = add ir<%v1>, ir<%val0>, vp<[[VP9]]> +; TMP2-NEXT: Successor(s): load.v2.check +; TMP2-EMPTY: +; TMP2-NEXT: load.v2.check: +; TMP2-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = not ir<%cond1> +; TMP2-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = logical-and vp<[[VP6]]>, vp<[[VP10]]> +; TMP2-NEXT: BLEND ir<%val2> = ir<%val1>/vp<[[VP9]]> ir<0>/vp<[[VP11]]> +; TMP2-NEXT: Successor(s): load.v2 +; TMP2-EMPTY: +; TMP2-NEXT: load.v2: +; TMP2-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c3> +; TMP2-NEXT: EMIT ir<%gep2> = getelementptr inbounds ir<%src2>, ir<%iv> +; TMP2-NEXT: EMIT-SCALAR ir<%v2> = load ir<%gep2>, vp<[[VP12]]> +; TMP2-NEXT: EMIT ir<%val3> = add ir<%v2>, ir<%val2>, vp<[[VP12]]> +; TMP2-NEXT: Successor(s): latch +; TMP2-EMPTY: +; TMP2-NEXT: latch: +; TMP2-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = not ir<%c3> +; TMP2-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP13]]> +; TMP2-NEXT: BLEND ir<%result> = ir<%val3>/vp<[[VP12]]> ir<%val2>/vp<[[VP14]]> +; TMP2-NEXT: EMIT ir<%out> = getelementptr inbounds ir<%dst>, ir<%iv> +; TMP2-NEXT: EMIT store ir<%result>, ir<%out>, vp<[[VP4]]> +; TMP2-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>, vp<[[VP4]]> +; TMP2-NEXT: EMIT ir<%exitcond> = icmp eq ir<%iv.next>, ir<%n>, vp<[[VP4]]> +; TMP2-NEXT: Successor(s): vector.latch +; TMP2-EMPTY: +; TMP2-NEXT: vector.latch: +; TMP2-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]> +; TMP2-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]> +; TMP2-NEXT: No successors +; TMP2-NEXT: } +; TMP2-NEXT: Successor(s): middle.block +; TMP2-EMPTY: +; TMP2-NEXT: middle.block: +; TMP2-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = exiting-iv-value ir<%iv> +; TMP2-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]> +; TMP2-NEXT: EMIT branch-on-cond ir<true> +; TMP2-NEXT: Successor(s): ir-bb<exit>, scalar.ph +; TMP2-EMPTY: +; TMP2-NEXT: ir-bb<exit>: +; TMP2-NEXT: No successors +; TMP2-EMPTY: +; TMP2-NEXT: scalar.ph: +; TMP2-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP16]]>, middle.block ], [ ir<0>, ir-bb<entry> ] +; TMP2-NEXT: Successor(s): ir-bb<loop> +; TMP2-EMPTY: +; TMP2-NEXT: ir-bb<loop>: +; TMP2-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ] (extra operand: vp<%bc.resume.val> from scalar.ph) +; TMP2-NEXT: No successors +; TMP2-NEXT: } +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ] + br i1 %c1, label %load.v0, label %check.cond1 + +check.cond1: + %not.c2 = xor i1 %c2, true + %cond1 = or i1 %c1, %not.c2 + br i1 %cond1, label %load.v1, label %load.v2.check + +load.v0: + %gep0 = getelementptr inbounds i32, ptr %src0, i64 %iv + %v0 = load i32, ptr %gep0, align 4 + br label %load.v1 + +load.v1: + %val0 = phi i32 [ %v0, %load.v0 ], [ 0, %check.cond1 ] + %gep1 = getelementptr inbounds i32, ptr %src1, i64 %iv + %v1 = load i32, ptr %gep1, align 4 + %val1 = add i32 %v1, %val0 + br label %load.v2.check + +load.v2.check: + %val2 = phi i32 [ %val1, %load.v1 ], [ 0, %check.cond1 ] + br i1 %c3, label %load.v2, label %latch + +load.v2: + %gep2 = getelementptr inbounds i32, ptr %src2, i64 %iv + %v2 = load i32, ptr %gep2, align 4 + %val3 = add i32 %v2, %val2 + br label %latch + +latch: + %result = phi i32 [ %val3, %load.v2 ], [ %val2, %load.v2.check ] + %out = getelementptr inbounds i32, ptr %dst, i64 %iv + store i32 %result, ptr %out, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, %n + br i1 %exitcond, label %exit, label %loop + +exit: + ret void +} >From 2920ae187a831f0993144d9aecac8ce77ef70d8b Mon Sep 17 00:00:00 2001 From: Andrei Elovikov <[email protected]> Date: Wed, 26 Aug 2026 08:52:32 -0700 Subject: [PATCH 2/3] [VPlan] Reassociate header mask in the predicator during mask creation --- .../Transforms/Vectorize/VPlanPredicator.cpp | 51 +++++++++++++-- .../Transforms/Vectorize/VPlanTransforms.cpp | 29 --------- .../LoopVectorize/RISCV/low-trip-count.ll | 19 +++--- .../RISCV/tail-folding-complex-mask.ll | 4 +- .../LoopVectorize/VPlan/predicator.ll | 65 +++++++++++-------- .../VPlan/vplan-print-before-after-all.ll | 2 - 6 files changed, 95 insertions(+), 75 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp index c9e1eae12eb99..29c3366f9560f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp @@ -56,6 +56,15 @@ class VPPredicator { /// possibly inserting new recipes at \p Dst (using Builder's insertion point) VPValue *createEdgeMask(const VPBasicBlock *Src, const VPBasicBlock *Dst); + /// Create a logical-and of a source mask and edge condition, keeping the + /// header mask outermost when present. E.g. createMaskAnd("H && M", "C") + /// would result in "H && (M && C)". The edge condition must not contain the + /// header mask. + VPValue *createMaskAnd(VPValue *SrcMask, VPValue *EdgeCond, DebugLoc DL); + + /// Create a logical-or, factoring out a common header mask if present. + VPValue *createMaskOr(VPValue *LHS, VPValue *RHS, DebugLoc DL); + /// Record \p Mask as the *entry* mask of \p VPBB, which is expected to not /// already have a mask. void setBlockInMask(const VPBasicBlock *VPBB, VPValue *Mask) { @@ -118,6 +127,38 @@ class VPPredicator { }; } // namespace +VPValue *VPPredicator::createMaskAnd(VPValue *SrcMask, VPValue *EdgeCond, + DebugLoc DL) { + VPValue *HeaderMask = Plan.getVectorLoopRegion()->getHeaderMask(); + VPValue *Remainder = nullptr; + if (!HeaderMask || + !match(SrcMask, m_RemoveMask(HeaderMask, Remainder))) + return Builder.createLogicalAnd(SrcMask, EdgeCond, DL); + + [[maybe_unused]] VPValue *EdgeRemainder = nullptr; + assert(!match(EdgeCond, m_RemoveMask(HeaderMask, EdgeRemainder)) && + "Edge condition must not contain the header mask"); + + if (!Remainder) + return Builder.createLogicalAnd(HeaderMask, EdgeCond, DL); + return Builder.createLogicalAnd( + HeaderMask, Builder.createLogicalAnd(Remainder, EdgeCond, DL), DL); +} + +VPValue *VPPredicator::createMaskOr(VPValue *LHS, VPValue *RHS, DebugLoc DL) { + VPValue *HeaderMask = Plan.getVectorLoopRegion()->getHeaderMask(); + VPValue *LHSRemainder = nullptr; + VPValue *RHSRemainder = nullptr; + if (!HeaderMask || !match(LHS, m_RemoveMask(HeaderMask, LHSRemainder)) || + !match(RHS, m_RemoveMask(HeaderMask, RHSRemainder))) + return Builder.createOr(LHS, RHS, DL); + + if (!LHSRemainder || !RHSRemainder) + return HeaderMask; + return Builder.createLogicalAnd( + HeaderMask, Builder.createOr(LHSRemainder, RHSRemainder, DL), DL); +} + VPValue *VPPredicator::createEdgeMask(const VPBasicBlock *Src, const VPBasicBlock *Dst) { assert(is_contained(Dst->getPredecessors(), Src) && "Invalid edge"); @@ -154,7 +195,7 @@ VPValue *VPPredicator::createEdgeMask(const VPBasicBlock *Src, // The bitwise 'And' of SrcMask and EdgeMask introduces new UB if SrcMask // is false and EdgeMask is poison. Avoid that by using 'LogicalAnd' // instead which generates 'select i1 SrcMask, i1 EdgeMask, i1 false'. - EdgeMask = Builder.createLogicalAnd(SrcMask, EdgeMask, Term->getDebugLoc()); + EdgeMask = createMaskAnd(SrcMask, EdgeMask, Term->getDebugLoc()); } return setEdgeMask(Src, Dst, EdgeMask); @@ -191,7 +232,7 @@ void VPPredicator::createBlockInMask(VPBasicBlock *VPBB) { continue; } - BlockMask = Builder.createOr(BlockMask, EdgeMask, {}); + BlockMask = createMaskOr(BlockMask, EdgeMask, {}); } setBlockInMask(VPBB, BlockMask); @@ -230,7 +271,7 @@ void VPPredicator::createSwitchEdgeMasks(const VPInstruction *SI) { for (VPValue *V : drop_begin(Conds)) Mask = Builder.createOr(Mask, V); if (SrcMask) - Mask = Builder.createLogicalAnd(SrcMask, Mask); + Mask = createMaskAnd(SrcMask, Mask, {}); setEdgeMask(Src, Dst, Mask); // 2. Create the mask for the default destination, which is reached if @@ -243,7 +284,7 @@ void VPPredicator::createSwitchEdgeMasks(const VPInstruction *SI) { if (DefaultMask) { DefaultMask = Builder.createNot(DefaultMask); if (SrcMask) - DefaultMask = Builder.createLogicalAnd(SrcMask, DefaultMask); + DefaultMask = createMaskAnd(SrcMask, DefaultMask, {}); } else { // There are no destinations other than the default destination, so this is // an unconditional branch. @@ -346,7 +387,7 @@ VPValue *VPPredicator::createBlendMaskForEdges(ArrayRef<EdgeTy> Edges, Builder.setInsertPoint(Dst, getMaskInsertPoint(Dst)); EdgeMask = createEdgeMask(Src, Dst); } - Mask = Mask ? Builder.createOr(Mask, EdgeMask) : EdgeMask; + Mask = Mask ? createMaskOr(Mask, EdgeMask, {}) : EdgeMask; } return Mask; } diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index dd85eeaa8592c..7499a02a5b897 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1661,33 +1661,6 @@ void VPlanTransforms::simplifyReverses(VPlan &Plan) { R.getVPSingleValue()->replaceAllUsesWith(X); } -/// Reassociate (headermask && x) && y -> headermask && (x && y) to allow the -/// header mask to be simplified further when tail folding, e.g. in -/// optimizeEVLMasks. -static void reassociateHeaderMask(VPlan &Plan) { - VPValue *HeaderMask = Plan.getVectorLoopRegion()->getHeaderMask(); - if (!HeaderMask) - return; - - SmallVector<VPUser *> Worklist; - for (VPUser *U : HeaderMask->users()) - if (match(U, m_LogicalAnd(m_Specific(HeaderMask), m_VPValue()))) - append_range(Worklist, cast<VPSingleDefRecipe>(U)->users()); - - while (!Worklist.empty()) { - auto *R = dyn_cast<VPSingleDefRecipe>(Worklist.pop_back_val()); - VPValue *X, *Y; - if (!R || !match(R, m_LogicalAnd( - m_LogicalAnd(m_Specific(HeaderMask), m_VPValue(X)), - m_VPValue(Y)))) - continue; - append_range(Worklist, R->users()); - VPBuilder Builder(R); - R->replaceAllUsesWith( - Builder.createLogicalAnd(HeaderMask, Builder.createLogicalAnd(X, Y))); - } -} - static std::optional<Instruction::BinaryOps> getUnmaskedDivRemOpcode(Intrinsic::ID ID) { switch (ID) { @@ -2567,14 +2540,12 @@ bool VPlanTransforms::removeBranchOnConst(VPlan &Plan, bool OnlyLatches) { void VPlanTransforms::optimize(VPlan &Plan) { RUN_VPLAN_PASS(removeRedundantInductionCasts, Plan); - RUN_VPLAN_PASS(reassociateHeaderMask, Plan); RUN_VPLAN_PASS(simplifyRecipes, Plan); RUN_VPLAN_PASS(removeDeadRecipes, Plan); RUN_VPLAN_PASS(simplifyBlends, Plan); RUN_VPLAN_PASS(legalizeAndOptimizeInductions, Plan); RUN_VPLAN_PASS(narrowToSingleScalarRecipes, Plan); RUN_VPLAN_PASS(removeRedundantExpandSCEVRecipes, Plan); - RUN_VPLAN_PASS(reassociateHeaderMask, Plan); RUN_VPLAN_PASS(simplifyRecipes, Plan); RUN_VPLAN_PASS(removeBranchOnConst, Plan, /*OnlyLatches=*/false); RUN_VPLAN_PASS(simplifyReverses, Plan); diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll index c4e3f51eff4d6..822e67b1b4394 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll @@ -280,18 +280,17 @@ define void @const_tc_with_predicated_store(i1 %c1, i1 %c2, i1 %c3, ptr %dst) #1 ; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer ; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C1:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP12:%.*]] = xor <vscale x 4 x i1> [[BROADCAST_SPLAT4]], splat (i1 true) -; CHECK-NEXT: [[TMP1:%.*]] = xor <vscale x 4 x i1> [[BROADCAST_SPLAT2]], splat (i1 true) -; CHECK-NEXT: [[TMP13:%.*]] = select <vscale x 4 x i1> [[TMP12]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i1> zeroinitializer -; CHECK-NEXT: [[TMP2:%.*]] = or <vscale x 4 x i1> [[TMP13]], [[BROADCAST_SPLAT4]] -; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C1]], <vscale x 4 x float> splat (float 1.000000e+00), <vscale x 4 x float> zeroinitializer ; CHECK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C3:%.*]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT4]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT4]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP12:%.*]] = xor <vscale x 4 x i1> [[BROADCAST_SPLAT5]], splat (i1 true) +; CHECK-NEXT: [[TMP1:%.*]] = xor <vscale x 4 x i1> [[BROADCAST_SPLAT4]], splat (i1 true) +; CHECK-NEXT: [[TMP13:%.*]] = select <vscale x 4 x i1> [[TMP12]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP3:%.*]] = or <vscale x 4 x i1> [[TMP13]], [[BROADCAST_SPLAT5]] +; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C3]], <vscale x 4 x float> splat (float 1.000000e+00), <vscale x 4 x float> zeroinitializer +; CHECK-NEXT: [[TMP10:%.*]] = select <vscale x 4 x i1> [[TMP3]], <vscale x 4 x i1> [[BROADCAST_SPLAT2]], <vscale x 4 x i1> zeroinitializer +; CHECK-NEXT: [[PREDPHI5:%.*]] = select <vscale x 4 x i1> [[TMP10]], <vscale x 4 x float> [[PREDPHI]], <vscale x 4 x float> splat (float 2.000000e+00) ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: -; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[TMP2]], <vscale x 4 x i1> zeroinitializer, i32 57) -; CHECK-NEXT: [[TMP10:%.*]] = select <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i1> [[BROADCAST_SPLAT]], <vscale x 4 x i1> zeroinitializer -; CHECK-NEXT: [[PREDPHI5:%.*]] = select <vscale x 4 x i1> [[TMP10]], <vscale x 4 x float> [[PREDPHI]], <vscale x 4 x float> splat (float 2.000000e+00) ; CHECK-NEXT: call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[PREDPHI5]], ptr align 4 [[DST:%.*]], <vscale x 4 x i1> splat (i1 true), i32 57) ; CHECK-NEXT: br label [[MIDDLE_BLOCK:%.*]] ; CHECK: middle.block: @@ -360,7 +359,7 @@ define i8 @mul_non_pow_2_low_trip_count(ptr noalias %a) { ; CHECK-NEXT: [[MUL]] = mul i8 [[TMP5]], [[RDX]] ; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 ; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 10 -; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: for.end: ; CHECK-NEXT: [[MUL_LCSSA:%.*]] = phi i8 [ [[MUL]], [[FOR_BODY]] ] ; CHECK-NEXT: ret i8 [[MUL_LCSSA]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll index 1ffe82b366a38..0792fc784b211 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll @@ -21,6 +21,8 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src ; IF-EVL-NEXT: [[TMP1:%.*]] = or <vscale x 4 x i1> [[BROADCAST_SPLAT]], [[BROADCAST_SPLAT2]] ; IF-EVL-NEXT: [[TMP3:%.*]] = select <vscale x 4 x i1> [[TMP2]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i1> zeroinitializer ; IF-EVL-NEXT: [[TMP4:%.*]] = or <vscale x 4 x i1> [[BROADCAST_SPLAT]], [[TMP3]] +; IF-EVL-NEXT: [[TMP5:%.*]] = xor <vscale x 4 x i1> [[TMP1]], splat (i1 true) +; IF-EVL-NEXT: [[TMP6:%.*]] = select <vscale x 4 x i1> [[TMP2]], <vscale x 4 x i1> [[TMP5]], <vscale x 4 x i1> zeroinitializer ; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C3]], i64 0 ; IF-EVL-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer ; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]] @@ -34,7 +36,7 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src ; IF-EVL-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[SRC1]], i64 [[EVL_BASED_IV]] ; IF-EVL-NEXT: [[VP_OP_LOAD7:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> [[TMP4]], i32 [[TMP7]]) ; IF-EVL-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD7]], [[PREDPHI]] -; IF-EVL-NEXT: [[PREDPHI8:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> [[TMP4]], <vscale x 4 x i32> [[TMP12]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP7]]) +; IF-EVL-NEXT: [[PREDPHI8:%.*]] = select <vscale x 4 x i1> [[TMP6]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[TMP12]] ; IF-EVL-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[SRC2]], i64 [[EVL_BASED_IV]] ; IF-EVL-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP18]], <vscale x 4 x i1> [[BROADCAST_SPLAT4]], i32 [[TMP7]]) ; IF-EVL-NEXT: [[TMP19:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[PREDPHI8]] diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll index 16f340818345f..a273b778fb2d9 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll @@ -1032,22 +1032,28 @@ define void @const_tc_with_predicated_store(i1 %c1, i1 %c2, i1 %c3, ptr %dst) #1 ; TMP1-EMPTY: ; TMP1-NEXT: if.else1: ; TMP1-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%c2> -; TMP1-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP6]]>, vp<[[VP7]]> -; TMP1-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c1> -; TMP1-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = or vp<[[VP8]]>, vp<[[VP9]]> -; TMP1-NEXT: BLEND ir<%phi1> = ir<0.000000e+00>/vp<[[VP8]]> ir<1.000000e+00>/vp<[[VP9]]> +; TMP1-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP5]]>, vp<[[VP7]]> +; TMP1-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP8]]> +; TMP1-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c1> +; TMP1-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = or vp<[[VP8]]>, ir<%c1> +; TMP1-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP11]]> +; TMP1-NEXT: BLEND ir<%phi1> = ir<0.000000e+00>/vp<[[VP9]]> ir<1.000000e+00>/vp<[[VP10]]> ; TMP1-NEXT: Successor(s): if.else2 ; TMP1-EMPTY: ; TMP1-NEXT: if.else2: -; TMP1-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = not ir<%c3> -; TMP1-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = logical-and vp<[[VP10]]>, vp<[[VP11]]> -; TMP1-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = logical-and vp<[[VP6]]>, ir<%c2> -; TMP1-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = or vp<[[VP12]]>, vp<[[VP13]]> +; TMP1-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = not ir<%c3> +; TMP1-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = logical-and vp<[[VP11]]>, vp<[[VP13]]> +; TMP1-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP14]]> +; TMP1-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = logical-and vp<[[VP5]]>, ir<%c2> +; TMP1-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP16]]> +; TMP1-NEXT: EMIT vp<[[VP18:%[0-9]+]]> = or vp<[[VP14]]>, vp<[[VP16]]> +; TMP1-NEXT: EMIT vp<[[VP19:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP18]]> ; TMP1-NEXT: Successor(s): latch ; TMP1-EMPTY: ; TMP1-NEXT: latch: -; TMP1-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = logical-and vp<[[VP10]]>, ir<%c3> -; TMP1-NEXT: BLEND ir<%phi> = ir<2.000000e+00>/vp<[[VP14]]> ir<%phi1>/vp<[[VP15]]> +; TMP1-NEXT: EMIT vp<[[VP20:%[0-9]+]]> = logical-and vp<[[VP11]]>, ir<%c3> +; TMP1-NEXT: EMIT vp<[[VP21:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP20]]> +; TMP1-NEXT: BLEND ir<%phi> = ir<2.000000e+00>/vp<[[VP19]]> ir<%phi1>/vp<[[VP21]]> ; TMP1-NEXT: EMIT ir<%gep> = getelementptr ir<%dst>, ir<%iv> ; TMP1-NEXT: EMIT store ir<%phi>, ir<%gep>, vp<[[VP4]]> ; TMP1-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>, vp<[[VP4]]> @@ -1062,7 +1068,7 @@ define void @const_tc_with_predicated_store(i1 %c1, i1 %c2, i1 %c3, ptr %dst) #1 ; TMP1-NEXT: Successor(s): middle.block ; TMP1-EMPTY: ; TMP1-NEXT: middle.block: -; TMP1-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = exiting-iv-value ir<%iv> +; TMP1-NEXT: EMIT vp<[[VP23:%[0-9]+]]> = exiting-iv-value ir<%iv> ; TMP1-NEXT: EMIT vp<%cmp.n> = icmp eq ir<57>, vp<[[VP2]]> ; TMP1-NEXT: EMIT branch-on-cond ir<true> ; TMP1-NEXT: Successor(s): ir-bb<exit>, scalar.ph @@ -1071,7 +1077,7 @@ define void @const_tc_with_predicated_store(i1 %c1, i1 %c2, i1 %c3, ptr %dst) #1 ; TMP1-NEXT: No successors ; TMP1-EMPTY: ; TMP1-NEXT: scalar.ph: -; TMP1-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP17]]>, middle.block ], [ ir<0>, ir-bb<entry> ] +; TMP1-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP23]]>, middle.block ], [ ir<0>, ir-bb<entry> ] ; TMP1-NEXT: Successor(s): ir-bb<header> ; TMP1-EMPTY: ; TMP1-NEXT: ir-bb<header>: @@ -1201,31 +1207,34 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src ; TMP2-NEXT: Successor(s): load.v1 ; TMP2-EMPTY: ; TMP2-NEXT: load.v1: -; TMP2-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP6]]>, ir<%cond1> -; TMP2-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = or vp<[[VP7]]>, vp<[[VP8]]> -; TMP2-NEXT: BLEND ir<%val0> = ir<%v0>/vp<[[VP7]]> ir<0>/vp<[[VP8]]> +; TMP2-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP5]]>, ir<%cond1> +; TMP2-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP8]]> +; TMP2-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = or ir<%c1>, vp<[[VP8]]> +; TMP2-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP10]]> +; TMP2-NEXT: BLEND ir<%val0> = ir<%v0>/vp<[[VP7]]> ir<0>/vp<[[VP9]]> ; TMP2-NEXT: EMIT ir<%gep1> = getelementptr inbounds ir<%src1>, ir<%iv> -; TMP2-NEXT: EMIT-SCALAR ir<%v1> = load ir<%gep1>, vp<[[VP9]]> -; TMP2-NEXT: EMIT ir<%val1> = add ir<%v1>, ir<%val0>, vp<[[VP9]]> +; TMP2-NEXT: EMIT-SCALAR ir<%v1> = load ir<%gep1>, vp<[[VP11]]> +; TMP2-NEXT: EMIT ir<%val1> = add ir<%v1>, ir<%val0>, vp<[[VP11]]> ; TMP2-NEXT: Successor(s): load.v2.check ; TMP2-EMPTY: ; TMP2-NEXT: load.v2.check: -; TMP2-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = not ir<%cond1> -; TMP2-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = logical-and vp<[[VP6]]>, vp<[[VP10]]> -; TMP2-NEXT: BLEND ir<%val2> = ir<%val1>/vp<[[VP9]]> ir<0>/vp<[[VP11]]> +; TMP2-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = not ir<%cond1> +; TMP2-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = logical-and vp<[[VP5]]>, vp<[[VP12]]> +; TMP2-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP13]]> +; TMP2-NEXT: BLEND ir<%val2> = ir<%val1>/vp<[[VP11]]> ir<0>/vp<[[VP14]]> ; TMP2-NEXT: Successor(s): load.v2 ; TMP2-EMPTY: ; TMP2-NEXT: load.v2: -; TMP2-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c3> +; TMP2-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c3> ; TMP2-NEXT: EMIT ir<%gep2> = getelementptr inbounds ir<%src2>, ir<%iv> -; TMP2-NEXT: EMIT-SCALAR ir<%v2> = load ir<%gep2>, vp<[[VP12]]> -; TMP2-NEXT: EMIT ir<%val3> = add ir<%v2>, ir<%val2>, vp<[[VP12]]> +; TMP2-NEXT: EMIT-SCALAR ir<%v2> = load ir<%gep2>, vp<[[VP15]]> +; TMP2-NEXT: EMIT ir<%val3> = add ir<%v2>, ir<%val2>, vp<[[VP15]]> ; TMP2-NEXT: Successor(s): latch ; TMP2-EMPTY: ; TMP2-NEXT: latch: -; TMP2-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = not ir<%c3> -; TMP2-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP13]]> -; TMP2-NEXT: BLEND ir<%result> = ir<%val3>/vp<[[VP12]]> ir<%val2>/vp<[[VP14]]> +; TMP2-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = not ir<%c3> +; TMP2-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP16]]> +; TMP2-NEXT: BLEND ir<%result> = ir<%val3>/vp<[[VP15]]> ir<%val2>/vp<[[VP17]]> ; TMP2-NEXT: EMIT ir<%out> = getelementptr inbounds ir<%dst>, ir<%iv> ; TMP2-NEXT: EMIT store ir<%result>, ir<%out>, vp<[[VP4]]> ; TMP2-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>, vp<[[VP4]]> @@ -1240,7 +1249,7 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src ; TMP2-NEXT: Successor(s): middle.block ; TMP2-EMPTY: ; TMP2-NEXT: middle.block: -; TMP2-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = exiting-iv-value ir<%iv> +; TMP2-NEXT: EMIT vp<[[VP19:%[0-9]+]]> = exiting-iv-value ir<%iv> ; TMP2-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]> ; TMP2-NEXT: EMIT branch-on-cond ir<true> ; TMP2-NEXT: Successor(s): ir-bb<exit>, scalar.ph @@ -1249,7 +1258,7 @@ define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src ; TMP2-NEXT: No successors ; TMP2-EMPTY: ; TMP2-NEXT: scalar.ph: -; TMP2-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP16]]>, middle.block ], [ ir<0>, ir-bb<entry> ] +; TMP2-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP19]]>, middle.block ], [ ir<0>, ir-bb<entry> ] ; TMP2-NEXT: Successor(s): ir-bb<loop> ; TMP2-EMPTY: ; TMP2-NEXT: ir-bb<loop>: diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll index c2bd046efae87..21e67e691a795 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll @@ -48,14 +48,12 @@ ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::truncateToMinimalBitwidths ; CHECK-BEFORE: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::optimize ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] removeRedundantInductionCasts -; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] reassociateHeaderMask ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] simplifyRecipes ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] removeDeadRecipes ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] simplifyBlends ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] legalizeAndOptimizeInductions ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] narrowToSingleScalarRecipes ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] removeRedundantExpandSCEVRecipes -; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] reassociateHeaderMask@2 ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] simplifyRecipes@2 ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] removeBranchOnConst ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] simplifyReverses >From 245630bde1c5221fe4b6173c2011ffb154653751 Mon Sep 17 00:00:00 2001 From: Andrei Elovikov <[email protected]> Date: Wed, 26 Aug 2026 09:01:41 -0700 Subject: [PATCH 3/3] Remove tests from predicator.ll --- .../LoopVectorize/VPlan/predicator.ll | 352 ------------------ 1 file changed, 352 deletions(-) diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll index a273b778fb2d9..483ab8ad94b50 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll @@ -959,355 +959,3 @@ latch: exit: ret void } - -; RUN: llvm-extract --func const_tc_with_predicated_store < %s | opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -vplan-verify-each -disable-output -vplan-print-after=introduce 2>&1 | FileCheck %s --check-prefix TMP1 -define void @const_tc_with_predicated_store(i1 %c1, i1 %c2, i1 %c3, ptr %dst) #1 { -; CHECK-LABEL: VPlan for loop in 'const_tc_with_predicated_store' -; CHECK-NEXT: <x1> vector loop: { -; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV -; CHECK-EMPTY: -; CHECK-NEXT: vector.body: -; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> -; CHECK-NEXT: Successor(s): if.then -; CHECK-EMPTY: -; CHECK-NEXT: if.then: -; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c1> -; CHECK-NEXT: Successor(s): if.else1 -; CHECK-EMPTY: -; CHECK-NEXT: if.else1: -; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c2> -; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]> -; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP6]]>, ir<%c1> -; CHECK-NEXT: BLEND ir<%phi1> = ir<0.000000e+00>/vp<[[VP6]]> ir<1.000000e+00>/ir<%c1> -; CHECK-NEXT: Successor(s): if.else2 -; CHECK-EMPTY: -; CHECK-NEXT: if.else2: -; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = not ir<%c3> -; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and vp<[[VP7]]>, vp<[[VP8]]> -; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c2> -; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = or vp<[[VP9]]>, vp<[[VP10]]> -; CHECK-NEXT: Successor(s): latch -; CHECK-EMPTY: -; CHECK-NEXT: latch: -; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = logical-and vp<[[VP7]]>, ir<%c3> -; CHECK-NEXT: BLEND ir<%phi> = ir<2.000000e+00>/vp<[[VP11]]> ir<%phi1>/vp<[[VP12]]> -; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%dst>, ir<%iv> -; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep> -; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1> -; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv>, ir<56> -; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]> -; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]> -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; -; TMP1-LABEL: VPlan for loop in 'const_tc_with_predicated_store' -; TMP1: VPlan ' for UF>=1' { -; TMP1-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF -; TMP1-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF -; TMP1-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count -; TMP1-NEXT: Live-in ir<57> = original trip-count -; TMP1-EMPTY: -; TMP1-NEXT: ir-bb<entry>: -; TMP1-NEXT: Successor(s): scalar.ph, vector.ph -; TMP1-EMPTY: -; TMP1-NEXT: vector.ph: -; TMP1-NEXT: Successor(s): vector loop -; TMP1-EMPTY: -; TMP1-NEXT: <x1> vector loop: { -; TMP1-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV -; TMP1-NEXT: vp<[[VP4:%[0-9]+]]> = HEADER-MASK -; TMP1-EMPTY: -; TMP1-NEXT: vector.body: -; TMP1-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]> -; TMP1-NEXT: Successor(s): vector.body.split -; TMP1-EMPTY: -; TMP1-NEXT: vector.body.split: -; TMP1-NEXT: Successor(s): if.then -; TMP1-EMPTY: -; TMP1-NEXT: if.then: -; TMP1-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1> -; TMP1-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]> -; TMP1-NEXT: Successor(s): if.else1 -; TMP1-EMPTY: -; TMP1-NEXT: if.else1: -; TMP1-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%c2> -; TMP1-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP5]]>, vp<[[VP7]]> -; TMP1-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP8]]> -; TMP1-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c1> -; TMP1-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = or vp<[[VP8]]>, ir<%c1> -; TMP1-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP11]]> -; TMP1-NEXT: BLEND ir<%phi1> = ir<0.000000e+00>/vp<[[VP9]]> ir<1.000000e+00>/vp<[[VP10]]> -; TMP1-NEXT: Successor(s): if.else2 -; TMP1-EMPTY: -; TMP1-NEXT: if.else2: -; TMP1-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = not ir<%c3> -; TMP1-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = logical-and vp<[[VP11]]>, vp<[[VP13]]> -; TMP1-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP14]]> -; TMP1-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = logical-and vp<[[VP5]]>, ir<%c2> -; TMP1-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP16]]> -; TMP1-NEXT: EMIT vp<[[VP18:%[0-9]+]]> = or vp<[[VP14]]>, vp<[[VP16]]> -; TMP1-NEXT: EMIT vp<[[VP19:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP18]]> -; TMP1-NEXT: Successor(s): latch -; TMP1-EMPTY: -; TMP1-NEXT: latch: -; TMP1-NEXT: EMIT vp<[[VP20:%[0-9]+]]> = logical-and vp<[[VP11]]>, ir<%c3> -; TMP1-NEXT: EMIT vp<[[VP21:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP20]]> -; TMP1-NEXT: BLEND ir<%phi> = ir<2.000000e+00>/vp<[[VP19]]> ir<%phi1>/vp<[[VP21]]> -; TMP1-NEXT: EMIT ir<%gep> = getelementptr ir<%dst>, ir<%iv> -; TMP1-NEXT: EMIT store ir<%phi>, ir<%gep>, vp<[[VP4]]> -; TMP1-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>, vp<[[VP4]]> -; TMP1-NEXT: EMIT ir<%ec> = icmp eq ir<%iv>, ir<56>, vp<[[VP4]]> -; TMP1-NEXT: Successor(s): vector.latch -; TMP1-EMPTY: -; TMP1-NEXT: vector.latch: -; TMP1-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]> -; TMP1-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]> -; TMP1-NEXT: No successors -; TMP1-NEXT: } -; TMP1-NEXT: Successor(s): middle.block -; TMP1-EMPTY: -; TMP1-NEXT: middle.block: -; TMP1-NEXT: EMIT vp<[[VP23:%[0-9]+]]> = exiting-iv-value ir<%iv> -; TMP1-NEXT: EMIT vp<%cmp.n> = icmp eq ir<57>, vp<[[VP2]]> -; TMP1-NEXT: EMIT branch-on-cond ir<true> -; TMP1-NEXT: Successor(s): ir-bb<exit>, scalar.ph -; TMP1-EMPTY: -; TMP1-NEXT: ir-bb<exit>: -; TMP1-NEXT: No successors -; TMP1-EMPTY: -; TMP1-NEXT: scalar.ph: -; TMP1-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP23]]>, middle.block ], [ ir<0>, ir-bb<entry> ] -; TMP1-NEXT: Successor(s): ir-bb<header> -; TMP1-EMPTY: -; TMP1-NEXT: ir-bb<header>: -; TMP1-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ] (extra operand: vp<%bc.resume.val> from scalar.ph) -; TMP1-NEXT: No successors -; TMP1-NEXT: } -; -entry: - br label %header - -header: - %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ] - br i1 %c1, label %if.else1, label %if.then - -if.then: - br i1 %c2, label %if.else2, label %if.else1 - -if.else1: - %phi1 = phi float [ 0.0, %if.then ], [ 1.0, %header ] - br i1 %c3, label %latch, label %if.else2 - -if.else2: - br label %latch - -latch: - %phi = phi float [ %phi1, %if.else1 ], [ 2.0, %if.else2 ] - %gep = getelementptr float, ptr %dst, i64 %iv - store float %phi, ptr %gep, align 4 - %iv.next = add i64 %iv, 1 - %ec = icmp eq i64 %iv, 56 - br i1 %ec, label %exit, label %header - -exit: - ret void -} - -; RUN: llvm-extract --func test < %s | opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -disable-output -vplan-print-after=introduce 2>&1 | FileCheck %s --check-prefix=TMP2 -define void @test(i64 %n, ptr noalias %src0, ptr noalias %src1, ptr noalias %src2, ptr noalias %dst, i1 %c1, i1 %c2, i1 %c3) { -; CHECK-LABEL: VPlan for loop in 'test' -; CHECK-NEXT: <x1> vector loop: { -; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV -; CHECK-EMPTY: -; CHECK-NEXT: vector.body: -; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> -; CHECK-NEXT: Successor(s): check.cond1 -; CHECK-EMPTY: -; CHECK-NEXT: check.cond1: -; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = not ir<%c1> -; CHECK-NEXT: EMIT ir<%not.c2> = xor ir<%c2>, ir<true>, vp<[[VP4]]> -; CHECK-NEXT: EMIT ir<%cond1> = or ir<%c1>, ir<%not.c2>, vp<[[VP4]]> -; CHECK-NEXT: Successor(s): load.v0 -; CHECK-EMPTY: -; CHECK-NEXT: load.v0: -; CHECK-NEXT: EMIT ir<%gep0> = getelementptr inbounds ir<%src0>, ir<%iv> -; CHECK-NEXT: EMIT-SCALAR ir<%v0> = load ir<%gep0>, ir<%c1> -; CHECK-NEXT: Successor(s): load.v1 -; CHECK-EMPTY: -; CHECK-NEXT: load.v1: -; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%cond1> -; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = or ir<%c1>, vp<[[VP5]]> -; CHECK-NEXT: BLEND ir<%val0> = ir<%v0>/ir<%c1> ir<0>/vp<[[VP5]]> -; CHECK-NEXT: EMIT ir<%gep1> = getelementptr inbounds ir<%src1>, ir<%iv> -; CHECK-NEXT: EMIT-SCALAR ir<%v1> = load ir<%gep1>, vp<[[VP6]]> -; CHECK-NEXT: EMIT ir<%val1> = add ir<%v1>, ir<%val0>, vp<[[VP6]]> -; CHECK-NEXT: Successor(s): load.v2.check -; CHECK-EMPTY: -; CHECK-NEXT: load.v2.check: -; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%cond1> -; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP7]]> -; CHECK-NEXT: BLEND ir<%val2> = ir<%val1>/vp<[[VP6]]> ir<0>/vp<[[VP8]]> -; CHECK-NEXT: Successor(s): load.v2 -; CHECK-EMPTY: -; CHECK-NEXT: load.v2: -; CHECK-NEXT: EMIT ir<%gep2> = getelementptr inbounds ir<%src2>, ir<%iv> -; CHECK-NEXT: EMIT-SCALAR ir<%v2> = load ir<%gep2>, ir<%c3> -; CHECK-NEXT: EMIT ir<%val3> = add ir<%v2>, ir<%val2>, ir<%c3> -; CHECK-NEXT: Successor(s): latch -; CHECK-EMPTY: -; CHECK-NEXT: latch: -; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = not ir<%c3> -; CHECK-NEXT: BLEND ir<%result> = ir<%val3>/ir<%c3> ir<%val2>/vp<[[VP9]]> -; CHECK-NEXT: EMIT ir<%out> = getelementptr inbounds ir<%dst>, ir<%iv> -; CHECK-NEXT: EMIT store ir<%result>, ir<%out> -; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1> -; CHECK-NEXT: EMIT ir<%exitcond> = icmp eq ir<%iv.next>, ir<%n> -; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]> -; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]> -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; -; TMP2-LABEL: VPlan for loop in 'test' -; TMP2: VPlan ' for UF>=1' { -; TMP2-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF -; TMP2-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF -; TMP2-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count -; TMP2-NEXT: Live-in ir<%n> = original trip-count -; TMP2-EMPTY: -; TMP2-NEXT: ir-bb<entry>: -; TMP2-NEXT: Successor(s): scalar.ph, vector.ph -; TMP2-EMPTY: -; TMP2-NEXT: vector.ph: -; TMP2-NEXT: Successor(s): vector loop -; TMP2-EMPTY: -; TMP2-NEXT: <x1> vector loop: { -; TMP2-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV -; TMP2-NEXT: vp<[[VP4:%[0-9]+]]> = HEADER-MASK -; TMP2-EMPTY: -; TMP2-NEXT: vector.body: -; TMP2-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]> -; TMP2-NEXT: Successor(s): vector.body.split -; TMP2-EMPTY: -; TMP2-NEXT: vector.body.split: -; TMP2-NEXT: Successor(s): check.cond1 -; TMP2-EMPTY: -; TMP2-NEXT: check.cond1: -; TMP2-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = not ir<%c1> -; TMP2-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP5]]> -; TMP2-NEXT: EMIT ir<%not.c2> = xor ir<%c2>, ir<true>, vp<[[VP6]]> -; TMP2-NEXT: EMIT ir<%cond1> = or ir<%c1>, ir<%not.c2>, vp<[[VP6]]> -; TMP2-NEXT: Successor(s): load.v0 -; TMP2-EMPTY: -; TMP2-NEXT: load.v0: -; TMP2-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c1> -; TMP2-NEXT: EMIT ir<%gep0> = getelementptr inbounds ir<%src0>, ir<%iv> -; TMP2-NEXT: EMIT-SCALAR ir<%v0> = load ir<%gep0>, vp<[[VP7]]> -; TMP2-NEXT: Successor(s): load.v1 -; TMP2-EMPTY: -; TMP2-NEXT: load.v1: -; TMP2-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP5]]>, ir<%cond1> -; TMP2-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP8]]> -; TMP2-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = or ir<%c1>, vp<[[VP8]]> -; TMP2-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP10]]> -; TMP2-NEXT: BLEND ir<%val0> = ir<%v0>/vp<[[VP7]]> ir<0>/vp<[[VP9]]> -; TMP2-NEXT: EMIT ir<%gep1> = getelementptr inbounds ir<%src1>, ir<%iv> -; TMP2-NEXT: EMIT-SCALAR ir<%v1> = load ir<%gep1>, vp<[[VP11]]> -; TMP2-NEXT: EMIT ir<%val1> = add ir<%v1>, ir<%val0>, vp<[[VP11]]> -; TMP2-NEXT: Successor(s): load.v2.check -; TMP2-EMPTY: -; TMP2-NEXT: load.v2.check: -; TMP2-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = not ir<%cond1> -; TMP2-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = logical-and vp<[[VP5]]>, vp<[[VP12]]> -; TMP2-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP13]]> -; TMP2-NEXT: BLEND ir<%val2> = ir<%val1>/vp<[[VP11]]> ir<0>/vp<[[VP14]]> -; TMP2-NEXT: Successor(s): load.v2 -; TMP2-EMPTY: -; TMP2-NEXT: load.v2: -; TMP2-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = logical-and vp<[[VP4]]>, ir<%c3> -; TMP2-NEXT: EMIT ir<%gep2> = getelementptr inbounds ir<%src2>, ir<%iv> -; TMP2-NEXT: EMIT-SCALAR ir<%v2> = load ir<%gep2>, vp<[[VP15]]> -; TMP2-NEXT: EMIT ir<%val3> = add ir<%v2>, ir<%val2>, vp<[[VP15]]> -; TMP2-NEXT: Successor(s): latch -; TMP2-EMPTY: -; TMP2-NEXT: latch: -; TMP2-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = not ir<%c3> -; TMP2-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP16]]> -; TMP2-NEXT: BLEND ir<%result> = ir<%val3>/vp<[[VP15]]> ir<%val2>/vp<[[VP17]]> -; TMP2-NEXT: EMIT ir<%out> = getelementptr inbounds ir<%dst>, ir<%iv> -; TMP2-NEXT: EMIT store ir<%result>, ir<%out>, vp<[[VP4]]> -; TMP2-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>, vp<[[VP4]]> -; TMP2-NEXT: EMIT ir<%exitcond> = icmp eq ir<%iv.next>, ir<%n>, vp<[[VP4]]> -; TMP2-NEXT: Successor(s): vector.latch -; TMP2-EMPTY: -; TMP2-NEXT: vector.latch: -; TMP2-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]> -; TMP2-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]> -; TMP2-NEXT: No successors -; TMP2-NEXT: } -; TMP2-NEXT: Successor(s): middle.block -; TMP2-EMPTY: -; TMP2-NEXT: middle.block: -; TMP2-NEXT: EMIT vp<[[VP19:%[0-9]+]]> = exiting-iv-value ir<%iv> -; TMP2-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]> -; TMP2-NEXT: EMIT branch-on-cond ir<true> -; TMP2-NEXT: Successor(s): ir-bb<exit>, scalar.ph -; TMP2-EMPTY: -; TMP2-NEXT: ir-bb<exit>: -; TMP2-NEXT: No successors -; TMP2-EMPTY: -; TMP2-NEXT: scalar.ph: -; TMP2-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP19]]>, middle.block ], [ ir<0>, ir-bb<entry> ] -; TMP2-NEXT: Successor(s): ir-bb<loop> -; TMP2-EMPTY: -; TMP2-NEXT: ir-bb<loop>: -; TMP2-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ] (extra operand: vp<%bc.resume.val> from scalar.ph) -; TMP2-NEXT: No successors -; TMP2-NEXT: } -; -entry: - br label %loop - -loop: - %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ] - br i1 %c1, label %load.v0, label %check.cond1 - -check.cond1: - %not.c2 = xor i1 %c2, true - %cond1 = or i1 %c1, %not.c2 - br i1 %cond1, label %load.v1, label %load.v2.check - -load.v0: - %gep0 = getelementptr inbounds i32, ptr %src0, i64 %iv - %v0 = load i32, ptr %gep0, align 4 - br label %load.v1 - -load.v1: - %val0 = phi i32 [ %v0, %load.v0 ], [ 0, %check.cond1 ] - %gep1 = getelementptr inbounds i32, ptr %src1, i64 %iv - %v1 = load i32, ptr %gep1, align 4 - %val1 = add i32 %v1, %val0 - br label %load.v2.check - -load.v2.check: - %val2 = phi i32 [ %val1, %load.v1 ], [ 0, %check.cond1 ] - br i1 %c3, label %load.v2, label %latch - -load.v2: - %gep2 = getelementptr inbounds i32, ptr %src2, i64 %iv - %v2 = load i32, ptr %gep2, align 4 - %val3 = add i32 %v2, %val2 - br label %latch - -latch: - %result = phi i32 [ %val3, %load.v2 ], [ %val2, %load.v2.check ] - %out = getelementptr inbounds i32, ptr %dst, i64 %iv - store i32 %result, ptr %out, align 4 - %iv.next = add nuw nsw i64 %iv, 1 - %exitcond = icmp eq i64 %iv.next, %n - br i1 %exitcond, label %exit, label %loop - -exit: - ret void -} _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
