https://github.com/MacDue created https://github.com/llvm/llvm-project/pull/220961
If the user of the original wide mask is an `extractelement` and the index is known to be within the first segment of the predicate-as-counter, replace it with `extractelement(pext(counter, 0))`. This avoids materializing the vector mask and produces a form that can be folded into a conditional branch when the predicate-as-counter is produced by a `whilelo`. Assisted-by: Codex --- <sub>Stack created with <a href="https://github.com/github/gh-stack";>GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback";>Give Feedback 💬</a></sub> >From c74d1e415b0d37d74987ff693f2278b80df9eb8b Mon Sep 17 00:00:00 2001 From: Benjamin Maxwell <[email protected]> Date: Thu, 3 Sep 2026 13:36:55 +0000 Subject: [PATCH] [AArch64] Avoid materializing full masks for extractelement users of predicate-as-counter If the user of the original wide mask is an `extractelement` and the index is known to be within the first segment of the predicate-as-counter, replace it with `extractelement(pext(counter, 0))`. This avoids materializing the vector mask and produces a form that can be folded into a conditional branch when the predicate-as-counter is produced by a `whilelo`. Assisted-by: Codex --- .../AArch64PredicateAsCounterLoopRewrites.cpp | 40 +++ .../predicate-as-counter-loop-rewrites.ll | 237 +++++++----------- 2 files changed, 128 insertions(+), 149 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64PredicateAsCounterLoopRewrites.cpp b/llvm/lib/Target/AArch64/AArch64PredicateAsCounterLoopRewrites.cpp index b701e6c176146..fa855d87793d3 100644 --- a/llvm/lib/Target/AArch64/AArch64PredicateAsCounterLoopRewrites.cpp +++ b/llvm/lib/Target/AArch64/AArch64PredicateAsCounterLoopRewrites.cpp @@ -228,6 +228,42 @@ static Value *createWhileLO(IRBuilder<> &Builder, unsigned ElementSizeInBits, return Builder.CreateCall( WhileLO, {Start, End, Builder.getInt32(VectorScale)}, "pac.mask"); } + +/// If \p UserI is an extractelement use of the original loop mask, attempt to +/// rewrite it to `extractelement(pext(count, 0), idx)` if the extract index is +/// known to be within the first mask section (which means pext index = 0). If +/// the user of the extract is a branch and the source of the mask is a whilelo, +/// this form can be optimized into checking the status flags. +static bool tryRewriteExtractElement(Instruction &UserI, + const MaskRewriteCandidate &C, + Value *Count) { + auto *EEI = dyn_cast<ExtractElementInst>(&UserI); + if (!EEI) + return false; + + ElementCount LegalEC = getSVEElementCount(C.ElementSizeInBits); + auto *Idx = dyn_cast<ConstantInt>(EEI->getIndexOperand()); + if (!Idx || Idx->getValue().uge(LegalEC.getKnownMinValue())) + return false; + + IRBuilder<> Builder(EEI); + Builder.SetCurrentDebugLocation(EEI->getDebugLoc()); + + Module *M = EEI->getModule(); + FunctionCallee PExt = Intrinsic::getOrInsertDeclaration( + M, Intrinsic::aarch64_sve_pext, + {VectorType::get(Builder.getInt1Ty(), LegalEC)}); + auto *ExtractMask = + Builder.CreateCall(PExt, {Count, Builder.getInt32(0)}, "pac.pext"); + + Value *Extracted = Builder.CreateExtractElement( + ExtractMask, EEI->getIndexOperand(), EEI->getName() + ".pac"); + + EEI->replaceAllUsesWith(Extracted); + EEI->eraseFromParent(); + return true; +} + class AArch64PredicateAsCounterLoopRewrites : public LoopPass { public: static char ID; @@ -427,6 +463,10 @@ bool AArch64PredicateAsCounterLoopRewrites::rewriteCandidate( Value *WideMask = nullptr; for (Use *U : UsesToRewrite) { + auto *UserI = cast<Instruction>(U->getUser()); + if (tryRewriteExtractElement(*UserI, C, Count)) + continue; + if (!WideMask) { BasicBlock::iterator InsertPt = OldMask->getIterator(); if (isa<PHINode>(OldMask)) diff --git a/llvm/test/CodeGen/AArch64/predicate-as-counter-loop-rewrites.ll b/llvm/test/CodeGen/AArch64/predicate-as-counter-loop-rewrites.ll index b064c81107795..dca70b3555b7d 100644 --- a/llvm/test/CodeGen/AArch64/predicate-as-counter-loop-rewrites.ll +++ b/llvm/test/CodeGen/AArch64/predicate-as-counter-loop-rewrites.ll @@ -11,31 +11,24 @@ define void @rewrite_masked_load_store_i64_vlx4(ptr %x, i64 %n) #0 { ; CHECK-NEXT: whilelo pn8.d, xzr, x1, vlx4 ; CHECK-NEXT: .LBB0_1: // %loop ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: pext { p2.d, p3.d }, pn8[1] -; CHECK-NEXT: pext { p0.d, p1.d }, pn8[0] +; CHECK-NEXT: pext { p0.d, p1.d }, pn8[1] +; CHECK-NEXT: pext { p2.d, p3.d }, pn8[0] ; CHECK-NEXT: whilelo pn8.d, x8, x1, vlx4 -; CHECK-NEXT: ld1d { z0.d }, p3/z, [x0, #3, mul vl] -; CHECK-NEXT: pext { p4.d, p5.d }, pn8[0] -; CHECK-NEXT: ld1d { z1.d }, p2/z, [x0, #2, mul vl] -; CHECK-NEXT: uzp1 p4.s, p4.s, p5.s -; CHECK-NEXT: pext { p5.d, p6.d }, pn8[1] -; CHECK-NEXT: ld1d { z2.d }, p1/z, [x0, #1, mul vl] -; CHECK-NEXT: uzp1 p5.s, p5.s, p6.s -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0] +; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0, #3, mul vl] +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl] +; CHECK-NEXT: ld1d { z2.d }, p3/z, [x0, #1, mul vl] +; CHECK-NEXT: ld1d { z3.d }, p2/z, [x0] ; CHECK-NEXT: inch x8 ; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1 ; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1 -; CHECK-NEXT: uzp1 p4.h, p4.h, p5.h ; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1 ; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1 -; CHECK-NEXT: st1d { z0.d }, p3, [x0, #3, mul vl] -; CHECK-NEXT: mov z0.h, p4/z, #1 // =0x1 -; CHECK-NEXT: st1d { z1.d }, p2, [x0, #2, mul vl] -; CHECK-NEXT: st1d { z2.d }, p1, [x0, #1, mul vl] -; CHECK-NEXT: st1d { z3.d }, p0, [x0] +; CHECK-NEXT: st1d { z0.d }, p1, [x0, #3, mul vl] +; CHECK-NEXT: st1d { z1.d }, p0, [x0, #2, mul vl] +; CHECK-NEXT: st1d { z2.d }, p3, [x0, #1, mul vl] +; CHECK-NEXT: st1d { z3.d }, p2, [x0] ; CHECK-NEXT: incb x0, all, mul #4 -; CHECK-NEXT: fmov w9, s0 -; CHECK-NEXT: tbnz w9, #0, .LBB0_1 +; CHECK-NEXT: b.mi .LBB0_1 ; CHECK-NEXT: // %bb.2: // %exit ; CHECK-NEXT: ret entry: @@ -71,31 +64,24 @@ define void @rewrite_masked_load_store_i64_multi_pred_header(ptr %x, i64 %n, i1 ; CHECK-NEXT: whilelo pn8.d, xzr, x1, vlx4 ; CHECK-NEXT: .LBB1_1: // %loop ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: pext { p2.d, p3.d }, pn8[1] -; CHECK-NEXT: pext { p0.d, p1.d }, pn8[0] +; CHECK-NEXT: pext { p0.d, p1.d }, pn8[1] +; CHECK-NEXT: pext { p2.d, p3.d }, pn8[0] ; CHECK-NEXT: whilelo pn8.d, x8, x1, vlx4 -; CHECK-NEXT: ld1d { z0.d }, p3/z, [x0, #3, mul vl] -; CHECK-NEXT: pext { p4.d, p5.d }, pn8[0] -; CHECK-NEXT: ld1d { z1.d }, p2/z, [x0, #2, mul vl] -; CHECK-NEXT: uzp1 p4.s, p4.s, p5.s -; CHECK-NEXT: pext { p5.d, p6.d }, pn8[1] -; CHECK-NEXT: ld1d { z2.d }, p1/z, [x0, #1, mul vl] -; CHECK-NEXT: uzp1 p5.s, p5.s, p6.s -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0] +; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0, #3, mul vl] +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl] +; CHECK-NEXT: ld1d { z2.d }, p3/z, [x0, #1, mul vl] +; CHECK-NEXT: ld1d { z3.d }, p2/z, [x0] ; CHECK-NEXT: inch x8 ; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1 ; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1 -; CHECK-NEXT: uzp1 p4.h, p4.h, p5.h ; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1 ; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1 -; CHECK-NEXT: st1d { z0.d }, p3, [x0, #3, mul vl] -; CHECK-NEXT: mov z0.h, p4/z, #1 // =0x1 -; CHECK-NEXT: st1d { z1.d }, p2, [x0, #2, mul vl] -; CHECK-NEXT: st1d { z2.d }, p1, [x0, #1, mul vl] -; CHECK-NEXT: st1d { z3.d }, p0, [x0] +; CHECK-NEXT: st1d { z0.d }, p1, [x0, #3, mul vl] +; CHECK-NEXT: st1d { z1.d }, p0, [x0, #2, mul vl] +; CHECK-NEXT: st1d { z2.d }, p3, [x0, #1, mul vl] +; CHECK-NEXT: st1d { z3.d }, p2, [x0] ; CHECK-NEXT: incb x0, all, mul #4 -; CHECK-NEXT: fmov w9, s0 -; CHECK-NEXT: tbnz w9, #0, .LBB1_1 +; CHECK-NEXT: b.mi .LBB1_1 ; CHECK-NEXT: // %bb.2: // %exit ; CHECK-NEXT: ret entry: @@ -136,54 +122,40 @@ define void @shared_start_mask_between_loops(ptr %x, ptr %y, i64 %n) #0 { ; CHECK-NEXT: mov x9, x8 ; CHECK-NEXT: .LBB2_1: // %loop1 ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: pext { p2.d, p3.d }, pn9[1] -; CHECK-NEXT: pext { p0.d, p1.d }, pn9[0] +; CHECK-NEXT: pext { p0.d, p1.d }, pn9[1] +; CHECK-NEXT: pext { p2.d, p3.d }, pn9[0] ; CHECK-NEXT: whilelo pn9.d, x9, x2, vlx4 -; CHECK-NEXT: ld1d { z0.d }, p3/z, [x0, #3, mul vl] -; CHECK-NEXT: pext { p4.d, p5.d }, pn9[0] -; CHECK-NEXT: ld1d { z1.d }, p2/z, [x0, #2, mul vl] -; CHECK-NEXT: uzp1 p4.s, p4.s, p5.s -; CHECK-NEXT: pext { p5.d, p6.d }, pn9[1] -; CHECK-NEXT: ld1d { z2.d }, p1/z, [x0, #1, mul vl] -; CHECK-NEXT: uzp1 p5.s, p5.s, p6.s -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0] +; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0, #3, mul vl] +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl] +; CHECK-NEXT: ld1d { z2.d }, p3/z, [x0, #1, mul vl] +; CHECK-NEXT: ld1d { z3.d }, p2/z, [x0] ; CHECK-NEXT: inch x9 ; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1 ; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1 -; CHECK-NEXT: uzp1 p4.h, p4.h, p5.h ; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1 ; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1 -; CHECK-NEXT: st1d { z0.d }, p3, [x0, #3, mul vl] -; CHECK-NEXT: mov z0.h, p4/z, #1 // =0x1 -; CHECK-NEXT: st1d { z1.d }, p2, [x0, #2, mul vl] -; CHECK-NEXT: st1d { z2.d }, p1, [x0, #1, mul vl] -; CHECK-NEXT: st1d { z3.d }, p0, [x0] +; CHECK-NEXT: st1d { z0.d }, p1, [x0, #3, mul vl] +; CHECK-NEXT: st1d { z1.d }, p0, [x0, #2, mul vl] +; CHECK-NEXT: st1d { z2.d }, p3, [x0, #1, mul vl] +; CHECK-NEXT: st1d { z3.d }, p2, [x0] ; CHECK-NEXT: incb x0, all, mul #4 -; CHECK-NEXT: fmov w10, s0 -; CHECK-NEXT: tbnz w10, #0, .LBB2_1 +; CHECK-NEXT: b.mi .LBB2_1 ; CHECK-NEXT: .LBB2_2: // %loop2 ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: pext { p0.d, p1.d }, pn8[0] -; CHECK-NEXT: pext { p2.d, p3.d }, pn8[1] +; CHECK-NEXT: pext { p0.d, p1.d }, pn8[1] +; CHECK-NEXT: pext { p2.d, p3.d }, pn8[0] ; CHECK-NEXT: whilelo pn8.d, x8, x2, vlx4 -; CHECK-NEXT: pext { p4.d, p5.d }, pn8[0] -; CHECK-NEXT: ld1d { z0.d }, p3/z, [x1, #3, mul vl] -; CHECK-NEXT: ld1d { z1.d }, p2/z, [x1, #2, mul vl] -; CHECK-NEXT: uzp1 p4.s, p4.s, p5.s -; CHECK-NEXT: pext { p5.d, p6.d }, pn8[1] -; CHECK-NEXT: ld1d { z2.d }, p1/z, [x1, #1, mul vl] -; CHECK-NEXT: uzp1 p5.s, p5.s, p6.s -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x1] +; CHECK-NEXT: ld1d { z0.d }, p1/z, [x1, #3, mul vl] +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1, #2, mul vl] +; CHECK-NEXT: ld1d { z2.d }, p3/z, [x1, #1, mul vl] +; CHECK-NEXT: ld1d { z3.d }, p2/z, [x1] ; CHECK-NEXT: inch x8 -; CHECK-NEXT: st1d { z0.d }, p3, [x1, #3, mul vl] -; CHECK-NEXT: uzp1 p4.h, p4.h, p5.h -; CHECK-NEXT: st1d { z1.d }, p2, [x1, #2, mul vl] -; CHECK-NEXT: st1d { z2.d }, p1, [x1, #1, mul vl] -; CHECK-NEXT: mov z0.h, p4/z, #1 // =0x1 -; CHECK-NEXT: st1d { z3.d }, p0, [x1] +; CHECK-NEXT: st1d { z0.d }, p1, [x1, #3, mul vl] +; CHECK-NEXT: st1d { z1.d }, p0, [x1, #2, mul vl] +; CHECK-NEXT: st1d { z2.d }, p3, [x1, #1, mul vl] +; CHECK-NEXT: st1d { z3.d }, p2, [x1] ; CHECK-NEXT: incb x1, all, mul #4 -; CHECK-NEXT: fmov w9, s0 -; CHECK-NEXT: tbnz w9, #0, .LBB2_2 +; CHECK-NEXT: b.mi .LBB2_2 ; CHECK-NEXT: // %bb.3: // %exit ; CHECK-NEXT: ret entry: @@ -235,20 +207,17 @@ define void @rewrite_masked_load_store_i8_i32_induction(ptr %x, i32 %n) #0 { ; CHECK-NEXT: pext { p0.b, p1.b }, pn8[0] ; CHECK-NEXT: mov w12, w9 ; CHECK-NEXT: decb x10, all, mul #2 -; CHECK-NEXT: whilelo pn8.b, x12, x8, vlx2 ; CHECK-NEXT: incb x9, all, mul #2 -; CHECK-NEXT: pext { p2.b, p3.b }, pn8[0] -; CHECK-NEXT: mov z2.b, p2/z, #1 // =0x1 +; CHECK-NEXT: whilelo pn8.b, x12, x8, vlx2 ; CHECK-NEXT: sxtw x10, w10 -; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x10] ; CHECK-NEXT: add x11, x0, x10 +; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x10] ; CHECK-NEXT: ld1b { z1.b }, p1/z, [x11, #1, mul vl] ; CHECK-NEXT: add z0.b, z0.b, #1 // =0x1 ; CHECK-NEXT: add z1.b, z1.b, #1 // =0x1 ; CHECK-NEXT: st1b { z0.b }, p0, [x0, x10] -; CHECK-NEXT: fmov w10, s2 ; CHECK-NEXT: st1b { z1.b }, p1, [x11, #1, mul vl] -; CHECK-NEXT: tbnz w10, #0, .LBB3_1 +; CHECK-NEXT: b.mi .LBB3_1 ; CHECK-NEXT: // %bb.2: // %exit ; CHECK-NEXT: ret entry: @@ -289,16 +258,12 @@ define void @rewrite_masked_load_store_i32_vlx2(ptr %x, i64 %n) #0 { ; CHECK-NEXT: inch x8 ; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0, #1, mul vl] ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: pext { p2.s, p3.s }, pn8[0] -; CHECK-NEXT: uzp1 p2.h, p2.h, p3.h ; CHECK-NEXT: add z0.s, z0.s, #1 // =0x1 ; CHECK-NEXT: add z1.s, z1.s, #1 // =0x1 -; CHECK-NEXT: mov z2.h, p2/z, #1 // =0x1 ; CHECK-NEXT: st1w { z0.s }, p1, [x0, #1, mul vl] -; CHECK-NEXT: fmov w9, s2 ; CHECK-NEXT: st1w { z1.s }, p0, [x0] ; CHECK-NEXT: incb x0, all, mul #2 -; CHECK-NEXT: tbnz w9, #0, .LBB4_1 +; CHECK-NEXT: b.mi .LBB4_1 ; CHECK-NEXT: // %bb.2: // %exit ; CHECK-NEXT: ret entry: @@ -335,33 +300,26 @@ define void @mixed_data_vector_types(ptr %x, ptr %y, i64 %n) #0 { ; CHECK-NEXT: .LBB5_1: // %loop ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: pext { p0.d, p1.d }, pn8[0] +; CHECK-NEXT: pext { p2.d, p3.d }, pn8[1] ; CHECK-NEXT: add x9, x0, x8, lsl #3 +; CHECK-NEXT: uzp1 p4.s, p0.s, p1.s ; CHECK-NEXT: add x10, x1, x8, lsl #2 -; CHECK-NEXT: uzp1 p2.s, p0.s, p1.s ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; CHECK-NEXT: ld1d { z5.d }, p1/z, [x9, #1, mul vl] -; CHECK-NEXT: ld1w { z1.s }, p2/z, [x1, x8, lsl #2] +; CHECK-NEXT: uzp1 p5.s, p2.s, p3.s +; CHECK-NEXT: ld1d { z1.d }, p3/z, [x9, #3, mul vl] +; CHECK-NEXT: ld1d { z3.d }, p2/z, [x9, #2, mul vl] +; CHECK-NEXT: ld1w { z2.s }, p4/z, [x1, x8, lsl #2] ; CHECK-NEXT: inch x8 -; CHECK-NEXT: pext { p2.d, p3.d }, pn8[1] -; CHECK-NEXT: ld1d { z3.d }, p3/z, [x9, #3, mul vl] -; CHECK-NEXT: ld1d { z4.d }, p2/z, [x9, #2, mul vl] +; CHECK-NEXT: ld1d { z4.d }, p1/z, [x9, #1, mul vl] +; CHECK-NEXT: ld1w { z5.s }, p5/z, [x10, #1, mul vl] ; CHECK-NEXT: whilelo pn8.d, x8, x2, vlx4 -; CHECK-NEXT: pext { p4.d, p5.d }, pn8[0] -; CHECK-NEXT: uzp1 p0.s, p4.s, p5.s -; CHECK-NEXT: pext { p4.d, p5.d }, pn8[1] -; CHECK-NEXT: uzp1 p4.s, p4.s, p5.s -; CHECK-NEXT: uzp1 p0.h, p0.h, p4.h -; CHECK-NEXT: uzp1 p4.s, p2.s, p3.s -; CHECK-NEXT: mov z2.h, p0/z, #1 // =0x1 -; CHECK-NEXT: ld1w { z6.s }, p4/z, [x10, #1, mul vl] -; CHECK-NEXT: fmov w9, s2 ; CHECK-NEXT: // fake_use: $z0 -; CHECK-NEXT: // fake_use: $z5 ; CHECK-NEXT: // fake_use: $z4 ; CHECK-NEXT: // fake_use: $z3 ; CHECK-NEXT: // fake_use: $z1 -; CHECK-NEXT: // fake_use: $z6 -; CHECK-NEXT: tbnz w9, #0, .LBB5_1 +; CHECK-NEXT: // fake_use: $z2 +; CHECK-NEXT: // fake_use: $z5 +; CHECK-NEXT: b.mi .LBB5_1 ; CHECK-NEXT: // %bb.2: // %exit ; CHECK-NEXT: ret entry: @@ -402,28 +360,24 @@ define void @prefer_more_common_masked_access_size(ptr %x16, ptr %y32, i64 %n) # ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: pext { p0.h, p1.h }, pn8[0] ; CHECK-NEXT: add x9, x0, x8, lsl #1 +; CHECK-NEXT: add x10, x1, x8, lsl #2 ; CHECK-NEXT: punpklo p2.h, p0.b ; CHECK-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1] ; CHECK-NEXT: st1h { z0.h }, p1, [x9, #1, mul vl] -; CHECK-NEXT: add x9, x1, x8, lsl #2 -; CHECK-NEXT: punpkhi p0.h, p0.b +; CHECK-NEXT: punpkhi p3.h, p1.b +; CHECK-NEXT: punpklo p4.h, p1.b ; CHECK-NEXT: ld1w { z1.s }, p2/z, [x1, x8, lsl #2] ; CHECK-NEXT: incb x8 -; CHECK-NEXT: punpkhi p2.h, p1.b -; CHECK-NEXT: punpklo p1.h, p1.b -; CHECK-NEXT: ld1w { z5.s }, p0/z, [x9, #1, mul vl] -; CHECK-NEXT: ld1w { z3.s }, p2/z, [x9, #3, mul vl] +; CHECK-NEXT: punpkhi p0.h, p0.b +; CHECK-NEXT: ld1w { z2.s }, p3/z, [x10, #3, mul vl] +; CHECK-NEXT: ld1w { z3.s }, p4/z, [x10, #2, mul vl] +; CHECK-NEXT: ld1w { z4.s }, p0/z, [x10, #1, mul vl] ; CHECK-NEXT: whilelo pn8.h, x8, x2, vlx2 -; CHECK-NEXT: ld1w { z4.s }, p1/z, [x9, #2, mul vl] -; CHECK-NEXT: pext { p3.h, p4.h }, pn8[0] -; CHECK-NEXT: uzp1 p3.b, p3.b, p4.b -; CHECK-NEXT: mov z2.b, p3/z, #1 // =0x1 -; CHECK-NEXT: fmov w9, s2 ; CHECK-NEXT: // fake_use: $z1 -; CHECK-NEXT: // fake_use: $z5 ; CHECK-NEXT: // fake_use: $z4 ; CHECK-NEXT: // fake_use: $z3 -; CHECK-NEXT: tbnz w9, #0, .LBB6_1 +; CHECK-NEXT: // fake_use: $z2 +; CHECK-NEXT: b.mi .LBB6_1 ; CHECK-NEXT: // %bb.2: // %exit ; CHECK-NEXT: ret entry: @@ -466,28 +420,21 @@ define void @prefer_larger_access_size_on_tie(ptr %x16, ptr %y32, i64 %n) #0 { ; CHECK-NEXT: pext { p2.s, p3.s }, pn8[1] ; CHECK-NEXT: add x9, x0, x8, lsl #1 ; CHECK-NEXT: uzp1 p4.h, p0.h, p1.h +; CHECK-NEXT: add x10, x1, x8, lsl #2 ; CHECK-NEXT: uzp1 p5.h, p2.h, p3.h ; CHECK-NEXT: st1h { z0.h }, p4, [x0, x8, lsl #1] ; CHECK-NEXT: st1h { z0.h }, p5, [x9, #1, mul vl] -; CHECK-NEXT: add x9, x1, x8, lsl #2 ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2] ; CHECK-NEXT: incb x8 -; CHECK-NEXT: ld1w { z3.s }, p3/z, [x9, #3, mul vl] -; CHECK-NEXT: ld1w { z4.s }, p2/z, [x9, #2, mul vl] -; CHECK-NEXT: ld1w { z5.s }, p1/z, [x9, #1, mul vl] +; CHECK-NEXT: ld1w { z2.s }, p3/z, [x10, #3, mul vl] +; CHECK-NEXT: ld1w { z3.s }, p2/z, [x10, #2, mul vl] +; CHECK-NEXT: ld1w { z4.s }, p1/z, [x10, #1, mul vl] ; CHECK-NEXT: whilelo pn8.s, x8, x2, vlx4 -; CHECK-NEXT: pext { p4.s, p5.s }, pn8[0] -; CHECK-NEXT: uzp1 p0.h, p4.h, p5.h -; CHECK-NEXT: pext { p4.s, p5.s }, pn8[1] -; CHECK-NEXT: uzp1 p4.h, p4.h, p5.h -; CHECK-NEXT: uzp1 p0.b, p0.b, p4.b -; CHECK-NEXT: mov z2.b, p0/z, #1 // =0x1 -; CHECK-NEXT: fmov w9, s2 ; CHECK-NEXT: // fake_use: $z1 -; CHECK-NEXT: // fake_use: $z5 ; CHECK-NEXT: // fake_use: $z4 ; CHECK-NEXT: // fake_use: $z3 -; CHECK-NEXT: tbnz w9, #0, .LBB7_1 +; CHECK-NEXT: // fake_use: $z2 +; CHECK-NEXT: b.mi .LBB7_1 ; CHECK-NEXT: // %bb.2: // %exit ; CHECK-NEXT: ret entry: @@ -527,11 +474,10 @@ define void @rewrite_extractelement_within_first_section_i32(ptr %x, i64 %n) #0 ; CHECK-NEXT: pext { p0.s, p1.s }, pn8[0] ; CHECK-NEXT: whilelo pn8.s, x8, x1, vlx2 ; CHECK-NEXT: inch x8 -; CHECK-NEXT: pext { p2.s, p3.s }, pn8[0] +; CHECK-NEXT: pext p2.s, pn8[0] ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: uzp1 p2.h, p2.h, p3.h -; CHECK-NEXT: mov z0.h, p2/z, #1 // =0x1 -; CHECK-NEXT: umov w9, v0.h[3] +; CHECK-NEXT: mov z0.s, p2/z, #1 // =0x1 +; CHECK-NEXT: mov w9, v0.s[3] ; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0, #1, mul vl] ; CHECK-NEXT: incb x0, all, mul #2 ; CHECK-NEXT: // fake_use: $z1 @@ -571,31 +517,24 @@ define void @masked_load_passthru_non_poison(ptr %x, i64 %n) #0 { ; CHECK-NEXT: whilelo pn8.d, xzr, x1, vlx4 ; CHECK-NEXT: .LBB9_1: // %loop ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: pext { p2.d, p3.d }, pn8[1] -; CHECK-NEXT: pext { p0.d, p1.d }, pn8[0] +; CHECK-NEXT: pext { p0.d, p1.d }, pn8[1] +; CHECK-NEXT: pext { p2.d, p3.d }, pn8[0] ; CHECK-NEXT: whilelo pn8.d, x8, x1, vlx4 -; CHECK-NEXT: ld1d { z0.d }, p3/z, [x0, #3, mul vl] -; CHECK-NEXT: pext { p4.d, p5.d }, pn8[0] -; CHECK-NEXT: ld1d { z1.d }, p2/z, [x0, #2, mul vl] -; CHECK-NEXT: uzp1 p4.s, p4.s, p5.s -; CHECK-NEXT: pext { p5.d, p6.d }, pn8[1] -; CHECK-NEXT: ld1d { z2.d }, p1/z, [x0, #1, mul vl] -; CHECK-NEXT: uzp1 p5.s, p5.s, p6.s -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0] +; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0, #3, mul vl] +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl] +; CHECK-NEXT: ld1d { z2.d }, p3/z, [x0, #1, mul vl] +; CHECK-NEXT: ld1d { z3.d }, p2/z, [x0] ; CHECK-NEXT: inch x8 ; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1 ; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1 -; CHECK-NEXT: uzp1 p4.h, p4.h, p5.h ; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1 ; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1 -; CHECK-NEXT: st1d { z0.d }, p3, [x0, #3, mul vl] -; CHECK-NEXT: mov z0.h, p4/z, #1 // =0x1 -; CHECK-NEXT: st1d { z1.d }, p2, [x0, #2, mul vl] -; CHECK-NEXT: st1d { z2.d }, p1, [x0, #1, mul vl] -; CHECK-NEXT: st1d { z3.d }, p0, [x0] +; CHECK-NEXT: st1d { z0.d }, p1, [x0, #3, mul vl] +; CHECK-NEXT: st1d { z1.d }, p0, [x0, #2, mul vl] +; CHECK-NEXT: st1d { z2.d }, p3, [x0, #1, mul vl] +; CHECK-NEXT: st1d { z3.d }, p2, [x0] ; CHECK-NEXT: incb x0, all, mul #4 -; CHECK-NEXT: fmov w9, s0 -; CHECK-NEXT: tbnz w9, #0, .LBB9_1 +; CHECK-NEXT: b.mi .LBB9_1 ; CHECK-NEXT: // %bb.2: // %exit ; CHECK-NEXT: ret entry: _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
