https://github.com/MacDue created https://github.com/llvm/llvm-project/pull/220962
If the user of the original wide mask is a masked load or store intrinsic (matching the element size of the predicate-as-counter), rewrite it directly to a masked multi-vector load/store. This avoids materializing the vector mask and is easier to handle here than later (e.g. in SelectionDAG), since we do not need to match the concatenation of all `pext` segments of the predicate-as-counter. Assisted-by: Codex --- <sub>Stack created with <a href="https://github.com/github/gh-stack";>GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback";>Give Feedback 💬</a></sub> >From 96d064855f4e60c4101e172f7f070244190113be Mon Sep 17 00:00:00 2001 From: Benjamin Maxwell <[email protected]> Date: Thu, 3 Sep 2026 13:39:16 +0000 Subject: [PATCH] [AArch64] Use multi-vector intrinsics for masked load/store users of predicate-as-counter If the user of the original wide mask is a masked load or store intrinsic (matching the element size of the predicate-as-counter), rewrite it directly to a masked multi-vector load/store. This avoids materializing the vector mask and is easier to handle here than later (e.g. in SelectionDAG), since we do not need to match the concatenation of all `pext` segments of the predicate-as-counter. Assisted-by: Codex --- .../AArch64PredicateAsCounterLoopRewrites.cpp | 111 +++++++- .../predicate-as-counter-loop-rewrites.ll | 238 +++++++----------- 2 files changed, 206 insertions(+), 143 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64PredicateAsCounterLoopRewrites.cpp b/llvm/lib/Target/AArch64/AArch64PredicateAsCounterLoopRewrites.cpp index fa855d87793d3..7881751137425 100644 --- a/llvm/lib/Target/AArch64/AArch64PredicateAsCounterLoopRewrites.cpp +++ b/llvm/lib/Target/AArch64/AArch64PredicateAsCounterLoopRewrites.cpp @@ -229,6 +229,113 @@ static Value *createWhileLO(IRBuilder<> &Builder, unsigned ElementSizeInBits, WhileLO, {Start, End, Builder.getInt32(VectorScale)}, "pac.mask"); } +/// Returns the multi-vector load/store intrinsic ID for \p VectorScale. +static Intrinsic::ID getPNLoadStoreIntrinsic(unsigned VectorScale, + bool IsLoad) { + if (VectorScale == 2) + return IsLoad ? Intrinsic::aarch64_sve_ld1_pn_x2 + : Intrinsic::aarch64_sve_st1_pn_x2; + if (VectorScale == 4) + return IsLoad ? Intrinsic::aarch64_sve_ld1_pn_x4 + : Intrinsic::aarch64_sve_st1_pn_x4; + llvm_unreachable("unsupported predicate-as-counter scale"); +} + +/// If \p UserI is a masked-load user of the original loop mask, rewrite it to +/// a masked multi-vector load (using the predicate-as-counter) if the load +/// access size matches the predicate-as-counter element size. +static bool tryRewriteMaskedLoadUser(Instruction &UserI, + const MaskRewriteCandidate &C, + Value *Count) { + auto *II = dyn_cast<IntrinsicInst>(&UserI); + if (!II || II->getIntrinsicID() != Intrinsic::masked_load) + return false; + + unsigned ElementSizeInBits = + getScalarSizeInBits(II->getModule()->getDataLayout(), II->getType()); + if (ElementSizeInBits != C.ElementSizeInBits) + return false; + + if (!isa<PoisonValue, UndefValue>(II->getArgOperand(2))) + return false; + + IRBuilder<> Builder(II); + Builder.SetCurrentDebugLocation(II->getDebugLoc()); + + ElementCount LegalEC = getSVEElementCount(C.ElementSizeInBits); + Type *ScalarType = II->getType()->getScalarType(); + auto *WideDataTy = VectorType::get(ScalarType, LegalEC * C.VectorScale); + auto *LegalDataTy = VectorType::get(ScalarType, LegalEC); + + Module *M = II->getModule(); + FunctionCallee LD1 = Intrinsic::getOrInsertDeclaration( + M, getPNLoadStoreIntrinsic(C.VectorScale, /*IsLoad=*/true), + {LegalDataTy, II->getArgOperand(0)->getType()}); + auto *PNLoad = + Builder.CreateCall(LD1, {Count, II->getArgOperand(0)}, "pac.ld1"); + + // Copy pointer parameter attributes. + for (Attribute ParamAttr : II->getParamAttributes(0)) + PNLoad->addParamAttr(1, ParamAttr); + + // Concatenate all results into the original wide value. + Value *WideData = PoisonValue::get(WideDataTy); + for (unsigned Slice = 0; Slice != C.VectorScale; ++Slice) { + Value *Part = Builder.CreateExtractValue(PNLoad, Slice, "pac.data"); + WideData = Builder.CreateInsertVector(WideDataTy, WideData, Part, + Slice * LegalEC.getKnownMinValue(), + "pac.vec"); + } + + II->replaceAllUsesWith(WideData); + II->eraseFromParent(); + return true; +} + +/// If \p UserI is a masked-store user of the original loop mask, rewrite it to +/// a masked multi-vector store (using the predicate-as-counter) if the store +/// access size matches the predicate-as-counter element size. +static bool tryRewriteMaskedStoreUser(Instruction &UserI, + const MaskRewriteCandidate &C, + Value *Count) { + auto *II = dyn_cast<IntrinsicInst>(&UserI); + if (!II || II->getIntrinsicID() != Intrinsic::masked_store) + return false; + + unsigned ElementSizeInBits = getScalarSizeInBits( + II->getModule()->getDataLayout(), II->getArgOperand(0)->getType()); + if (ElementSizeInBits != C.ElementSizeInBits) + return false; + + IRBuilder<> Builder(II); + Builder.SetCurrentDebugLocation(II->getDebugLoc()); + + Type *ScalarType = II->getArgOperand(0)->getType()->getScalarType(); + ElementCount LegalEC = getSVEElementCount(C.ElementSizeInBits); + auto *LegalDataTy = VectorType::get(ScalarType, LegalEC); + + SmallVector<Value *, 6> StoreArgs; + for (unsigned Slice = 0; Slice != C.VectorScale; ++Slice) + StoreArgs.push_back(Builder.CreateExtractVector( + LegalDataTy, II->getArgOperand(0), Slice * LegalEC.getKnownMinValue(), + "pac.data")); + StoreArgs.push_back(Count); + StoreArgs.push_back(II->getArgOperand(1)); + + Module *M = II->getModule(); + FunctionCallee ST1 = Intrinsic::getOrInsertDeclaration( + M, getPNLoadStoreIntrinsic(C.VectorScale, /*IsLoad=*/false), + {LegalDataTy, II->getArgOperand(1)->getType()}); + auto *PNStore = Builder.CreateCall(ST1, StoreArgs); + + // Copy pointer parameter attributes. + for (Attribute ParamAttr : II->getParamAttributes(1)) + PNStore->addParamAttr(C.VectorScale + 1, ParamAttr); + + II->eraseFromParent(); + return true; +} + /// If \p UserI is an extractelement use of the original loop mask, attempt to /// rewrite it to `extractelement(pext(count, 0), idx)` if the extract index is /// known to be within the first mask section (which means pext index = 0). If @@ -464,7 +571,9 @@ bool AArch64PredicateAsCounterLoopRewrites::rewriteCandidate( Value *WideMask = nullptr; for (Use *U : UsesToRewrite) { auto *UserI = cast<Instruction>(U->getUser()); - if (tryRewriteExtractElement(*UserI, C, Count)) + if (tryRewriteMaskedLoadUser(*UserI, C, Count) || + tryRewriteMaskedStoreUser(*UserI, C, Count) || + tryRewriteExtractElement(*UserI, C, Count)) continue; if (!WideMask) { diff --git a/llvm/test/CodeGen/AArch64/predicate-as-counter-loop-rewrites.ll b/llvm/test/CodeGen/AArch64/predicate-as-counter-loop-rewrites.ll index dca70b3555b7d..f887048eb8ad5 100644 --- a/llvm/test/CodeGen/AArch64/predicate-as-counter-loop-rewrites.ll +++ b/llvm/test/CodeGen/AArch64/predicate-as-counter-loop-rewrites.ll @@ -11,22 +11,14 @@ define void @rewrite_masked_load_store_i64_vlx4(ptr %x, i64 %n) #0 { ; CHECK-NEXT: whilelo pn8.d, xzr, x1, vlx4 ; CHECK-NEXT: .LBB0_1: // %loop ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: pext { p0.d, p1.d }, pn8[1] -; CHECK-NEXT: pext { p2.d, p3.d }, pn8[0] +; CHECK-NEXT: ld1d { z0.d - z3.d }, pn8/z, [x0] +; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1 +; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1 +; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1 +; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1 +; CHECK-NEXT: st1d { z0.d - z3.d }, pn8, [x0] ; CHECK-NEXT: whilelo pn8.d, x8, x1, vlx4 -; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0, #3, mul vl] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl] -; CHECK-NEXT: ld1d { z2.d }, p3/z, [x0, #1, mul vl] -; CHECK-NEXT: ld1d { z3.d }, p2/z, [x0] ; CHECK-NEXT: inch x8 -; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1 -; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1 -; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1 -; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1 -; CHECK-NEXT: st1d { z0.d }, p1, [x0, #3, mul vl] -; CHECK-NEXT: st1d { z1.d }, p0, [x0, #2, mul vl] -; CHECK-NEXT: st1d { z2.d }, p3, [x0, #1, mul vl] -; CHECK-NEXT: st1d { z3.d }, p2, [x0] ; CHECK-NEXT: incb x0, all, mul #4 ; CHECK-NEXT: b.mi .LBB0_1 ; CHECK-NEXT: // %bb.2: // %exit @@ -64,22 +56,14 @@ define void @rewrite_masked_load_store_i64_multi_pred_header(ptr %x, i64 %n, i1 ; CHECK-NEXT: whilelo pn8.d, xzr, x1, vlx4 ; CHECK-NEXT: .LBB1_1: // %loop ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: pext { p0.d, p1.d }, pn8[1] -; CHECK-NEXT: pext { p2.d, p3.d }, pn8[0] +; CHECK-NEXT: ld1d { z0.d - z3.d }, pn8/z, [x0] +; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1 +; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1 +; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1 +; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1 +; CHECK-NEXT: st1d { z0.d - z3.d }, pn8, [x0] ; CHECK-NEXT: whilelo pn8.d, x8, x1, vlx4 -; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0, #3, mul vl] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl] -; CHECK-NEXT: ld1d { z2.d }, p3/z, [x0, #1, mul vl] -; CHECK-NEXT: ld1d { z3.d }, p2/z, [x0] ; CHECK-NEXT: inch x8 -; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1 -; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1 -; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1 -; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1 -; CHECK-NEXT: st1d { z0.d }, p1, [x0, #3, mul vl] -; CHECK-NEXT: st1d { z1.d }, p0, [x0, #2, mul vl] -; CHECK-NEXT: st1d { z2.d }, p3, [x0, #1, mul vl] -; CHECK-NEXT: st1d { z3.d }, p2, [x0] ; CHECK-NEXT: incb x0, all, mul #4 ; CHECK-NEXT: b.mi .LBB1_1 ; CHECK-NEXT: // %bb.2: // %exit @@ -122,38 +106,22 @@ define void @shared_start_mask_between_loops(ptr %x, ptr %y, i64 %n) #0 { ; CHECK-NEXT: mov x9, x8 ; CHECK-NEXT: .LBB2_1: // %loop1 ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: pext { p0.d, p1.d }, pn9[1] -; CHECK-NEXT: pext { p2.d, p3.d }, pn9[0] +; CHECK-NEXT: ld1d { z0.d - z3.d }, pn9/z, [x0] +; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1 +; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1 +; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1 +; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1 +; CHECK-NEXT: st1d { z0.d - z3.d }, pn9, [x0] ; CHECK-NEXT: whilelo pn9.d, x9, x2, vlx4 -; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0, #3, mul vl] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl] -; CHECK-NEXT: ld1d { z2.d }, p3/z, [x0, #1, mul vl] -; CHECK-NEXT: ld1d { z3.d }, p2/z, [x0] ; CHECK-NEXT: inch x9 -; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1 -; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1 -; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1 -; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1 -; CHECK-NEXT: st1d { z0.d }, p1, [x0, #3, mul vl] -; CHECK-NEXT: st1d { z1.d }, p0, [x0, #2, mul vl] -; CHECK-NEXT: st1d { z2.d }, p3, [x0, #1, mul vl] -; CHECK-NEXT: st1d { z3.d }, p2, [x0] ; CHECK-NEXT: incb x0, all, mul #4 ; CHECK-NEXT: b.mi .LBB2_1 ; CHECK-NEXT: .LBB2_2: // %loop2 ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: pext { p0.d, p1.d }, pn8[1] -; CHECK-NEXT: pext { p2.d, p3.d }, pn8[0] +; CHECK-NEXT: ld1d { z0.d - z3.d }, pn8/z, [x1] +; CHECK-NEXT: st1d { z0.d - z3.d }, pn8, [x1] ; CHECK-NEXT: whilelo pn8.d, x8, x2, vlx4 -; CHECK-NEXT: ld1d { z0.d }, p1/z, [x1, #3, mul vl] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1, #2, mul vl] -; CHECK-NEXT: ld1d { z2.d }, p3/z, [x1, #1, mul vl] -; CHECK-NEXT: ld1d { z3.d }, p2/z, [x1] ; CHECK-NEXT: inch x8 -; CHECK-NEXT: st1d { z0.d }, p1, [x1, #3, mul vl] -; CHECK-NEXT: st1d { z1.d }, p0, [x1, #2, mul vl] -; CHECK-NEXT: st1d { z2.d }, p3, [x1, #1, mul vl] -; CHECK-NEXT: st1d { z3.d }, p2, [x1] ; CHECK-NEXT: incb x1, all, mul #4 ; CHECK-NEXT: b.mi .LBB2_2 ; CHECK-NEXT: // %bb.3: // %exit @@ -204,19 +172,16 @@ define void @rewrite_masked_load_store_i8_i32_induction(ptr %x, i32 %n) #0 { ; CHECK-NEXT: .LBB3_1: // %loop ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: mov x10, x9 -; CHECK-NEXT: pext { p0.b, p1.b }, pn8[0] -; CHECK-NEXT: mov w12, w9 -; CHECK-NEXT: decb x10, all, mul #2 +; CHECK-NEXT: mov w11, w9 ; CHECK-NEXT: incb x9, all, mul #2 -; CHECK-NEXT: whilelo pn8.b, x12, x8, vlx2 +; CHECK-NEXT: decb x10, all, mul #2 ; CHECK-NEXT: sxtw x10, w10 -; CHECK-NEXT: add x11, x0, x10 -; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x10] -; CHECK-NEXT: ld1b { z1.b }, p1/z, [x11, #1, mul vl] -; CHECK-NEXT: add z0.b, z0.b, #1 // =0x1 +; CHECK-NEXT: ld1b { z0.b, z1.b }, pn8/z, [x0, x10] +; CHECK-NEXT: add x10, x0, x10 ; CHECK-NEXT: add z1.b, z1.b, #1 // =0x1 -; CHECK-NEXT: st1b { z0.b }, p0, [x0, x10] -; CHECK-NEXT: st1b { z1.b }, p1, [x11, #1, mul vl] +; CHECK-NEXT: add z0.b, z0.b, #1 // =0x1 +; CHECK-NEXT: st1b { z0.b, z1.b }, pn8, [x10] +; CHECK-NEXT: whilelo pn8.b, x11, x8, vlx2 ; CHECK-NEXT: b.mi .LBB3_1 ; CHECK-NEXT: // %bb.2: // %exit ; CHECK-NEXT: ret @@ -253,15 +218,12 @@ define void @rewrite_masked_load_store_i32_vlx2(ptr %x, i64 %n) #0 { ; CHECK-NEXT: whilelo pn8.s, xzr, x1, vlx2 ; CHECK-NEXT: .LBB4_1: // %loop ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: pext { p0.s, p1.s }, pn8[0] +; CHECK-NEXT: ld1w { z0.s, z1.s }, pn8/z, [x0] +; CHECK-NEXT: add z1.s, z1.s, #1 // =0x1 +; CHECK-NEXT: add z0.s, z0.s, #1 // =0x1 +; CHECK-NEXT: st1w { z0.s, z1.s }, pn8, [x0] ; CHECK-NEXT: whilelo pn8.s, x8, x1, vlx2 ; CHECK-NEXT: inch x8 -; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0, #1, mul vl] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: add z0.s, z0.s, #1 // =0x1 -; CHECK-NEXT: add z1.s, z1.s, #1 // =0x1 -; CHECK-NEXT: st1w { z0.s }, p1, [x0, #1, mul vl] -; CHECK-NEXT: st1w { z1.s }, p0, [x0] ; CHECK-NEXT: incb x0, all, mul #2 ; CHECK-NEXT: b.mi .LBB4_1 ; CHECK-NEXT: // %bb.2: // %exit @@ -295,30 +257,27 @@ exit: define void @mixed_data_vector_types(ptr %x, ptr %y, i64 %n) #0 { ; CHECK-LABEL: mixed_data_vector_types: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: whilelo pn8.d, xzr, x2, vlx4 +; CHECK-NEXT: cnth x8 ; CHECK-NEXT: .LBB5_1: // %loop ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: pext { p0.d, p1.d }, pn8[0] ; CHECK-NEXT: pext { p2.d, p3.d }, pn8[1] -; CHECK-NEXT: add x9, x0, x8, lsl #3 -; CHECK-NEXT: uzp1 p4.s, p0.s, p1.s -; CHECK-NEXT: add x10, x1, x8, lsl #2 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; CHECK-NEXT: uzp1 p5.s, p2.s, p3.s -; CHECK-NEXT: ld1d { z1.d }, p3/z, [x9, #3, mul vl] -; CHECK-NEXT: ld1d { z3.d }, p2/z, [x9, #2, mul vl] -; CHECK-NEXT: ld1w { z2.s }, p4/z, [x1, x8, lsl #2] -; CHECK-NEXT: inch x8 -; CHECK-NEXT: ld1d { z4.d }, p1/z, [x9, #1, mul vl] -; CHECK-NEXT: ld1w { z5.s }, p5/z, [x10, #1, mul vl] +; CHECK-NEXT: ld1d { z0.d - z3.d }, pn8/z, [x0] +; CHECK-NEXT: uzp1 p0.s, p0.s, p1.s ; CHECK-NEXT: whilelo pn8.d, x8, x2, vlx4 +; CHECK-NEXT: inch x8 +; CHECK-NEXT: uzp1 p1.s, p2.s, p3.s +; CHECK-NEXT: incb x0, all, mul #4 +; CHECK-NEXT: ld1w { z5.s }, p0/z, [x1] +; CHECK-NEXT: ld1w { z4.s }, p1/z, [x1, #1, mul vl] +; CHECK-NEXT: incb x1, all, mul #2 ; CHECK-NEXT: // fake_use: $z0 -; CHECK-NEXT: // fake_use: $z4 -; CHECK-NEXT: // fake_use: $z3 ; CHECK-NEXT: // fake_use: $z1 ; CHECK-NEXT: // fake_use: $z2 +; CHECK-NEXT: // fake_use: $z3 ; CHECK-NEXT: // fake_use: $z5 +; CHECK-NEXT: // fake_use: $z4 ; CHECK-NEXT: b.mi .LBB5_1 ; CHECK-NEXT: // %bb.2: // %exit ; CHECK-NEXT: ret @@ -354,26 +313,27 @@ define void @prefer_more_common_masked_access_size(ptr %x16, ptr %y32, i64 %n) # ; CHECK-LABEL: prefer_more_common_masked_access_size: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: movi v0.2d, #0000000000000000 -; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: whilelo pn8.h, xzr, x2, vlx2 +; CHECK-NEXT: rdvl x8, #1 +; CHECK-NEXT: mov z1.d, z0.d ; CHECK-NEXT: .LBB6_1: // %loop ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: pext { p0.h, p1.h }, pn8[0] -; CHECK-NEXT: add x9, x0, x8, lsl #1 -; CHECK-NEXT: add x10, x1, x8, lsl #2 -; CHECK-NEXT: punpklo p2.h, p0.b -; CHECK-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1] -; CHECK-NEXT: st1h { z0.h }, p1, [x9, #1, mul vl] -; CHECK-NEXT: punpkhi p3.h, p1.b -; CHECK-NEXT: punpklo p4.h, p1.b -; CHECK-NEXT: ld1w { z1.s }, p2/z, [x1, x8, lsl #2] -; CHECK-NEXT: incb x8 -; CHECK-NEXT: punpkhi p0.h, p0.b -; CHECK-NEXT: ld1w { z2.s }, p3/z, [x10, #3, mul vl] -; CHECK-NEXT: ld1w { z3.s }, p4/z, [x10, #2, mul vl] -; CHECK-NEXT: ld1w { z4.s }, p0/z, [x10, #1, mul vl] +; CHECK-NEXT: st1h { z0.h, z1.h }, pn8, [x0] +; CHECK-NEXT: st1h { z0.h, z1.h }, pn8, [x0] +; CHECK-NEXT: punpkhi p2.h, p1.b ; CHECK-NEXT: whilelo pn8.h, x8, x2, vlx2 -; CHECK-NEXT: // fake_use: $z1 +; CHECK-NEXT: incb x8 +; CHECK-NEXT: punpklo p1.h, p1.b +; CHECK-NEXT: incb x0, all, mul #2 +; CHECK-NEXT: punpkhi p3.h, p0.b +; CHECK-NEXT: ld1w { z2.s }, p2/z, [x1, #3, mul vl] +; CHECK-NEXT: punpklo p0.h, p0.b +; CHECK-NEXT: ld1w { z3.s }, p1/z, [x1, #2, mul vl] +; CHECK-NEXT: ld1w { z4.s }, p3/z, [x1, #1, mul vl] +; CHECK-NEXT: ld1w { z5.s }, p0/z, [x1] +; CHECK-NEXT: incb x1, all, mul #4 +; CHECK-NEXT: // fake_use: $z5 ; CHECK-NEXT: // fake_use: $z4 ; CHECK-NEXT: // fake_use: $z3 ; CHECK-NEXT: // fake_use: $z2 @@ -412,28 +372,25 @@ define void @prefer_larger_access_size_on_tie(ptr %x16, ptr %y32, i64 %n) #0 { ; CHECK-LABEL: prefer_larger_access_size_on_tie: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: movi v0.2d, #0000000000000000 -; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: whilelo pn8.s, xzr, x2, vlx4 +; CHECK-NEXT: rdvl x8, #1 ; CHECK-NEXT: .LBB7_1: // %loop ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: pext { p0.s, p1.s }, pn8[0] -; CHECK-NEXT: pext { p2.s, p3.s }, pn8[1] -; CHECK-NEXT: add x9, x0, x8, lsl #1 -; CHECK-NEXT: uzp1 p4.h, p0.h, p1.h -; CHECK-NEXT: add x10, x1, x8, lsl #2 -; CHECK-NEXT: uzp1 p5.h, p2.h, p3.h -; CHECK-NEXT: st1h { z0.h }, p4, [x0, x8, lsl #1] -; CHECK-NEXT: st1h { z0.h }, p5, [x9, #1, mul vl] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2] -; CHECK-NEXT: incb x8 -; CHECK-NEXT: ld1w { z2.s }, p3/z, [x10, #3, mul vl] -; CHECK-NEXT: ld1w { z3.s }, p2/z, [x10, #2, mul vl] -; CHECK-NEXT: ld1w { z4.s }, p1/z, [x10, #1, mul vl] +; CHECK-NEXT: pext { p0.s, p1.s }, pn8[1] +; CHECK-NEXT: pext { p2.s, p3.s }, pn8[0] +; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h +; CHECK-NEXT: uzp1 p1.h, p2.h, p3.h +; CHECK-NEXT: st1h { z0.h }, p0, [x0, #1, mul vl] +; CHECK-NEXT: st1h { z0.h }, p1, [x0] +; CHECK-NEXT: incb x0, all, mul #2 +; CHECK-NEXT: ld1w { z4.s - z7.s }, pn8/z, [x1] ; CHECK-NEXT: whilelo pn8.s, x8, x2, vlx4 -; CHECK-NEXT: // fake_use: $z1 +; CHECK-NEXT: incb x8 +; CHECK-NEXT: incb x1, all, mul #4 ; CHECK-NEXT: // fake_use: $z4 -; CHECK-NEXT: // fake_use: $z3 -; CHECK-NEXT: // fake_use: $z2 +; CHECK-NEXT: // fake_use: $z5 +; CHECK-NEXT: // fake_use: $z6 +; CHECK-NEXT: // fake_use: $z7 ; CHECK-NEXT: b.mi .LBB7_1 ; CHECK-NEXT: // %bb.2: // %exit ; CHECK-NEXT: ret @@ -471,17 +428,15 @@ define void @rewrite_extractelement_within_first_section_i32(ptr %x, i64 %n) #0 ; CHECK-NEXT: whilelo pn8.s, xzr, x1, vlx2 ; CHECK-NEXT: .LBB8_1: // %loop ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: pext { p0.s, p1.s }, pn8[0] +; CHECK-NEXT: ld1w { z0.s, z1.s }, pn8/z, [x0] ; CHECK-NEXT: whilelo pn8.s, x8, x1, vlx2 ; CHECK-NEXT: inch x8 -; CHECK-NEXT: pext p2.s, pn8[0] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: mov z0.s, p2/z, #1 // =0x1 -; CHECK-NEXT: mov w9, v0.s[3] -; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0, #1, mul vl] +; CHECK-NEXT: pext p0.s, pn8[0] ; CHECK-NEXT: incb x0, all, mul #2 -; CHECK-NEXT: // fake_use: $z1 +; CHECK-NEXT: mov z2.s, p0/z, #1 // =0x1 +; CHECK-NEXT: mov w9, v2.s[3] ; CHECK-NEXT: // fake_use: $z0 +; CHECK-NEXT: // fake_use: $z1 ; CHECK-NEXT: tbnz w9, #0, .LBB8_1 ; CHECK-NEXT: // %bb.2: // %exit ; CHECK-NEXT: ret @@ -518,21 +473,22 @@ define void @masked_load_passthru_non_poison(ptr %x, i64 %n) #0 { ; CHECK-NEXT: .LBB9_1: // %loop ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: pext { p0.d, p1.d }, pn8[1] -; CHECK-NEXT: pext { p2.d, p3.d }, pn8[0] -; CHECK-NEXT: whilelo pn8.d, x8, x1, vlx4 ; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0, #3, mul vl] +; CHECK-NEXT: pext { p1.d, p2.d }, pn8[0] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0, #2, mul vl] -; CHECK-NEXT: ld1d { z2.d }, p3/z, [x0, #1, mul vl] -; CHECK-NEXT: ld1d { z3.d }, p2/z, [x0] +; CHECK-NEXT: ld1d { z2.d }, p2/z, [x0, #1, mul vl] +; CHECK-NEXT: ld1d { z3.d }, p1/z, [x0] +; CHECK-NEXT: movprfx z7, z0 +; CHECK-NEXT: add z7.d, z7.d, #1 // =0x1 +; CHECK-NEXT: movprfx z6, z1 +; CHECK-NEXT: add z6.d, z6.d, #1 // =0x1 +; CHECK-NEXT: movprfx z5, z2 +; CHECK-NEXT: add z5.d, z5.d, #1 // =0x1 +; CHECK-NEXT: movprfx z4, z3 +; CHECK-NEXT: add z4.d, z4.d, #1 // =0x1 +; CHECK-NEXT: st1d { z4.d - z7.d }, pn8, [x0] +; CHECK-NEXT: whilelo pn8.d, x8, x1, vlx4 ; CHECK-NEXT: inch x8 -; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1 -; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1 -; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1 -; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1 -; CHECK-NEXT: st1d { z0.d }, p1, [x0, #3, mul vl] -; CHECK-NEXT: st1d { z1.d }, p0, [x0, #2, mul vl] -; CHECK-NEXT: st1d { z2.d }, p3, [x0, #1, mul vl] -; CHECK-NEXT: st1d { z3.d }, p2, [x0] ; CHECK-NEXT: incb x0, all, mul #4 ; CHECK-NEXT: b.mi .LBB9_1 ; CHECK-NEXT: // %bb.2: // %exit @@ -570,18 +526,16 @@ define void @extractelement_outside_first_section_i32(ptr %x, i64 %n) #0 { ; CHECK-NEXT: whilelo pn8.s, xzr, x1, vlx2 ; CHECK-NEXT: .LBB10_1: // %loop ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: pext { p0.s, p1.s }, pn8[0] +; CHECK-NEXT: ld1w { z0.s, z1.s }, pn8/z, [x0] ; CHECK-NEXT: whilelo pn8.s, x8, x1, vlx2 ; CHECK-NEXT: inch x8 -; CHECK-NEXT: pext { p2.s, p3.s }, pn8[0] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: uzp1 p2.h, p2.h, p3.h -; CHECK-NEXT: mov z0.h, p2/z, #1 // =0x1 -; CHECK-NEXT: umov w9, v0.h[6] -; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0, #1, mul vl] +; CHECK-NEXT: pext { p0.s, p1.s }, pn8[0] ; CHECK-NEXT: incb x0, all, mul #2 -; CHECK-NEXT: // fake_use: $z1 +; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h +; CHECK-NEXT: mov z2.h, p0/z, #1 // =0x1 +; CHECK-NEXT: umov w9, v2.h[6] ; CHECK-NEXT: // fake_use: $z0 +; CHECK-NEXT: // fake_use: $z1 ; CHECK-NEXT: tbnz w9, #0, .LBB10_1 ; CHECK-NEXT: // %bb.2: // %exit ; CHECK-NEXT: ret _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
