https://github.com/huntergr-arm updated https://github.com/llvm/llvm-project/pull/226201
>From ea9401e8f27bdeef2f762712a9571e9be3e3c669 Mon Sep 17 00:00:00 2001 From: Graham Hunter <[email protected]> Date: Thu, 24 Sep 2026 14:48:50 +0000 Subject: [PATCH 1/2] Implement support for unrolled vector case --- llvm/lib/CodeGen/CodeGenPrepare.cpp | 30 ++++++++++- .../AArch64/strength-reduce-vector-as-data.ll | 53 +++++++++---------- 2 files changed, 53 insertions(+), 30 deletions(-) diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp index 29983cfbef024..885497b75fc47 100644 --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -8988,8 +8988,30 @@ static bool strengthReduceVectorPhiUsers(PHINode *Phi, LoopInfo *LI) { if (!match(Step, m_c_Add(m_Specific(Phi), m_Value(LoopStride)))) return false; + unsigned AddCount = 0; + Value *ShiftedVScale = LoopStride; + const int DepthLimit = 4; + // We're looking for updates by a multiple of the number of elements in + // the vector. + // TODO: Support cases where the total stride is created directly by a + // shifted vscale where we have interleaving. + if (match(LoopStride, m_c_Add(m_Value(LoopStride), + m_Value(ShiftedVScale, m_Splat(m_Value()))))) + for (int I = 0; I < DepthLimit; ++I) { + AddCount++; + if (!match(LoopStride, + m_c_Add(m_Value(LoopStride), m_Specific(ShiftedVScale)))) + break; + } + const APInt *ShiftAmt = nullptr; - if (!match(LoopStride, m_Splat(m_Shl(m_VScale(), m_APInt(ShiftAmt))))) + if (!match(LoopStride, m_Splat(m_Shl(m_VScale(), m_APInt(ShiftAmt)))) || + LoopStride != ShiftedVScale) + return false; + + // Make sure the shift amount matches the minimum element count. + auto EltCnt = cast<VectorType>(Phi->getType())->getElementCount(); + if (1 << ShiftAmt->getZExtValue() != EltCnt.getKnownMinValue()) return false; // Record users of interest. @@ -9057,11 +9079,15 @@ static bool strengthReduceVectorPhiUsers(PHINode *Phi, LoopInfo *LI) { // iteration. Value *StructStride = ConstantInt::get(ITy, ShiftAmt->getZExtValue()); StructStride = PHBuilder.CreateMul(StructStride, PHBuilder.CreateVScale(ITy)); + Value *TotalStride = + PHBuilder.CreateMul(StructStride, ConstantInt::get(ITy, AddCount + 1)); StructStride = PHBuilder.CreateVectorSplat(VTy->getElementCount(), StructStride); + TotalStride = + PHBuilder.CreateVectorSplat(VTy->getElementCount(), TotalStride); IRBuilder<> LBuilder(cast<Instruction>(Step)); - Value *NewStep = LBuilder.CreateAdd(Phi, StructStride); + Value *NewStep = LBuilder.CreateAdd(Phi, TotalStride); // Update the phi to the new start and step. Phi->setIncomingValueForBlock(PreHeader, NewStart); diff --git a/llvm/test/CodeGen/AArch64/strength-reduce-vector-as-data.ll b/llvm/test/CodeGen/AArch64/strength-reduce-vector-as-data.ll index e49c276ba38dd..dcad17b887347 100644 --- a/llvm/test/CodeGen/AArch64/strength-reduce-vector-as-data.ll +++ b/llvm/test/CodeGen/AArch64/strength-reduce-vector-as-data.ll @@ -51,38 +51,35 @@ middle.block: define void @init_array_of_ptrs_to_structs_interleave4(ptr noalias %arc_ptrs, ptr %arc_new, i64 %num_arcs) #0 { ; CHECK-LABEL: init_array_of_ptrs_to_structs_interleave4: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: cntd x8 -; CHECK-NEXT: mov w9, #2147483647 // =0x7fffffff -; CHECK-NEXT: cnth x10 -; CHECK-NEXT: mov z0.d, x8 -; CHECK-NEXT: cntw x8 -; CHECK-NEXT: inch x9 -; CHECK-NEXT: mov z1.d, x8 -; CHECK-NEXT: cntd x8, all, mul #3 -; CHECK-NEXT: index z2.d, #0, #1 -; CHECK-NEXT: mov z3.d, x8 -; CHECK-NEXT: mov z4.d, x10 -; CHECK-NEXT: mov z5.d, x1 -; CHECK-NEXT: mov z6.d, #72 // =0x48 -; CHECK-NEXT: and x8, x9, x2 -; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: rdvl x8, #1 +; CHECK-NEXT: cntd x9, all, mul #3 +; CHECK-NEXT: cntw x10 +; CHECK-NEXT: lsr x8, x8, #4 +; CHECK-NEXT: cntd x11 +; CHECK-NEXT: mov w13, #2147483647 // =0x7fffffff +; CHECK-NEXT: inch x13 +; CHECK-NEXT: mov z0.d, x10 +; CHECK-NEXT: umull x9, w9, w8 +; CHECK-NEXT: umull x12, w10, w8 +; CHECK-NEXT: umull x8, w11, w8 +; CHECK-NEXT: mov w11, #72 // =0x48 +; CHECK-NEXT: index z1.d, x1, x11 +; CHECK-NEXT: mov z2.d, x9 +; CHECK-NEXT: mov z3.d, x12 +; CHECK-NEXT: mov z4.d, x8 +; CHECK-NEXT: and x8, x13, x2 ; CHECK-NEXT: sub x8, x8, x2 ; CHECK-NEXT: .LBB1_1: // %vector.body ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: add z7.d, z2.d, z0.d -; CHECK-NEXT: add z16.d, z2.d, z1.d -; CHECK-NEXT: movprfx z18, z5 -; CHECK-NEXT: mla z18.d, p0/m, z2.d, z6.d -; CHECK-NEXT: add z17.d, z2.d, z3.d +; CHECK-NEXT: add z5.d, z1.d, z4.d +; CHECK-NEXT: add z6.d, z1.d, z3.d +; CHECK-NEXT: str z1, [x0] +; CHECK-NEXT: add z7.d, z1.d, z2.d ; CHECK-NEXT: inch x8 -; CHECK-NEXT: add z2.d, z2.d, z4.d -; CHECK-NEXT: mad z7.d, p0/m, z6.d, z5.d -; CHECK-NEXT: mad z16.d, p0/m, z6.d, z5.d -; CHECK-NEXT: mad z17.d, p0/m, z6.d, z5.d -; CHECK-NEXT: str z18, [x0] -; CHECK-NEXT: str z7, [x0, #1, mul vl] -; CHECK-NEXT: str z16, [x0, #2, mul vl] -; CHECK-NEXT: str z17, [x0, #3, mul vl] +; CHECK-NEXT: add z1.d, z1.d, z0.d +; CHECK-NEXT: str z5, [x0, #1, mul vl] +; CHECK-NEXT: str z6, [x0, #2, mul vl] +; CHECK-NEXT: str z7, [x0, #3, mul vl] ; CHECK-NEXT: incb x0, all, mul #4 ; CHECK-NEXT: cbnz x8, .LBB1_1 ; CHECK-NEXT: // %bb.2: // %exit >From ca5624d2188e126e70a11567f1ec35756dbd0511 Mon Sep 17 00:00:00 2001 From: Graham Hunter <[email protected]> Date: Fri, 25 Sep 2026 13:54:40 +0000 Subject: [PATCH 2/2] update new IR test --- .../AArch64/strength-reduce-vector-as-data.ll | 42 +++++++++++++++---- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/strength-reduce-vector-as-data.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/strength-reduce-vector-as-data.ll index 1bf5a3d2d8477..dcfe56a243057 100644 --- a/llvm/test/Transforms/CodeGenPrepare/AArch64/strength-reduce-vector-as-data.ll +++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/strength-reduce-vector-as-data.ll @@ -20,12 +20,15 @@ define void @init_array_of_ptrs_to_structs(ptr noalias %arc_ptrs, ptr %arc_new, ; CHECK-NEXT: [[TMP9:%.*]] = add <vscale x 2 x i64> [[TMP7]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP5:%.*]] = mul i64 1, [[TMP11]] +; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP5]], 1 ; CHECK-NEXT: [[DOTSPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP5]], i64 0 ; CHECK-NEXT: [[DOTSPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer +; CHECK-NEXT: [[DOTSPLATINSERT3:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP10]], i64 0 +; CHECK-NEXT: [[DOTSPLAT4:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT3]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP9]], %[[ENTRY]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP9]], %[[ENTRY]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP6:%.*]] = inttoptr <vscale x 2 x i64> [[VEC_IND]] to <vscale x 2 x ptr> ; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds nuw [72 x i8], ptr [[ARC_NEW]], <vscale x 2 x i64> [[VEC_IND]] ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[ARC_PTRS]], i64 [[INDEX]] @@ -33,7 +36,7 @@ define void @init_array_of_ptrs_to_structs(ptr noalias %arc_ptrs, ptr %arc_new, ; CHECK-NEXT: [[TMP2:%.*]] = tail call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 1 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]] -; CHECK-NEXT: [[TMP10]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT2]] +; CHECK-NEXT: [[TMP12]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT4]] ; CHECK-NEXT: [[VEC_IND_NEXT:%.*]] = add nuw nsw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]] ; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]] @@ -83,17 +86,39 @@ define void @init_array_of_ptrs_to_structs_interleave4(ptr noalias %arc_ptrs, pt ; CHECK-NEXT: [[INVARIANT_OP:%.*]] = add nuw <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[BROADCAST_SPLAT]] ; CHECK-NEXT: [[INVARIANT_OP26:%.*]] = add nuw <vscale x 2 x i64> [[INVARIANT_OP]], [[BROADCAST_SPLAT]] ; CHECK-NEXT: [[INVARIANT_OP27:%.*]] = add nuw <vscale x 2 x i64> [[INVARIANT_OP26]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: [[TMP22:%.*]] = mul <vscale x 2 x i64> [[TMP5]], splat (i64 72) +; CHECK-NEXT: [[TMP23:%.*]] = ptrtoint ptr [[ARC_NEW]] to i64 +; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP23]], i64 0 +; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer +; CHECK-NEXT: [[TMP24:%.*]] = add <vscale x 2 x i64> [[TMP22]], [[DOTSPLAT]] +; CHECK-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP26:%.*]] = mul i64 1, [[TMP25]] +; CHECK-NEXT: [[TMP27:%.*]] = mul i64 [[TMP26]], 4 +; CHECK-NEXT: [[DOTSPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP26]], i64 0 +; CHECK-NEXT: [[DOTSPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer +; CHECK-NEXT: [[DOTSPLATINSERT3:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP27]], i64 0 +; CHECK-NEXT: [[DOTSPLAT4:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT3]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer +; CHECK-NEXT: [[TMP29:%.*]] = mul <vscale x 2 x i64> [[INVARIANT_OP26]], [[DOTSPLAT2]] +; CHECK-NEXT: [[TMP30:%.*]] = mul <vscale x 2 x i64> [[INVARIANT_OP]], [[DOTSPLAT2]] +; CHECK-NEXT: [[TMP31:%.*]] = mul <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[DOTSPLAT2]] ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[LSR_IV36:%.*]] = phi ptr [ [[SCEVGEP37:%.*]], %[[VECTOR_BODY]] ], [ [[ARC_PTRS]], %[[ENTRY]] ] ; CHECK-NEXT: [[LSR_IV34:%.*]] = phi i64 [ [[LSR_IV_NEXT35:%.*]], %[[VECTOR_BODY]] ], [ [[N_VEC]], %[[ENTRY]] ] -; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP5]], %[[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP24]], %[[ENTRY]] ], [ [[TMP28:%.*]], %[[VECTOR_BODY]] ] ; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]] ; CHECK-NEXT: [[STEP_ADD_2_REASS:%.*]] = add nuw <vscale x 2 x i64> [[VEC_IND]], [[INVARIANT_OP]] ; CHECK-NEXT: [[STEP_ADD_3_REASS:%.*]] = add nuw <vscale x 2 x i64> [[VEC_IND]], [[INVARIANT_OP26]] +; CHECK-NEXT: [[TMP32:%.*]] = inttoptr <vscale x 2 x i64> [[VEC_IND]] to <vscale x 2 x ptr> ; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds nuw [72 x i8], ptr [[ARC_NEW]], <vscale x 2 x i64> [[VEC_IND]] +; CHECK-NEXT: [[TMP33:%.*]] = add <vscale x 2 x i64> [[VEC_IND]], [[TMP31]] +; CHECK-NEXT: [[TMP17:%.*]] = inttoptr <vscale x 2 x i64> [[TMP33]] to <vscale x 2 x ptr> ; CHECK-NEXT: [[WIDE_GEP10:%.*]] = getelementptr inbounds nuw [72 x i8], ptr [[ARC_NEW]], <vscale x 2 x i64> [[STEP_ADD]] +; CHECK-NEXT: [[TMP18:%.*]] = add <vscale x 2 x i64> [[VEC_IND]], [[TMP30]] +; CHECK-NEXT: [[TMP19:%.*]] = inttoptr <vscale x 2 x i64> [[TMP18]] to <vscale x 2 x ptr> ; CHECK-NEXT: [[WIDE_GEP11:%.*]] = getelementptr inbounds nuw [72 x i8], ptr [[ARC_NEW]], <vscale x 2 x i64> [[STEP_ADD_2_REASS]] +; CHECK-NEXT: [[TMP20:%.*]] = add <vscale x 2 x i64> [[VEC_IND]], [[TMP29]] +; CHECK-NEXT: [[TMP21:%.*]] = inttoptr <vscale x 2 x i64> [[TMP20]] to <vscale x 2 x ptr> ; CHECK-NEXT: [[WIDE_GEP12:%.*]] = getelementptr inbounds nuw [72 x i8], ptr [[ARC_NEW]], <vscale x 2 x i64> [[STEP_ADD_3_REASS]] ; CHECK-NEXT: [[TMP6:%.*]] = tail call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP7:%.*]] = shl i64 [[TMP6]], 4 @@ -104,11 +129,12 @@ define void @init_array_of_ptrs_to_structs_interleave4(ptr noalias %arc_ptrs, pt ; CHECK-NEXT: [[TMP10:%.*]] = tail call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 48 ; CHECK-NEXT: [[SCEVGEP38:%.*]] = getelementptr i8, ptr [[LSR_IV36]], i64 [[TMP11]] -; CHECK-NEXT: store <vscale x 2 x ptr> [[WIDE_GEP]], ptr [[LSR_IV36]], align 8 -; CHECK-NEXT: store <vscale x 2 x ptr> [[WIDE_GEP10]], ptr [[SCEVGEP40]], align 8 -; CHECK-NEXT: store <vscale x 2 x ptr> [[WIDE_GEP11]], ptr [[SCEVGEP39]], align 8 -; CHECK-NEXT: store <vscale x 2 x ptr> [[WIDE_GEP12]], ptr [[SCEVGEP38]], align 8 -; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <vscale x 2 x i64> [[VEC_IND]], [[INVARIANT_OP27]] +; CHECK-NEXT: store <vscale x 2 x ptr> [[TMP32]], ptr [[LSR_IV36]], align 8 +; CHECK-NEXT: store <vscale x 2 x ptr> [[TMP17]], ptr [[SCEVGEP40]], align 8 +; CHECK-NEXT: store <vscale x 2 x ptr> [[TMP19]], ptr [[SCEVGEP39]], align 8 +; CHECK-NEXT: store <vscale x 2 x ptr> [[TMP21]], ptr [[SCEVGEP38]], align 8 +; CHECK-NEXT: [[TMP28]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT4]] +; CHECK-NEXT: [[VEC_IND_NEXT:%.*]] = add nuw <vscale x 2 x i64> [[VEC_IND]], [[INVARIANT_OP27]] ; CHECK-NEXT: [[TMP12:%.*]] = tail call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP13:%.*]] = shl nuw nsw i64 [[TMP12]], 3 ; CHECK-NEXT: [[LSR_IV_NEXT35]] = sub i64 [[LSR_IV34]], [[TMP13]] _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
