https://github.com/kasuga-fj updated https://github.com/llvm/llvm-project/pull/205564
>From ce56a6ac0fc43a884ac2fe6575c1bf0163b5d996 Mon Sep 17 00:00:00 2001 From: Ryotaro Kasuga <[email protected]> Date: Wed, 24 Jun 2026 13:37:49 +0000 Subject: [PATCH] [LoopInterchange] Prevent the transformation stage from stopping partway --- .../lib/Transforms/Scalar/LoopInterchange.cpp | 55 +++++++------------ .../LoopInterchange/guarded-inner-loop.ll | 20 ++++--- .../interchanged-loop-nest-3.ll | 24 ++++---- .../lcssa-incoming-value-is-not-instr.ll | 4 +- .../pr43326-ideal-access-pattern.ll | 26 ++++----- .../Transforms/LoopInterchange/pr57148.ll | 32 ++++++----- .../reduction-extra-use-in-inner-loop.ll | 12 ++-- .../reduction-not-involve-innermost.ll | 6 +- .../LoopInterchange/transform-stop-partway.ll | 35 ++++++------ 9 files changed, 102 insertions(+), 112 deletions(-) diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp index 392912486705c..264bc700a5d6f 100644 --- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp +++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp @@ -627,7 +627,7 @@ class LoopInterchangeTransform { : OuterLoop(Outer), InnerLoop(Inner), SE(SE), LI(LI), DT(DT), LIL(LIL) {} /// Interchange OuterLoop and InnerLoop. - bool transform(ArrayRef<Instruction *> DropNoWrapInsts, + void transform(ArrayRef<Instruction *> DropNoWrapInsts, ArrayRef<Instruction *> DropNoInfInsts); void reduction2Memory(); void restructureLoops(Loop *NewInner, Loop *NewOuter, @@ -636,8 +636,8 @@ class LoopInterchangeTransform { void removeChildLoop(Loop *OuterLoop, Loop *InnerLoop); private: - bool adjustLoopLinks(); - bool adjustLoopBranches(); + void adjustLoopLinks(); + void adjustLoopBranches(); Loop *OuterLoop; Loop *InnerLoop; @@ -2103,10 +2103,9 @@ void LoopInterchangeTransform::reduction2Memory() { SR.LcssaStore->moveAfter(dyn_cast<Instruction>(SR.Next)); } -bool LoopInterchangeTransform::transform( +void LoopInterchangeTransform::transform( ArrayRef<Instruction *> DropNoWrapInsts, ArrayRef<Instruction *> DropNoInfInsts) { - bool Transformed = false; ArrayRef<LoopInterchangeLegality::InnerReduction> InnerReductions = LIL.getInnerReductions(); @@ -2184,11 +2183,8 @@ bool LoopInterchangeTransform::transform( // Ensure the inner loop phi nodes have a separate basic block. BasicBlock *InnerLoopHeader = InnerLoop->getHeader(); - if (&*InnerLoopHeader->getFirstNonPHIIt() != - InnerLoopHeader->getTerminator()) { - SplitBlock(InnerLoopHeader, InnerLoopHeader->getFirstNonPHIIt(), DT, LI); - LLVM_DEBUG(dbgs() << "splitting InnerLoopHeader done\n"); - } + SplitBlock(InnerLoopHeader, InnerLoopHeader->getFirstNonPHIIt(), DT, LI); + LLVM_DEBUG(dbgs() << "splitting InnerLoopHeader done\n"); // Instructions in the original inner loop preheader may depend on values // defined in the outer loop header. Move them there, because the original @@ -2212,11 +2208,7 @@ bool LoopInterchangeTransform::transform( I.moveBeforePreserving(OuterLoopHeader->getTerminator()->getIterator()); } - Transformed |= adjustLoopLinks(); - if (!Transformed) { - LLVM_DEBUG(dbgs() << "adjustLoopLinks failed\n"); - return false; - } + adjustLoopLinks(); // Finally, drop the nsw/nuw/ninf flags from the instructions for reduction // calculations. @@ -2226,8 +2218,6 @@ bool LoopInterchangeTransform::transform( } for (Instruction *I : DropNoInfInsts) I->setHasNoInfs(false); - - return true; } /// \brief Move all instructions except the terminator from FromBB right before @@ -2434,7 +2424,7 @@ static void simplifyLCSSAPhis(Loop *OuterLoop, Loop *InnerLoop) { } } -bool LoopInterchangeTransform::adjustLoopBranches() { +void LoopInterchangeTransform::adjustLoopBranches() { LLVM_DEBUG(dbgs() << "adjustLoopBranches called\n"); std::vector<DominatorTree::UpdateType> DTUpdates; @@ -2487,11 +2477,8 @@ bool LoopInterchangeTransform::adjustLoopBranches() { Instruction *OuterLoopPredecessorBI = OuterLoopPredecessor->getTerminator(); BasicBlock *InnerLoopHeaderSuccessor = InnerLoopHeader->getUniqueSuccessor(); - if (!InnerLoopHeaderSuccessor) { - LLVM_DEBUG( - dbgs() << "Inner loop header does not have a unique successor\n"); - return false; - } + assert(InnerLoopHeaderSuccessor && + "Failed to find a unique successor for the inner loop header"); // Adjust Loop Preheader and headers. // The branches in the outer loop predecessor and the outer loop header can @@ -2588,22 +2575,18 @@ bool LoopInterchangeTransform::adjustLoopBranches() { make_range(OuterLoopHeader->begin(), std::prev(OuterLoopHeader->end()))) MayNeedLCSSAPhis.push_back(&I); formLCSSAForInstructions(MayNeedLCSSAPhis, *DT, *LI, SE); - - return true; } -bool LoopInterchangeTransform::adjustLoopLinks() { +void LoopInterchangeTransform::adjustLoopLinks() { // Adjust all branches in the inner and outer loop. - bool Changed = adjustLoopBranches(); - if (Changed) { - // We have interchanged the preheaders so we need to interchange the data in - // the preheaders as well. This is because the content of the inner - // preheader was previously executed inside the outer loop. - BasicBlock *OuterLoopPreHeader = OuterLoop->getLoopPreheader(); - BasicBlock *InnerLoopPreHeader = InnerLoop->getLoopPreheader(); - swapBBContents(OuterLoopPreHeader, InnerLoopPreHeader); - } - return Changed; + adjustLoopBranches(); + + // We have interchanged the preheaders so we need to interchange the data in + // the preheaders as well. This is because the content of the inner + // preheader was previously executed inside the outer loop. + BasicBlock *OuterLoopPreHeader = OuterLoop->getLoopPreheader(); + BasicBlock *InnerLoopPreHeader = InnerLoop->getLoopPreheader(); + swapBBContents(OuterLoopPreHeader, InnerLoopPreHeader); } PreservedAnalyses LoopInterchangePass::run(LoopNest &LN, diff --git a/llvm/test/Transforms/LoopInterchange/guarded-inner-loop.ll b/llvm/test/Transforms/LoopInterchange/guarded-inner-loop.ll index ec12ff2878743..f71bad815cf48 100644 --- a/llvm/test/Transforms/LoopInterchange/guarded-inner-loop.ll +++ b/llvm/test/Transforms/LoopInterchange/guarded-inner-loop.ll @@ -23,19 +23,21 @@ define i32 @main() { ; CHECK-LABEL: define i32 @main() { ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: br label %[[FOR_I_PREHEADER:.*]] -; CHECK: [[FOR_I_PREHEADER]]: +; CHECK: [[FOR_I_PREHEADER1:.*]]: ; CHECK-NEXT: br label %[[FOR_I:.*]] ; CHECK: [[FOR_I]]: -; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], %[[FOR_I_INC:.*]] ], [ 0, %[[FOR_I_PREHEADER]] ] +; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], %[[FOR_J_INC_SPLIT:.*]] ], [ 0, %[[FOR_I_PREHEADER1]] ] ; CHECK-NEXT: [[I_IS_ZERO:%.*]] = icmp eq i32 [[I]], 0 ; CHECK-NEXT: [[XBASE:%.*]] = getelementptr [9 x i32], ptr @x, i32 [[I]] ; CHECK-NEXT: [[WBASE:%.*]] = getelementptr [9 x i32], ptr @w, i32 [[I]] ; CHECK-NEXT: [[YBASE:%.*]] = getelementptr [3 x i32], ptr @y, i32 [[I]] ; CHECK-NEXT: br label %[[FOR_J_PREHEADER:.*]] -; CHECK: [[FOR_J_PREHEADER]]: +; CHECK: [[FOR_I_PREHEADER]]: ; CHECK-NEXT: br label %[[FOR_J:.*]] ; CHECK: [[FOR_J]]: -; CHECK-NEXT: [[J:%.*]] = phi i32 [ [[TMP0:%.*]], %[[FOR_J_INC_SPLIT:.*]] ], [ 0, %[[FOR_J_PREHEADER]] ] +; CHECK-NEXT: [[J:%.*]] = phi i32 [ [[TMP0:%.*]], %[[EXIT:.*]] ], [ 0, %[[FOR_I_PREHEADER]] ] +; CHECK-NEXT: br label %[[FOR_I_PREHEADER1]] +; CHECK: [[FOR_J_PREHEADER]]: ; CHECK-NEXT: br i1 [[I_IS_ZERO]], label %[[FOR_J_INC:.*]], label %[[FOR_K_PH:.*]] ; CHECK: [[FOR_K_PH]]: ; CHECK-NEXT: [[XP:%.*]] = getelementptr i32, ptr [[XBASE]], i32 [[J]] @@ -60,15 +62,15 @@ define i32 @main() { ; CHECK-NEXT: [[J_NEXT:%.*]] = add i32 [[J]], 1 ; CHECK-NEXT: [[J_CMP:%.*]] = icmp eq i32 [[J]], 0 ; CHECK-NEXT: br label %[[FOR_J_INC_SPLIT]] -; CHECK: [[FOR_J_INC_SPLIT]]: +; CHECK: [[EXIT]]: ; CHECK-NEXT: [[TMP0]] = add i32 [[J]], 1 ; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[J]], 0 -; CHECK-NEXT: br i1 [[TMP1]], label %[[FOR_J]], label %[[FOR_I_INC]] -; CHECK: [[FOR_I_INC]]: +; CHECK-NEXT: br i1 [[TMP1]], label %[[FOR_J]], label %[[FOR_I_INC:.*]] +; CHECK: [[FOR_J_INC_SPLIT]]: ; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1 ; CHECK-NEXT: [[I_DONE:%.*]] = icmp eq i32 [[I_NEXT]], 3 -; CHECK-NEXT: br i1 [[I_DONE]], label %[[EXIT:.*]], label %[[FOR_I]] -; CHECK: [[EXIT]]: +; CHECK-NEXT: br i1 [[I_DONE]], label %[[EXIT]], label %[[FOR_I]] +; CHECK: [[FOR_I_INC]]: ; CHECK-NEXT: ret i32 0 ; entry: diff --git a/llvm/test/Transforms/LoopInterchange/interchanged-loop-nest-3.ll b/llvm/test/Transforms/LoopInterchange/interchanged-loop-nest-3.ll index 3cf3b02e8fd21..3d13733bf3fde 100644 --- a/llvm/test/Transforms/LoopInterchange/interchanged-loop-nest-3.ll +++ b/llvm/test/Transforms/LoopInterchange/interchanged-loop-nest-3.ll @@ -19,42 +19,40 @@ define void @interchange_08(i32 %t){ ; CHECK-SAME: i32 [[T:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: br label %[[FOR_BODY6_PREHEADER:.*]] -; CHECK: [[FOR_COND1_PREHEADER_PREHEADER:.*]]: +; CHECK: [[FOR_COND4_PREHEADER_PREHEADER:.*]]: ; CHECK-NEXT: br label %[[FOR_COND1_PREHEADER:.*]] ; CHECK: [[FOR_COND1_PREHEADER]]: -; CHECK-NEXT: [[I_028:%.*]] = phi i64 [ [[INC16:%.*]], %[[FOR_INC16:.*]] ], [ 0, %[[FOR_COND1_PREHEADER_PREHEADER]] ] +; CHECK-NEXT: [[I_028:%.*]] = phi i64 [ [[INC16:%.*]], %[[FOR_INC16:.*]] ], [ 0, %[[FOR_COND4_PREHEADER_PREHEADER]] ] ; CHECK-NEXT: br label %[[FOR_BODY6_SPLIT1:.*]] -; CHECK: [[FOR_COND4_PREHEADER_PREHEADER:.*]]: +; CHECK: [[FOR_COND4_PREHEADER_PREHEADER1:.*]]: ; CHECK-NEXT: br label %[[FOR_COND4_PREHEADER:.*]] ; CHECK: [[FOR_COND4_PREHEADER]]: -; CHECK-NEXT: [[J_027:%.*]] = phi i64 [ [[TMP3:%.*]], %[[FOR_INC12_SPLIT:.*]] ], [ 0, %[[FOR_COND4_PREHEADER_PREHEADER]] ] -; CHECK-NEXT: br label %[[FOR_COND1_PREHEADER_PREHEADER]] +; CHECK-NEXT: [[J_027:%.*]] = phi i64 [ [[TMP3:%.*]], %[[FOR_INC12:.*]] ], [ 0, %[[FOR_COND4_PREHEADER_PREHEADER1]] ] +; CHECK-NEXT: br label %[[FOR_COND1_PREHEADER_PREHEADER:.*]] ; CHECK: [[FOR_BODY6_PREHEADER]]: ; CHECK-NEXT: br label %[[FOR_BODY6:.*]] ; CHECK: [[FOR_BODY6]]: -; CHECK-NEXT: [[K_026:%.*]] = phi i64 [ [[TMP5:%.*]], %[[FOR_BODY6_SPLIT:.*]] ], [ 0, %[[FOR_BODY6_PREHEADER]] ] +; CHECK-NEXT: [[K_026:%.*]] = phi i64 [ [[TMP5:%.*]], %[[FOR_INC12_SPLIT:.*]] ], [ 0, %[[FOR_BODY6_PREHEADER]] ] ; CHECK-NEXT: br label %[[FOR_COND4_PREHEADER_PREHEADER]] ; CHECK: [[FOR_BODY6_SPLIT1]]: +; CHECK-NEXT: br label %[[FOR_COND4_PREHEADER_PREHEADER1]] +; CHECK: [[FOR_COND1_PREHEADER_PREHEADER]]: ; CHECK-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds [100 x [100 x [100 x i32]]], ptr @D, i64 0, i64 [[K_026]], i64 [[J_027]], i64 [[I_028]] ; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX8]], align 4 ; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP0]], [[T]] ; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX8]], align 4 ; CHECK-NEXT: [[INC:%.*]] = add nuw nsw i64 [[K_026]], 1 ; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INC]], 100 -; CHECK-NEXT: br label %[[FOR_INC12:.*]] -; CHECK: [[FOR_INC15:.*]]: +; CHECK-NEXT: br label %[[FOR_INC12]] +; CHECK: [[FOR_BODY6_SPLIT:.*]]: ; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[K_026]], 1 ; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 100 ; CHECK-NEXT: br label %[[FOR_INC16]] -; CHECK: [[FOR_BODY6_SPLIT]]: +; CHECK: [[FOR_INC12_SPLIT]]: ; CHECK-NEXT: [[TMP5]] = add nuw nsw i64 [[K_026]], 1 ; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 100 ; CHECK-NEXT: br i1 [[TMP6]], label %[[FOR_END17:.*]], label %[[FOR_BODY6]] ; CHECK: [[FOR_INC12]]: -; CHECK-NEXT: [[INC13:%.*]] = add nuw nsw i64 [[J_027]], 1 -; CHECK-NEXT: [[EXITCOND29:%.*]] = icmp eq i64 [[INC13]], 100 -; CHECK-NEXT: br label %[[FOR_INC15]] -; CHECK: [[FOR_INC12_SPLIT]]: ; CHECK-NEXT: [[TMP3]] = add nuw nsw i64 [[J_027]], 1 ; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[TMP3]], 100 ; CHECK-NEXT: br i1 [[TMP4]], label %[[FOR_BODY6_SPLIT]], label %[[FOR_COND4_PREHEADER]] diff --git a/llvm/test/Transforms/LoopInterchange/lcssa-incoming-value-is-not-instr.ll b/llvm/test/Transforms/LoopInterchange/lcssa-incoming-value-is-not-instr.ll index b0abe98b5f5d7..3397e4295c933 100644 --- a/llvm/test/Transforms/LoopInterchange/lcssa-incoming-value-is-not-instr.ll +++ b/llvm/test/Transforms/LoopInterchange/lcssa-incoming-value-is-not-instr.ll @@ -82,7 +82,9 @@ define i32 @g() { ; CHECK-NEXT: [[J:%.*]] = phi i64 [ [[TMP1:%.*]], %[[INNER_LATCH_SPLIT:.*]] ], [ 0, %[[INNER_HEADER_PREHEADER]] ] ; CHECK-NEXT: br label %[[OUTER_HEADER_PREHEADER]] ; CHECK: [[INNER_LATCH]]: -; CHECK-NEXT: [[X:%.*]] = phi i32 [ 42, %[[OUTER_HEADER]] ] +; CHECK-NEXT: br label %[[INNER_LATCH1:.*]] +; CHECK: [[INNER_LATCH1]]: +; CHECK-NEXT: [[X:%.*]] = phi i32 [ 42, %[[INNER_LATCH]] ] ; CHECK-NEXT: [[J_NEXT:%.*]] = add i64 [[J]], 1 ; CHECK-NEXT: [[EC_J:%.*]] = icmp eq i64 [[J_NEXT]], 2 ; CHECK-NEXT: br label %[[INNER_EXIT:.*]] diff --git a/llvm/test/Transforms/LoopInterchange/pr43326-ideal-access-pattern.ll b/llvm/test/Transforms/LoopInterchange/pr43326-ideal-access-pattern.ll index d3bccaff36c26..48cc2ebd7102f 100644 --- a/llvm/test/Transforms/LoopInterchange/pr43326-ideal-access-pattern.ll +++ b/llvm/test/Transforms/LoopInterchange/pr43326-ideal-access-pattern.ll @@ -18,36 +18,34 @@ define void @pr43326-triply-nested(ptr noalias %e, ptr noalias %f) { ; CHECK-SAME: ptr noalias [[E:%.*]], ptr noalias [[F:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: br label %[[FOR_INNERMOST_PREHEADER:.*]] -; CHECK: [[FOR_OUTERMOST_HEADER_PREHEADER:.*]]: +; CHECK: [[FOR_MIDDLE_HEADER_PREHEADER:.*]]: ; CHECK-NEXT: br label %[[FOR_OUTERMOST_HEADER:.*]] ; CHECK: [[FOR_OUTERMOST_HEADER]]: -; CHECK-NEXT: [[INDVARS_OUTERMOST:%.*]] = phi i64 [ [[INDVARS_OUTERMOST_NEXT:%.*]], %[[FOR_OUTERMOST_LATCH:.*]] ], [ 0, %[[FOR_OUTERMOST_HEADER_PREHEADER]] ] +; CHECK-NEXT: [[INDVARS_OUTERMOST:%.*]] = phi i64 [ [[INDVARS_OUTERMOST_NEXT:%.*]], %[[FOR_OUTERMOST_LATCH:.*]] ], [ 0, %[[FOR_MIDDLE_HEADER_PREHEADER]] ] ; CHECK-NEXT: br label %[[FOR_INNERMOST_SPLIT1:.*]] -; CHECK: [[FOR_MIDDLE_HEADER_PREHEADER:.*]]: +; CHECK: [[FOR_MIDDLE_HEADER_PREHEADER1:.*]]: ; CHECK-NEXT: br label %[[FOR_MIDDLE_HEADER:.*]] ; CHECK: [[FOR_COND_CLEANUP:.*]]: ; CHECK-NEXT: ret void ; CHECK: [[FOR_MIDDLE_HEADER]]: -; CHECK-NEXT: [[INDVARS_MIDDLE:%.*]] = phi i64 [ [[TMP0:%.*]], %[[FOR_MIDDLE_LATCH_SPLIT:.*]] ], [ 0, %[[FOR_MIDDLE_HEADER_PREHEADER]] ] -; CHECK-NEXT: br label %[[FOR_OUTERMOST_HEADER_PREHEADER]] +; CHECK-NEXT: [[INDVARS_MIDDLE:%.*]] = phi i64 [ [[TMP0:%.*]], %[[FOR_MIDDLE_LATCH:.*]] ], [ 0, %[[FOR_MIDDLE_HEADER_PREHEADER1]] ] +; CHECK-NEXT: br label %[[FOR_OUTERMOST_HEADER_PREHEADER:.*]] ; CHECK: [[FOR_INNERMOST_PREHEADER]]: ; CHECK-NEXT: br label %[[FOR_INNERMOST:.*]] ; CHECK: [[FOR_OUTERMOST_LATCH]]: ; CHECK-NEXT: [[INDVARS_OUTERMOST_NEXT]] = add nuw nsw i64 [[INDVARS_OUTERMOST]], 1 ; CHECK-NEXT: [[EXITCOND_OUTERMOST:%.*]] = icmp ne i64 [[INDVARS_OUTERMOST_NEXT]], 10 -; CHECK-NEXT: br i1 [[EXITCOND_OUTERMOST]], label %[[FOR_OUTERMOST_HEADER]], label %[[FOR_MIDDLE_LATCH_SPLIT]] -; CHECK: [[FOR_MIDDLE_LATCH:.*]]: -; CHECK-NEXT: [[INDVARS_MIDDLE_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_MIDDLE]], 1 -; CHECK-NEXT: [[EXITCOND_MIDDLE:%.*]] = icmp ne i64 [[INDVARS_MIDDLE_NEXT]], 10 -; CHECK-NEXT: br label %[[FOR_INNERMOST_SPLIT:.*]] -; CHECK: [[FOR_MIDDLE_LATCH_SPLIT]]: +; CHECK-NEXT: br i1 [[EXITCOND_OUTERMOST]], label %[[FOR_OUTERMOST_HEADER]], label %[[FOR_MIDDLE_LATCH_SPLIT:.*]] +; CHECK: [[FOR_MIDDLE_LATCH]]: ; CHECK-NEXT: [[TMP0]] = add nuw nsw i64 [[INDVARS_MIDDLE]], 1 ; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i64 [[TMP0]], 10 ; CHECK-NEXT: br i1 [[TMP1]], label %[[FOR_MIDDLE_HEADER]], label %[[FOR_INNERMOST_SPLIT_SPLIT:.*]] ; CHECK: [[FOR_INNERMOST]]: -; CHECK-NEXT: [[INDVARS_INNERMOST:%.*]] = phi i64 [ [[TMP5:%.*]], %[[FOR_INNERMOST_SPLIT_SPLIT]] ], [ 0, %[[FOR_INNERMOST_PREHEADER]] ] +; CHECK-NEXT: [[INDVARS_INNERMOST:%.*]] = phi i64 [ [[TMP5:%.*]], %[[FOR_MIDDLE_LATCH_SPLIT]] ], [ 0, %[[FOR_INNERMOST_PREHEADER]] ] ; CHECK-NEXT: br label %[[FOR_MIDDLE_HEADER_PREHEADER]] ; CHECK: [[FOR_INNERMOST_SPLIT1]]: +; CHECK-NEXT: br label %[[FOR_MIDDLE_HEADER_PREHEADER1]] +; CHECK: [[FOR_OUTERMOST_HEADER_PREHEADER]]: ; CHECK-NEXT: [[ARRAYIDX12:%.*]] = getelementptr inbounds [10 x [10 x i32]], ptr [[E]], i64 [[INDVARS_INNERMOST]], i64 [[INDVARS_MIDDLE]], i64 [[INDVARS_OUTERMOST]] ; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX12]], align 4 ; CHECK-NEXT: [[ARRAYIDX18:%.*]] = getelementptr inbounds [10 x [10 x i32]], ptr [[F]], i64 [[INDVARS_INNERMOST]], i64 [[INDVARS_MIDDLE]], i64 [[INDVARS_OUTERMOST]] @@ -55,11 +53,11 @@ define void @pr43326-triply-nested(ptr noalias %e, ptr noalias %f) { ; CHECK-NEXT: [[INDVARS_INNERMOST_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_INNERMOST]], 1 ; CHECK-NEXT: [[EXITCOND_INNERMOST:%.*]] = icmp ne i64 [[INDVARS_INNERMOST_NEXT]], 10 ; CHECK-NEXT: br label %[[FOR_MIDDLE_LATCH]] -; CHECK: [[FOR_INNERMOST_SPLIT]]: +; CHECK: [[FOR_INNERMOST_SPLIT_SPLIT]]: ; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[INDVARS_INNERMOST]], 1 ; CHECK-NEXT: [[TMP4:%.*]] = icmp ne i64 [[TMP3]], 10 ; CHECK-NEXT: br label %[[FOR_OUTERMOST_LATCH]] -; CHECK: [[FOR_INNERMOST_SPLIT_SPLIT]]: +; CHECK: [[FOR_MIDDLE_LATCH_SPLIT]]: ; CHECK-NEXT: [[TMP5]] = add nuw nsw i64 [[INDVARS_INNERMOST]], 1 ; CHECK-NEXT: [[TMP6:%.*]] = icmp ne i64 [[TMP5]], 10 ; CHECK-NEXT: br i1 [[TMP6]], label %[[FOR_INNERMOST]], label %[[FOR_COND_CLEANUP]] diff --git a/llvm/test/Transforms/LoopInterchange/pr57148.ll b/llvm/test/Transforms/LoopInterchange/pr57148.ll index 2a1a21c57bfe7..37e01a07b00d6 100644 --- a/llvm/test/Transforms/LoopInterchange/pr57148.ll +++ b/llvm/test/Transforms/LoopInterchange/pr57148.ll @@ -25,16 +25,18 @@ define void @test1() { ; CHECK-NEXT: [[J_010:%.*]] = phi i16 [ [[TMP3:%.*]], %[[FOR_END67:.*]] ], [ 0, %[[FOR_BODY42_SPLIT1]] ] ; CHECK-NEXT: br label %[[FOR_COND33_PREHEADER_PREHEADER]] ; CHECK: [[FOR_COND37_PREHEADER]]: -; CHECK-NEXT: [[K_09:%.*]] = phi i16 [ -512, %[[FOR_BODY42]] ], [ [[TMP1:%.*]], %[[FOR_COND37_PREHEADER]] ] +; CHECK-NEXT: br label %[[FOR_BODY43:.*]] +; CHECK: [[FOR_BODY43]]: +; CHECK-NEXT: [[K_09:%.*]] = phi i16 [ -512, %[[FOR_COND37_PREHEADER]] ], [ [[TMP1:%.*]], %[[FOR_BODY43]] ] ; CHECK-NEXT: [[SUB51:%.*]] = add nsw i16 [[K_09]], 512 ; CHECK-NEXT: [[ARRAYIDX55:%.*]] = getelementptr inbounds [512 x [4 x i32]], ptr @b, i16 0, i16 [[SUB51]], i16 [[J_010]] ; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX55]], align 1 ; CHECK-NEXT: [[ADD61:%.*]] = add i32 undef, undef ; CHECK-NEXT: [[TMP1]] = add nsw i16 [[K_09]], 1 ; CHECK-NEXT: [[TMP2:%.*]] = icmp slt i16 [[K_09]], 42 -; CHECK-NEXT: br i1 [[TMP2]], label %[[FOR_COND37_PREHEADER]], label %[[FOR_END64:.*]] +; CHECK-NEXT: br i1 [[TMP2]], label %[[FOR_BODY43]], label %[[FOR_END64:.*]] ; CHECK: [[FOR_END64]]: -; CHECK-NEXT: [[ADD61_LCSSA_LCSSA:%.*]] = phi i32 [ [[ADD61]], %[[FOR_COND37_PREHEADER]] ] +; CHECK-NEXT: [[ADD61_LCSSA_LCSSA:%.*]] = phi i32 [ [[ADD61]], %[[FOR_BODY43]] ] ; CHECK-NEXT: store i32 [[ADD61_LCSSA_LCSSA]], ptr undef, align 1 ; CHECK-NEXT: [[INC66:%.*]] = add nuw nsw i16 [[J_010]], 1 ; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i16 [[J_010]], 43 @@ -185,27 +187,29 @@ define void @test3() { ; CHECK: [[FOR_BODY42]]: ; CHECK-NEXT: [[I_011:%.*]] = phi i16 [ [[INC69:%.*]], %[[FOR_END68:.*]] ], [ 0, %[[FOR_COND33_PREHEADER_PREHEADER]] ] ; CHECK-NEXT: br label %[[FOR_COND38_PREHEADER:.*]] -; CHECK: [[FOR_BODY42_PREHEADER:.*]]: -; CHECK-NEXT: br label %[[FOR_COND37_PREHEADER:.*]] ; CHECK: [[FOR_BODY42_SPLIT1]]: +; CHECK-NEXT: br label %[[FOR_COND37_PREHEADER:.*]] +; CHECK: [[FOR_COND37_PREHEADER]]: +; CHECK-NEXT: [[J_010:%.*]] = phi i16 [ [[TMP5:%.*]], %[[FOR_END67:.*]] ], [ 0, %[[FOR_BODY42_SPLIT1]] ] ; CHECK-NEXT: br label %[[FOR_COND38_PREHEADER_PREHEADER:.*]] ; CHECK: [[FOR_COND38_PREHEADER_PREHEADER]]: -; CHECK-NEXT: [[J_010:%.*]] = phi i16 [ [[TMP5:%.*]], %[[FOR_END67:.*]] ], [ 0, %[[FOR_BODY42_SPLIT1]] ] -; CHECK-NEXT: br label %[[FOR_BODY42_PREHEADER]] -; CHECK: [[FOR_COND37_PREHEADER]]: -; CHECK-NEXT: [[K_010:%.*]] = phi i16 [ [[TMP1:%.*]], %[[FOR_END64:.*]] ], [ 0, %[[FOR_BODY42_PREHEADER]] ] +; CHECK-NEXT: br label %[[FOR_BODY42_PREHEADER:.*]] +; CHECK: [[FOR_BODY42_PREHEADER]]: +; CHECK-NEXT: [[K_010:%.*]] = phi i16 [ 0, %[[FOR_COND38_PREHEADER_PREHEADER]] ], [ [[TMP1:%.*]], %[[FOR_END64:.*]] ] ; CHECK-NEXT: br label %[[FOR_COND33_PREHEADER_PREHEADER]] ; CHECK: [[FOR_COND38_PREHEADER]]: -; CHECK-NEXT: [[K_09:%.*]] = phi i16 [ -512, %[[FOR_BODY42]] ], [ [[TMP3:%.*]], %[[FOR_COND38_PREHEADER]] ] +; CHECK-NEXT: br label %[[FOR_BODY43:.*]] +; CHECK: [[FOR_BODY43]]: +; CHECK-NEXT: [[K_09:%.*]] = phi i16 [ -512, %[[FOR_COND38_PREHEADER]] ], [ [[TMP3:%.*]], %[[FOR_BODY43]] ] ; CHECK-NEXT: [[SUB51:%.*]] = add nsw i16 [[K_09]], 512 ; CHECK-NEXT: [[ARRAYIDX55:%.*]] = getelementptr inbounds [1024 x [512 x [4 x i32]]], ptr @d, i16 0, i16 [[SUB51]], i16 [[J_010]], i16 [[K_010]] ; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX55]], align 1 ; CHECK-NEXT: [[ADD61:%.*]] = add i32 undef, undef ; CHECK-NEXT: [[TMP3]] = add nsw i16 [[K_09]], 1 ; CHECK-NEXT: [[TMP2:%.*]] = icmp slt i16 [[K_09]], 42 -; CHECK-NEXT: br i1 [[TMP2]], label %[[FOR_COND38_PREHEADER]], label %[[FOR_END65:.*]] +; CHECK-NEXT: br i1 [[TMP2]], label %[[FOR_BODY43]], label %[[FOR_END65:.*]] ; CHECK: [[FOR_END65]]: -; CHECK-NEXT: [[ADD61_LCSSA_LCSSA:%.*]] = phi i32 [ [[ADD61]], %[[FOR_COND38_PREHEADER]] ] +; CHECK-NEXT: [[ADD61_LCSSA_LCSSA:%.*]] = phi i32 [ [[ADD61]], %[[FOR_BODY43]] ] ; CHECK-NEXT: store i32 [[ADD61_LCSSA_LCSSA]], ptr undef, align 1 ; CHECK-NEXT: [[INC67:%.*]] = add nuw nsw i16 [[K_010]], 1 ; CHECK-NEXT: [[CMP3:%.*]] = icmp slt i16 [[K_010]], 44 @@ -213,7 +217,7 @@ define void @test3() { ; CHECK: [[FOR_END64]]: ; CHECK-NEXT: [[TMP1]] = add nuw nsw i16 [[K_010]], 1 ; CHECK-NEXT: [[TMP6:%.*]] = icmp slt i16 [[K_010]], 44 -; CHECK-NEXT: br i1 [[TMP6]], label %[[FOR_COND37_PREHEADER]], label %[[FOR_END67]] +; CHECK-NEXT: br i1 [[TMP6]], label %[[FOR_BODY42_PREHEADER]], label %[[FOR_END67]] ; CHECK: [[FOR_END66]]: ; CHECK-NEXT: [[INC66:%.*]] = add nuw nsw i16 [[J_010]], 1 ; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i16 [[J_010]], 43 @@ -221,7 +225,7 @@ define void @test3() { ; CHECK: [[FOR_END67]]: ; CHECK-NEXT: [[TMP5]] = add nuw nsw i16 [[J_010]], 1 ; CHECK-NEXT: [[TMP4:%.*]] = icmp slt i16 [[J_010]], 43 -; CHECK-NEXT: br i1 [[TMP4]], label %[[FOR_COND38_PREHEADER_PREHEADER]], label %[[FOR_COND75_PREHEADER:.*]] +; CHECK-NEXT: br i1 [[TMP4]], label %[[FOR_COND37_PREHEADER]], label %[[FOR_COND75_PREHEADER:.*]] ; CHECK: [[FOR_END68]]: ; CHECK-NEXT: [[INC69]] = add nuw nsw i16 [[I_011]], 1 ; CHECK-NEXT: [[EXITCOND13_NOT:%.*]] = icmp eq i16 [[INC69]], 2 diff --git a/llvm/test/Transforms/LoopInterchange/reduction-extra-use-in-inner-loop.ll b/llvm/test/Transforms/LoopInterchange/reduction-extra-use-in-inner-loop.ll index 82ba8e234c5da..55e8e23505fbc 100644 --- a/llvm/test/Transforms/LoopInterchange/reduction-extra-use-in-inner-loop.ll +++ b/llvm/test/Transforms/LoopInterchange/reduction-extra-use-in-inner-loop.ll @@ -191,13 +191,15 @@ define i8 @extra_reduction_use_in_inner2(ptr noalias %A, ptr noalias %B) { ; CHECK-NEXT: [[J:%.*]] = phi i64 [ [[TMP2:%.*]], %[[MIDDLE_LATCH_SPLIT:.*]] ], [ 0, %[[MIDDLE_HEADER_PREHEADER]] ] ; CHECK-NEXT: [[SUM_MIDDLE:%.*]] = phi i8 [ [[SUM_MIDDLE_LCSSA:%.*]], %[[MIDDLE_LATCH_SPLIT]] ], [ [[SUM_OUTER]], %[[MIDDLE_HEADER_PREHEADER]] ] ; CHECK-NEXT: br label %[[INNER_PREHEADER:.*]] -; CHECK: [[INNER_PREHEADER]]: +; CHECK: [[MIDDLE_HEADER_SPLIT:.*]]: ; CHECK-NEXT: br label %[[INNER:.*]] -; CHECK: [[INNER]]: -; CHECK-NEXT: [[K:%.*]] = phi i64 [ [[TMP0:%.*]], %[[INNER_SPLIT:.*]] ], [ 0, %[[INNER_PREHEADER]] ] -; CHECK-NEXT: [[SUM_INNER:%.*]] = phi i8 [ [[SUM_INNER_NEXT:%.*]], %[[INNER_SPLIT]] ], [ [[SUM_MIDDLE]], %[[INNER_PREHEADER]] ] +; CHECK: [[INNER_PREHEADER]]: ; CHECK-NEXT: br label %[[INNER_SPLIT1:.*]] ; CHECK: [[INNER_SPLIT1]]: +; CHECK-NEXT: [[K:%.*]] = phi i64 [ [[TMP0:%.*]], %[[INNER_SPLIT:.*]] ], [ 0, %[[INNER_PREHEADER]] ] +; CHECK-NEXT: [[SUM_INNER:%.*]] = phi i8 [ [[SUM_INNER_NEXT:%.*]], %[[INNER_SPLIT]] ], [ [[SUM_MIDDLE]], %[[INNER_PREHEADER]] ] +; CHECK-NEXT: br label %[[MIDDLE_HEADER_SPLIT]] +; CHECK: [[INNER]]: ; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr [2 x [2 x i8]], ptr [[A]], i64 [[K]], i64 [[J]], i64 [[I]] ; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[GEP_A]], align 1 ; CHECK-NEXT: [[SUM_INNER_NEXT]] = add i8 [[SUM_INNER]], [[A]] @@ -209,7 +211,7 @@ define i8 @extra_reduction_use_in_inner2(ptr noalias %A, ptr noalias %B) { ; CHECK: [[INNER_SPLIT]]: ; CHECK-NEXT: [[TMP0]] = add i64 [[K]], 1 ; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[TMP0]], 2 -; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_LATCH_SPLIT]], label %[[INNER]] +; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_LATCH_SPLIT]], label %[[INNER_SPLIT1]] ; CHECK: [[MIDDLE_LATCH]]: ; CHECK-NEXT: [[J_NEXT:%.*]] = add i64 [[J]], 1 ; CHECK-NEXT: [[EC_J:%.*]] = icmp eq i64 [[J_NEXT]], 2 diff --git a/llvm/test/Transforms/LoopInterchange/reduction-not-involve-innermost.ll b/llvm/test/Transforms/LoopInterchange/reduction-not-involve-innermost.ll index 094e73db54ff3..0ec843f38a370 100644 --- a/llvm/test/Transforms/LoopInterchange/reduction-not-involve-innermost.ll +++ b/llvm/test/Transforms/LoopInterchange/reduction-not-involve-innermost.ll @@ -25,10 +25,12 @@ define void @f() { ; CHECK-NEXT: [[RED_I]] = phi i8 [ [[RED_J_NEXT_LCSSA:%.*]], %[[LOOP_J_LATCH_SPLIT]] ], [ 0, %[[LOOP_J_HEADER_PREHEADER]] ] ; CHECK-NEXT: br label %[[LOOP_I_HEADER_PREHEADER]] ; CHECK: [[LOOP_K]]: -; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[LOOP_I_HEADER]] ], [ [[K_INC:%.*]], %[[LOOP_K]] ] +; CHECK-NEXT: br label %[[LOOP_K1:.*]] +; CHECK: [[LOOP_K1]]: +; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[LOOP_K]] ], [ [[K_INC:%.*]], %[[LOOP_K1]] ] ; CHECK-NEXT: [[K_INC]] = add i64 [[K]], 1 ; CHECK-NEXT: [[EC_K:%.*]] = icmp eq i64 [[K_INC]], 10 -; CHECK-NEXT: br i1 [[EC_K]], label %[[LOOP_J_LATCH:.*]], label %[[LOOP_K]] +; CHECK-NEXT: br i1 [[EC_K]], label %[[LOOP_J_LATCH:.*]], label %[[LOOP_K1]] ; CHECK: [[LOOP_J_LATCH]]: ; CHECK-NEXT: [[RED_J_NEXT]] = or i8 [[RED_J]], 42 ; CHECK-NEXT: [[J_INC:%.*]] = add i64 [[J]], 1 diff --git a/llvm/test/Transforms/LoopInterchange/transform-stop-partway.ll b/llvm/test/Transforms/LoopInterchange/transform-stop-partway.ll index 74aefe9c8c74c..09246ddb3647d 100644 --- a/llvm/test/Transforms/LoopInterchange/transform-stop-partway.ll +++ b/llvm/test/Transforms/LoopInterchange/transform-stop-partway.ll @@ -1,29 +1,28 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 -; RUN: opt < %s -passes=loop-interchange -loop-interchange-profitabilities=ignore -debug-only=loop-interchange -S 2>&1 | FileCheck %s -; REQUIRES: asserts +; RUN: opt < %s -passes=loop-interchange -loop-interchange-profitabilities=ignore -S 2>&1 | FileCheck %s -; This test shows a case where the IR‑transformation phase of loop interchange -; can fail partway. As a result, the final output becomes a partially -; transformed IR. The IR‑transformation phase must not stop partway; in such -; cases, the entire process should bail out early during the legality‑check -; phase. - -; CHECK: Inner loop header does not have a unique successor +; In the past, this test showed a case where the IR‑transformation phase of +; loop interchange can fail partway. As a result, the final output became a +; partially transformed IR. The IR‑transformation phase must not stop partway; +; in such cases, the entire process should bail out early during the +; legality‑check phase. define void @f(i1 %cond, ptr %A) { ; CHECK-LABEL: define void @f( ; CHECK-SAME: i1 [[COND:%.*]], ptr [[A:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: br label %[[OUTER_HEADER_PREHEADER:.*]] -; CHECK: [[OUTER_HEADER_PREHEADER]]: +; CHECK: [[OUTER_HEADER_PREHEADER1:.*]]: ; CHECK-NEXT: br label %[[OUTER_HEADER:.*]] ; CHECK: [[OUTER_HEADER]]: -; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_INC:%.*]], %[[OUTER_LATCH:.*]] ], [ 0, %[[OUTER_HEADER_PREHEADER]] ] +; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_INC:%.*]], %[[OUTER_LATCH:.*]] ], [ 0, %[[OUTER_HEADER_PREHEADER1]] ] ; CHECK-NEXT: br label %[[INNER_HEADER_PREHEADER:.*]] -; CHECK: [[INNER_HEADER_PREHEADER]]: +; CHECK: [[OUTER_HEADER_PREHEADER]]: ; CHECK-NEXT: br label %[[INNER_HEADER:.*]] ; CHECK: [[INNER_HEADER]]: -; CHECK-NEXT: [[J:%.*]] = phi i64 [ [[TMP0:%.*]], %[[INNER_LATCH_SPLIT:.*]] ], [ 0, %[[INNER_HEADER_PREHEADER]] ] +; CHECK-NEXT: [[J:%.*]] = phi i64 [ [[TMP0:%.*]], %[[EXIT:.*]] ], [ 0, %[[OUTER_HEADER_PREHEADER]] ] +; CHECK-NEXT: br label %[[OUTER_HEADER_PREHEADER1]] +; CHECK: [[INNER_HEADER_PREHEADER]]: ; CHECK-NEXT: br i1 [[COND]], label %[[INNER_BODY_0:.*]], label %[[INNER_BODY_1:.*]] ; CHECK: [[INNER_BODY_0]]: ; CHECK-NEXT: br label %[[INNER_LATCH:.*]] @@ -34,20 +33,20 @@ define void @f(i1 %cond, ptr %A) { ; CHECK-NEXT: store i8 1, ptr [[GEP]], align 1 ; CHECK-NEXT: [[J_INC:%.*]] = add i64 [[J]], 1 ; CHECK-NEXT: [[EC_J:%.*]] = icmp eq i64 [[J_INC]], 42 -; CHECK-NEXT: br label %[[INNER_LATCH_SPLIT]] -; CHECK: [[INNER_LATCH_SPLIT]]: +; CHECK-NEXT: br label %[[INNER_LATCH_SPLIT:.*]] +; CHECK: [[EXIT]]: ; CHECK-NEXT: [[TMP0]] = add i64 [[J]], 1 ; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[TMP0]], 42 ; CHECK-NEXT: br i1 [[TMP1]], label %[[INNER_EXIT:.*]], label %[[INNER_HEADER]] ; CHECK: [[BB2:.*:]] ; CHECK-NEXT: unreachable -; CHECK: [[INNER_EXIT]]: +; CHECK: [[INNER_LATCH_SPLIT]]: ; CHECK-NEXT: br label %[[OUTER_LATCH]] ; CHECK: [[OUTER_LATCH]]: ; CHECK-NEXT: [[I_INC]] = add i64 [[I]], 1 ; CHECK-NEXT: [[EC_I:%.*]] = icmp eq i64 [[I_INC]], 42 -; CHECK-NEXT: br i1 [[EC_I]], label %[[EXIT:.*]], label %[[OUTER_HEADER]] -; CHECK: [[EXIT]]: +; CHECK-NEXT: br i1 [[EC_I]], label %[[EXIT]], label %[[OUTER_HEADER]] +; CHECK: [[INNER_EXIT]]: ; CHECK-NEXT: ret void ; entry: _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
