https://github.com/momchil-velikov created https://github.com/llvm/llvm-project/pull/210337
RFC/discussion: https://lists.llvm.org/pipermail/llvm-dev/2021-September/152665.html This patch is a update of https://reviews.llvm.org/D110817 This patch implements simple hoisting of instructions from two single-predecessor blocks to their common predecessor, as a subroutine in the GVN pass. The patch pairs two instructions (A and B) with the same value number, moves A to the predecessor block, replaces all uses of B with A, and deletes B. Outline of the algorithm follows: Scan the then-block to collect hoist candidates ("then-" and "else-" prefixes are purely naming and have no connection to the condition in the predecessor block) Scan the else-block for hoist candidates, that match some already selected instruction from the then-block. During both scans, instructions which are not guaranteed to transfer control to the following instruction act as "hoist barriers" - after we encounter such an instruction, we select for potential hoisting/merge only instructions, which are safe to execute speculatively. Also instructions which read/write memory are not considered for hoisting, subject for a follow-up patch. The hoist barriers can itself be hoisted, opening opportunities for other instructions. For each hoist candidate pair, the immediately preceding hoist barriers from then- and else-blocks are recorded as prerequisites for hoisting the pair. Next we try hoist to hoist each candidate pair. We begin by trying to hoist dependencies of the then-instruction, which would be its immediately preceding hoist barrier and its operands. Each of these dependencies must already be in a dominating block or is itself paired with an instruction from the else-block. If we cannot hoist an dependency for whatever reason, the we stop trying to hoist the pair. Now that all the operands of the then-instruction are in a dominating block, we check the barriers/operands of the else-instruction. They all must already be in a dominating block, either initially or as a result of hoisting barriers/operands of the then-instruction. If any dependency is still in the else-block, we stop trying to hoist the pair. As a last step, we move the then-instruction to the predecessor block and delete the else-instruction. >From 9d7161d7eb6d82fa4564428d0a0bbccd356d2a2d Mon Sep 17 00:00:00 2001 From: Momchil Velikov <[email protected]> Date: Wed, 8 Jul 2026 12:48:20 +0000 Subject: [PATCH] [GVN] Simple GVN-based hoisring of scalars RFC/discussion: https://lists.llvm.org/pipermail/llvm-dev/2021-September/152665.html This patch is a update of https://reviews.llvm.org/D110817 This patch implements simple hoisting of instructions from two single-predecessor blocks to their common predecessor, as a subroutine in the GVN pass. The patch pairs two instructions (A and B) with the same value number, moves A to the predecessor block, replaces all uses of B with A, and deletes B. Outline of the algorithm follows: Scan the then-block to collect hoist candidates ("then-" and "else-" prefixes are purely naming and have no connection to the condition in the predecessor block) Scan the else-block for hoist candidates, that match some already selected instruction from the then-block. During both scans, instructions which are not guaranteed to transfer control to the following instruction act as "hoist barriers" - after we encounter such an instruction, we select for potential hoisting/merge only instructions, which are safe to execute speculatively. Also instructions which read/write memory are not considered for hoisting, subject for a follow-up patch. The hoist barriers can itself be hoisted, opening opportunities for other instructions. For each hoist candidate pair, the immediately preceding hoist barriers from then- and else-blocks are recorded as prerequisites for hoisting the pair. Next we try hoist to hoist each candidate pair. We begin by trying to hoist dependencies of the then-instruction, which would be its immediately preceding hoist barrier and its operands. Each of these dependencies must already be in a dominating block or is itself paired with an instruction from the else-block. If we cannot hoist an dependency for whatever reason, the we stop trying to hoist the pair. Now that all the operands of the then-instruction are in a dominating block, we check the barriers/operands of the else-instruction. They all must already be in a dominating block, either initially or as a result of hoisting barriers/operands of the then-instruction. If any dependency is still in the else-block, we stop trying to hoist the pair. As a last step, we move the then-instruction to the predecessor block and delete the else-instruction. --- .../CodeGen/attr-counted-by-with-sanitizers.c | 54 ++-- llvm/include/llvm/Transforms/Scalar/GVN.h | 25 ++ llvm/lib/Transforms/Scalar/GVN.cpp | 232 +++++++++++++++++- .../CodeGen/AMDGPU/memcpy-crash-issue63986.ll | 147 +++++------ .../NVPTX/gvn-scalar-pre-reg-pressure.ll | 4 +- .../Transforms/GVN/2012-05-22-PreCrash.ll | 2 +- .../GVN/PRE/load-pre-across-backedge.ll | 4 +- llvm/test/Transforms/GVN/PRE/local-pre.ll | 4 +- llvm/test/Transforms/GVN/PRE/no-scalar-pre.ll | 8 +- llvm/test/Transforms/GVN/PRE/phi-translate.ll | 4 +- llvm/test/Transforms/GVN/PRE/pre-basic-add.ll | 6 +- .../GVN/PRE/pre-load-through-select.ll | 4 +- .../Transforms/GVN/PRE/pre-no-cost-phi.ll | 4 +- .../test/Transforms/GVN/PRE/pre-poison-add.ll | 4 +- llvm/test/Transforms/GVN/freeze.ll | 2 +- llvm/test/Transforms/GVN/gc_relocate.ll | 2 +- .../Transforms/GVN/simple-gvn-hoist-limits.ll | 15 +- .../GVN/simple-gvn-hoist-scalars.ll | 107 +++++--- 18 files changed, 449 insertions(+), 179 deletions(-) diff --git a/clang/test/CodeGen/attr-counted-by-with-sanitizers.c b/clang/test/CodeGen/attr-counted-by-with-sanitizers.c index e840db632957e..81be6bdad9936 100644 --- a/clang/test/CodeGen/attr-counted-by-with-sanitizers.c +++ b/clang/test/CodeGen/attr-counted-by-with-sanitizers.c @@ -234,16 +234,16 @@ size_t test_return_bdos_cast_of_whole_struct(struct annotated *p) { // SANITIZE-WITH-ATTR: [[CONT1]]: // SANITIZE-WITH-ATTR-NEXT: [[FLEXIBLE_ARRAY_MEMBER_SIZE:%.*]] = shl i32 [[DOTCOUNTED_BY_LOAD]], 2 // SANITIZE-WITH-ATTR-NEXT: [[TMP1:%.*]] = icmp ult i32 [[INDEX]], [[DOTCOUNTED_BY_LOAD]], !nosanitize [[META6]] -// SANITIZE-WITH-ATTR-NEXT: [[IDXPROM:%.*]] = zext i32 [[INDEX]] to i64 +// SANITIZE-WITH-ATTR-NEXT: [[TMP2:%.*]] = zext i32 [[INDEX]] to i64 // SANITIZE-WITH-ATTR-NEXT: br i1 [[TMP1]], label %[[CONT12:.*]], label %[[HANDLER_OUT_OF_BOUNDS8:.*]], !prof [[PROF7]], !nosanitize [[META6]] // SANITIZE-WITH-ATTR: [[HANDLER_OUT_OF_BOUNDS8]]: -// SANITIZE-WITH-ATTR-NEXT: tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB6:[0-9]+]], i64 [[IDXPROM]]) #[[ATTR7]], !nosanitize [[META6]] +// SANITIZE-WITH-ATTR-NEXT: tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB6:[0-9]+]], i64 [[TMP2]]) #[[ATTR7]], !nosanitize [[META6]] // SANITIZE-WITH-ATTR-NEXT: unreachable, !nosanitize [[META6]] // SANITIZE-WITH-ATTR: [[CONT12]]: // SANITIZE-WITH-ATTR-NEXT: [[RESULT:%.*]] = add i32 [[FLEXIBLE_ARRAY_MEMBER_SIZE]], 244 -// SANITIZE-WITH-ATTR-NEXT: [[TMP2:%.*]] = and i32 [[RESULT]], 252 -// SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARRAY]], i64 [[IDXPROM]] -// SANITIZE-WITH-ATTR-NEXT: store i32 [[TMP2]], ptr [[ARRAYIDX10]], align 4, !tbaa [[INT_TBAA8]] +// SANITIZE-WITH-ATTR-NEXT: [[TMP3:%.*]] = and i32 [[RESULT]], 252 +// SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARRAY]], i64 [[TMP2]] +// SANITIZE-WITH-ATTR-NEXT: store i32 [[TMP3]], ptr [[ARRAYIDX10]], align 4, !tbaa [[INT_TBAA8]] // SANITIZE-WITH-ATTR-NEXT: [[DOTNOT79:%.*]] = icmp eq i32 [[DOTCOUNTED_BY_LOAD]], 3 // SANITIZE-WITH-ATTR-NEXT: br i1 [[DOTNOT79]], label %[[HANDLER_OUT_OF_BOUNDS18:.*]], label %[[CONT19:.*]], !prof [[PROF9:![0-9]+]], !nosanitize [[META6]] // SANITIZE-WITH-ATTR: [[HANDLER_OUT_OF_BOUNDS18]]: @@ -251,37 +251,37 @@ size_t test_return_bdos_cast_of_whole_struct(struct annotated *p) { // SANITIZE-WITH-ATTR-NEXT: unreachable, !nosanitize [[META6]] // SANITIZE-WITH-ATTR: [[CONT19]]: // SANITIZE-WITH-ATTR-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[INDEX]], 1 -// SANITIZE-WITH-ATTR-NEXT: [[TMP3:%.*]] = icmp samesign ult i32 [[ADD]], [[DOTCOUNTED_BY_LOAD]], !nosanitize [[META6]] -// SANITIZE-WITH-ATTR-NEXT: [[IDXPROM31:%.*]] = zext nneg i32 [[ADD]] to i64 -// SANITIZE-WITH-ATTR-NEXT: br i1 [[TMP3]], label %[[CONT38:.*]], label %[[HANDLER_OUT_OF_BOUNDS34:.*]], !prof [[PROF7]], !nosanitize [[META6]] +// SANITIZE-WITH-ATTR-NEXT: [[TMP4:%.*]] = icmp samesign ult i32 [[ADD]], [[DOTCOUNTED_BY_LOAD]], !nosanitize [[META6]] +// SANITIZE-WITH-ATTR-NEXT: [[TMP5:%.*]] = zext nneg i32 [[ADD]] to i64 +// SANITIZE-WITH-ATTR-NEXT: br i1 [[TMP4]], label %[[CONT38:.*]], label %[[HANDLER_OUT_OF_BOUNDS34:.*]], !prof [[PROF7]], !nosanitize [[META6]] // SANITIZE-WITH-ATTR: [[HANDLER_OUT_OF_BOUNDS34]]: -// SANITIZE-WITH-ATTR-NEXT: tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB8:[0-9]+]], i64 [[IDXPROM31]]) #[[ATTR7]], !nosanitize [[META6]] +// SANITIZE-WITH-ATTR-NEXT: tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB8:[0-9]+]], i64 [[TMP5]]) #[[ATTR7]], !nosanitize [[META6]] // SANITIZE-WITH-ATTR-NEXT: unreachable, !nosanitize [[META6]] // SANITIZE-WITH-ATTR: [[CONT38]]: // SANITIZE-WITH-ATTR-NEXT: [[RESULT25:%.*]] = add i32 [[FLEXIBLE_ARRAY_MEMBER_SIZE]], 240 -// SANITIZE-WITH-ATTR-NEXT: [[TMP4:%.*]] = and i32 [[RESULT25]], 252 -// SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX36:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARRAY]], i64 [[IDXPROM31]] -// SANITIZE-WITH-ATTR-NEXT: store i32 [[TMP4]], ptr [[ARRAYIDX36]], align 4, !tbaa [[INT_TBAA8]] +// SANITIZE-WITH-ATTR-NEXT: [[TMP6:%.*]] = and i32 [[RESULT25]], 252 +// SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX36:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARRAY]], i64 [[TMP5]] +// SANITIZE-WITH-ATTR-NEXT: store i32 [[TMP6]], ptr [[ARRAYIDX36]], align 4, !tbaa [[INT_TBAA8]] // SANITIZE-WITH-ATTR-NEXT: [[DOTNOT:%.*]] = icmp ugt i32 [[FAM_IDX]], [[DOTCOUNTED_BY_LOAD]], !nosanitize [[META6]] // SANITIZE-WITH-ATTR-NEXT: br i1 [[DOTNOT]], label %[[HANDLER_OUT_OF_BOUNDS45:.*]], label %[[CONT46:.*]], !prof [[PROF9]], !nosanitize [[META6]] // SANITIZE-WITH-ATTR: [[HANDLER_OUT_OF_BOUNDS45]]: -// SANITIZE-WITH-ATTR-NEXT: [[TMP5:%.*]] = zext i32 [[FAM_IDX]] to i64, !nosanitize [[META6]] -// SANITIZE-WITH-ATTR-NEXT: tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB9:[0-9]+]], i64 [[TMP5]]) #[[ATTR7]], !nosanitize [[META6]] +// SANITIZE-WITH-ATTR-NEXT: [[TMP7:%.*]] = zext i32 [[FAM_IDX]] to i64, !nosanitize [[META6]] +// SANITIZE-WITH-ATTR-NEXT: tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB9:[0-9]+]], i64 [[TMP7]]) #[[ATTR7]], !nosanitize [[META6]] // SANITIZE-WITH-ATTR-NEXT: unreachable, !nosanitize [[META6]] // SANITIZE-WITH-ATTR: [[CONT46]]: // SANITIZE-WITH-ATTR-NEXT: [[ADD59:%.*]] = add nuw nsw i32 [[INDEX]], 2 -// SANITIZE-WITH-ATTR-NEXT: [[TMP6:%.*]] = icmp samesign ult i32 [[ADD59]], [[DOTCOUNTED_BY_LOAD]], !nosanitize [[META6]] -// SANITIZE-WITH-ATTR-NEXT: [[IDXPROM60:%.*]] = zext nneg i32 [[ADD59]] to i64 -// SANITIZE-WITH-ATTR-NEXT: br i1 [[TMP6]], label %[[CONT67:.*]], label %[[HANDLER_OUT_OF_BOUNDS63:.*]], !prof [[PROF7]], !nosanitize [[META6]] +// SANITIZE-WITH-ATTR-NEXT: [[TMP8:%.*]] = icmp samesign ult i32 [[ADD59]], [[DOTCOUNTED_BY_LOAD]], !nosanitize [[META6]] +// SANITIZE-WITH-ATTR-NEXT: [[TMP9:%.*]] = zext nneg i32 [[ADD59]] to i64 +// SANITIZE-WITH-ATTR-NEXT: br i1 [[TMP8]], label %[[CONT67:.*]], label %[[HANDLER_OUT_OF_BOUNDS63:.*]], !prof [[PROF7]], !nosanitize [[META6]] // SANITIZE-WITH-ATTR: [[HANDLER_OUT_OF_BOUNDS63]]: -// SANITIZE-WITH-ATTR-NEXT: tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB10:[0-9]+]], i64 [[IDXPROM60]]) #[[ATTR7]], !nosanitize [[META6]] +// SANITIZE-WITH-ATTR-NEXT: tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB10:[0-9]+]], i64 [[TMP9]]) #[[ATTR7]], !nosanitize [[META6]] // SANITIZE-WITH-ATTR-NEXT: unreachable, !nosanitize [[META6]] // SANITIZE-WITH-ATTR: [[CONT67]]: -// SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX65:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARRAY]], i64 [[IDXPROM60]] +// SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX65:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[ARRAY]], i64 [[TMP9]] // SANITIZE-WITH-ATTR-NEXT: [[DOTTR:%.*]] = sub nsw i32 [[DOTCOUNTED_BY_LOAD]], [[FAM_IDX]] -// SANITIZE-WITH-ATTR-NEXT: [[TMP7:%.*]] = shl i32 [[DOTTR]], 2 -// SANITIZE-WITH-ATTR-NEXT: [[TMP8:%.*]] = and i32 [[TMP7]], 252 -// SANITIZE-WITH-ATTR-NEXT: store i32 [[TMP8]], ptr [[ARRAYIDX65]], align 4, !tbaa [[INT_TBAA8]] +// SANITIZE-WITH-ATTR-NEXT: [[TMP10:%.*]] = shl i32 [[DOTTR]], 2 +// SANITIZE-WITH-ATTR-NEXT: [[TMP11:%.*]] = and i32 [[TMP10]], 252 +// SANITIZE-WITH-ATTR-NEXT: store i32 [[TMP11]], ptr [[ARRAYIDX65]], align 4, !tbaa [[INT_TBAA8]] // SANITIZE-WITH-ATTR-NEXT: ret void // // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test_assign_size_of_pointer_into_fam( @@ -483,15 +483,14 @@ size_t test_return_bdos_of_fam_in_anon_struct(struct anon_struct *p) { // SANITIZE-WITH-ATTR-NEXT: [[DOTCOUNTED_BY_LOAD:%.*]] = load i8, ptr [[TMP0]], align 4 // SANITIZE-WITH-ATTR-NEXT: [[TMP1:%.*]] = zext i8 [[DOTCOUNTED_BY_LOAD]] to i32, !nosanitize [[META6]] // SANITIZE-WITH-ATTR-NEXT: [[TMP2:%.*]] = icmp ult i32 [[INDEX]], [[TMP1]], !nosanitize [[META6]] +// SANITIZE-WITH-ATTR-NEXT: [[TMP3:%.*]] = zext i32 [[INDEX]] to i64 // SANITIZE-WITH-ATTR-NEXT: br i1 [[TMP2]], label %[[CONT7:.*]], label %[[HANDLER_OUT_OF_BOUNDS:.*]], !prof [[PROF7]], !nosanitize [[META6]] // SANITIZE-WITH-ATTR: [[HANDLER_OUT_OF_BOUNDS]]: -// SANITIZE-WITH-ATTR-NEXT: [[TMP3:%.*]] = zext i32 [[INDEX]] to i64, !nosanitize [[META6]] // SANITIZE-WITH-ATTR-NEXT: tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB15:[0-9]+]], i64 [[TMP3]]) #[[ATTR7]], !nosanitize [[META6]] // SANITIZE-WITH-ATTR-NEXT: unreachable, !nosanitize [[META6]] // SANITIZE-WITH-ATTR: [[CONT7]]: // SANITIZE-WITH-ATTR-NEXT: [[INTS:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 9 -// SANITIZE-WITH-ATTR-NEXT: [[IDXPROM:%.*]] = zext nneg i32 [[INDEX]] to i64 -// SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[INTS]], i64 [[IDXPROM]] +// SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[INTS]], i64 [[TMP3]] // SANITIZE-WITH-ATTR-NEXT: store i8 -1, ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA10:![0-9]+]] // SANITIZE-WITH-ATTR-NEXT: ret void // @@ -529,15 +528,14 @@ size_t test_return_bdos_of_anon_struct(struct union_of_fams *p) { // SANITIZE-WITH-ATTR-NEXT: [[COUNTED_BY_LOAD:%.*]] = load i8, ptr [[TMP0]], align 4 // SANITIZE-WITH-ATTR-NEXT: [[TMP1:%.*]] = zext i8 [[COUNTED_BY_LOAD]] to i32, !nosanitize [[META6]] // SANITIZE-WITH-ATTR-NEXT: [[TMP2:%.*]] = icmp ult i32 [[INDEX]], [[TMP1]], !nosanitize [[META6]] +// SANITIZE-WITH-ATTR-NEXT: [[TMP3:%.*]] = zext i32 [[INDEX]] to i64 // SANITIZE-WITH-ATTR-NEXT: br i1 [[TMP2]], label %[[CONT14:.*]], label %[[HANDLER_OUT_OF_BOUNDS:.*]], !prof [[PROF7]], !nosanitize [[META6]] // SANITIZE-WITH-ATTR: [[HANDLER_OUT_OF_BOUNDS]]: -// SANITIZE-WITH-ATTR-NEXT: [[TMP3:%.*]] = zext i32 [[INDEX]] to i64, !nosanitize [[META6]] // SANITIZE-WITH-ATTR-NEXT: tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB16:[0-9]+]], i64 [[TMP3]]) #[[ATTR7]], !nosanitize [[META6]] // SANITIZE-WITH-ATTR-NEXT: unreachable, !nosanitize [[META6]] // SANITIZE-WITH-ATTR: [[CONT14]]: // SANITIZE-WITH-ATTR-NEXT: [[INTS:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 9 -// SANITIZE-WITH-ATTR-NEXT: [[IDXPROM:%.*]] = zext nneg i32 [[INDEX]] to i64 -// SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[INTS]], i64 [[IDXPROM]] +// SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[INTS]], i64 [[TMP3]] // SANITIZE-WITH-ATTR-NEXT: store i8 [[COUNTED_BY_LOAD]], ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA10]] // SANITIZE-WITH-ATTR-NEXT: ret void // diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h index 46c54363298a2..dd0fbcee1e253 100644 --- a/llvm/include/llvm/Transforms/Scalar/GVN.h +++ b/llvm/include/llvm/Transforms/Scalar/GVN.h @@ -322,6 +322,24 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> { // List of critical edges to be split between iterations. SmallVector<std::pair<Instruction *, unsigned>, 4> ToSplit; + // A pair of instructions with the same value number to be hoisted and merged, + // together with their respective hoist barriers. A pair of insructions can be + // hoisted iff both their barriers (if not null) are hoisted as well. The + // `WeakVH` is used to track when the barrier instruction itself is hoisted. + struct HoistPair { + Instruction *ThenI = nullptr; + Instruction *ThenB = nullptr; + Instruction *ElseI = nullptr; + WeakVH ElseB = nullptr; + }; + + /// A mapping from value numbers to a pair of instructions. This map + /// stores pairs of instructions with the same value number, from two blocks + /// having a single common predecessor, for the duration of a single top level + /// iteration in `performHoist`. + using HoistMap = DenseMap<uint32_t, HoistPair>; + HoistMap HoistPairs; + public: GVNPass(GVNOptions Options = {}) : Options(Options) {} @@ -507,6 +525,13 @@ class GVNPass : public OptionalPassInfoMixin<GVNPass> { bool performScalarPRE(Instruction *I); bool performPRE(Function &F); + void collectHoistCandidates(BasicBlock *ThenBB); + void matchHoistCandidates(BasicBlock *ElseBB); + void replaceInstruction(Instruction *I, Instruction *Repl); + std::pair<bool, bool> hoistPair(BasicBlock *DestBB, BasicBlock *ThenBB, + BasicBlock *ElseBB, Instruction *ThenI); + bool performHoist(Function &F); + /// Main entry point for the GVN pass. Also used by the GVNLegacyPass. bool runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT, const TargetLibraryInfo &RunTLI, AAResults &RunAA, diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp index 10b9dd77a3acb..b8bec704d9411 100644 --- a/llvm/lib/Transforms/Scalar/GVN.cpp +++ b/llvm/lib/Transforms/Scalar/GVN.cpp @@ -118,7 +118,8 @@ GVNEnableSplitBackedgeInLoadPRE("enable-split-backedge-in-load-pre", static cl::opt<bool> GVNEnableMemDep("enable-gvn-memdep", cl::init(true)); static cl::opt<bool> GVNEnableMemorySSA("enable-gvn-memoryssa", cl::init(false)); - +static cl::opt<bool> GVNEnableSimpleGVNHoist("enable-simple-gvn-hoist", + cl::init(true)); static cl::opt<unsigned> ScanUsersLimit( "gvn-scan-users-limit", cl::Hidden, cl::init(100), cl::desc("The number of memory accesses to scan in a block in reaching " @@ -3723,6 +3724,230 @@ bool GVNPass::performPRE(Function &F) { return Changed; } +// Won't reorder above these instructions. +static bool isHoistBarrier(const Instruction &I) { + return I.mayWriteToMemory() || I.mayHaveSideEffects() || !isGuaranteedToTransferExecutionToSuccessor(&I); +} + +static bool isHoistCandidate(const Instruction &I) { + if (I.mayReadOrWriteMemory()) + return false; + if (!isa<CallBase>(I)) + return true; + const auto &CB = cast<CallBase>(I); + if (CB.isMustTailCall() || CB.cannotMerge()) + return false; + return true; +} + +void GVNPass::collectHoistCandidates(BasicBlock *BB) { + uint32_t Depth = 0; + Instruction *Barrier = nullptr; + for (Instruction &I : *BB) { + if (++Depth > MaxNumInsnsPerBlock) + break; + if (I.isTerminator()) + break; + if (isa<PHINode>(I)) + continue; + if (isHoistCandidate(I)) { + HoistPair &HP = HoistPairs[VN.lookupOrAdd(&I)]; + HP.ThenI = &I; + HP.ThenB = isSafeToSpeculativelyExecute(&I) ? nullptr : Barrier; + } + Barrier = isHoistBarrier(I) ? &I : Barrier; + } +} + +void GVNPass::matchHoistCandidates(BasicBlock *BB) { + uint32_t Depth = 0; + Instruction *Barrier = nullptr; + for (Instruction &I : *BB) { + if (++Depth > MaxNumInsnsPerBlock) + break; + if (I.isTerminator()) + break; + if (isa<PHINode>(I)) + continue; + if (isHoistCandidate(I)) { + uint32_t N = VN.lookupOrAdd(&I); + if (auto It = HoistPairs.find(N); + It != HoistPairs.end() && It->second.ElseI == nullptr) { + It->second.ElseI = &I; + It->second.ElseB = isSafeToSpeculativelyExecute(&I) ? nullptr : Barrier; + } + } + Barrier = isHoistBarrier(I) ? &I : Barrier; + } +} + +void GVNPass::replaceInstruction(Instruction *I, Instruction *Repl) { + LLVM_DEBUG(dbgs() << "Simple GVNHoist: replacing" << *I << " by" << *Repl + << '\n';); + patchReplacementInstruction(I, Repl); + ICF->removeUsersOf(I); + I->replaceAllUsesWith(Repl); + salvageKnowledge(I, AC); + salvageDebugInfo(*I); + if (MD) + MD->removeInstruction(I); + if (MSSAU) + MSSAU->removeMemoryAccess(I); + VN.erase(I); + ICF->removeInstruction(I); + LLVM_DEBUG(verifyRemoved(I)); + I->eraseFromParent(); + ++NumGVNInstr; +} + +// Only hoist instructions from the "then" block. +// Each hoisted instruction must be paired with an instruction from the "else" +// block. +std::pair<bool, bool> GVNPass::hoistPair(BasicBlock *DestBB, BasicBlock *ThenBB, + BasicBlock *ElseBB, Instruction *ThenI) { + // If the instruction is moved out of the "then" block there's nothing to do. + if (ThenI->getParent() != ThenBB) + return {false, false}; + + // Instruction must have already been selected for hoisting and matched with + // another instruction. + auto It = HoistPairs.find(VN.lookupOrAdd(ThenI)); + if (It == HoistPairs.end()) + return {false, true}; + + // Do not attempt to hoist a pair twice. If `ElseI` is nullptr, it means + // either there was no match for `ThenI` or there was already an attempt + // (successful or not) to hoist the pair. + Instruction *ElseI = It->second.ElseI; + if (ElseI == nullptr) + return {false, true}; + It->second.ElseI = nullptr; + + assert(ElseI->getParent() == ElseBB && "Instruction already removed"); + assert(!ThenI->mayReadOrWriteMemory() && !ElseI->mayReadOrWriteMemory() && + "Memory read/write instructions must not be hoisted."); + + bool Change = false; + + // Hoist the `Then` barrier, if any. + Instruction *ThenB = It->second.ThenB; + if (ThenB != nullptr && ThenB->getParent() == ThenBB) { + auto [LocalChange, StopHoisting] = hoistPair(DestBB, ThenBB, ElseBB, ThenB); + Change |= LocalChange; + if (StopHoisting) + return {Change, true}; + } + + // Check the `Else` barrier instruction, if any, was deleted from the `Else` + // block as a result of a previous hoisting. + if (dyn_cast_or_null<Instruction>(It->second.ElseB) != nullptr) + return {Change, true}; + + // Hoist operands. Begin by hoisting all of the operands of the "then" + // instruction, then check that all of the operands of the "else" instruction + // strictly dominate its block. + for (unsigned I = 0, N = ThenI->getNumOperands(); I < N; ++I) { + auto *Op = dyn_cast<Instruction>(ThenI->getOperand(I)); + if (Op == nullptr) + continue; + auto [LocalChange, StopHoisting] = hoistPair(DestBB, ThenBB, ElseBB, Op); + Change |= LocalChange; + if (StopHoisting) + return {Change, true}; + } + + for (unsigned I = 0, N = ElseI->getNumOperands(); I < N; ++I) { + auto *Op = dyn_cast<Instruction>(ElseI->getOperand(I)); + if (Op == nullptr) + continue; + if (Op->getParent() == ElseBB) + return {Change, true}; + } + + // Hoist one of the instructions and replace all uses of the other with it. + ICF->removeInstruction(ThenI); + ICF->insertInstructionTo(ThenI, DestBB); + ThenI->moveBefore(DestBB->getTerminator()->getIterator()); + replaceInstruction(ElseI, ThenI); + + return {true, false}; +} + +// Determine if an instruction should be used to initiate hoisting a +// dependency chain. The aim is to avoid separating instructions, for which it's +// (heuristically) considered better to keep them together, as it's common that +// they can be fused in some way. An instruction, which is denied hoisting by +// this function can still be hoisted if it appears as a dependency (e.g +// operand) of another hoisted instruction. +static bool shouldNotInitiateHoisting(const Instruction *I) { + // Don't separate GEP's from their loads/stores. + if (isa<GetElementPtrInst>(I)) + return true; + const bool IsBinop = isa<BinaryOperator>(I); + for (const User *U : I->users()) { + // Don't separate conditions from `br` or `select`. + if ((isa<CondBrInst>(U) || isa<SelectInst>(U)) && U->getOperand(0) == I) + return true; + // Don't separate a value from converting that value to a boolean by + // comparing it to zero. + if (!IsBinop) + continue; + const auto *ICmp = dyn_cast<ICmpInst>(U); + if (ICmp == nullptr || (ICmp->getPredicate() != CmpInst::ICMP_EQ && + ICmp->getPredicate() != CmpInst::ICMP_NE)) + continue; + const auto *Zero = dyn_cast<ConstantInt>(ICmp->getOperand(1)); + if (Zero != nullptr && Zero->isZero()) + return true; + } + return false; +} + +// Perform trivial hoisting of values from two blocks to their common +// predecessor. +bool GVNPass::performHoist(Function &F) { + LLVM_DEBUG(dbgs() << "Simple GVNHoist: running on function " << F.getName() + << '\n';); + bool Change = false; + ReversePostOrderTraversal<Function *> RPOT(&F); + for (BasicBlock *BB : RPOT) { + // Check we have a block of the desired shape. + auto *BI = dyn_cast<CondBrInst>(BB->getTerminator()); + if (!BI) + continue; + + BasicBlock *Then = BI->getSuccessor(0); + BasicBlock *Else = BI->getSuccessor(1); + + if (!Then->getSinglePredecessor() || !Else->getSinglePredecessor()) + continue; + + LLVM_DEBUG(dbgs() << "Simple GVNHoist: looking at block " << BB->getName() + << '\n'); + + // Collect all hoistable instructions from the smaller block, then match + // them by value number with the instructions from the other block. + if (Then->size() > Else->size()) + std::swap(Then, Else); + + HoistPairs.clear(); + collectHoistCandidates(Then); + matchHoistCandidates(Else); + + // Hoist matched pairs. + for (const auto &P : HoistPairs) { + const HoistPair &HP = P.second; + if (shouldNotInitiateHoisting(HP.ThenI)) + continue; + auto [LocalChange, _] = hoistPair(BB, Then, Else, HP.ThenI); + Change |= LocalChange; + } + } + + return Change; +} + + /// runOnFunction - This is the main transformation entry point for a function. bool GVNPass::runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT, const TargetLibraryInfo &RunTLI, AAResults &RunAA, @@ -3783,6 +4008,11 @@ bool GVNPass::runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT, } } + if (GVNEnableSimpleGVNHoist) { + LeaderTable.clear(); + Changed |= performHoist(F); + } + // FIXME: Should perform GVN again after PRE does something. PRE can move // computations into blocks where they become fully redundant. Note that // we can't do this until PRE's critical edge splitting updates memdep. diff --git a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll index d90c9771c3e4a..531b4ca00f2ec 100644 --- a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll +++ b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll @@ -8,144 +8,127 @@ define void @issue63986(i64 %0, i64 %idxprom, ptr inreg %ptr) { ; CHECK: ; %bb.0: ; %entry ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; CHECK-NEXT: v_lshlrev_b64 v[4:5], 6, v[2:3] -; CHECK-NEXT: v_mov_b32_e32 v6, s17 -; CHECK-NEXT: v_add_co_u32_e32 v8, vcc, s16, v4 -; CHECK-NEXT: v_addc_co_u32_e32 v9, vcc, v6, v5, vcc +; CHECK-NEXT: v_mov_b32_e32 v2, s17 +; CHECK-NEXT: v_add_co_u32_e32 v6, vcc, s16, v4 +; CHECK-NEXT: v_addc_co_u32_e32 v7, vcc, v2, v5, vcc ; CHECK-NEXT: s_mov_b64 s[4:5], 0 ; CHECK-NEXT: .LBB0_1: ; %dynamic-memcpy-expansion-main-body ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: v_mov_b32_e32 v7, s5 -; CHECK-NEXT: v_mov_b32_e32 v6, s4 -; CHECK-NEXT: flat_load_dwordx4 v[10:13], v[6:7] -; CHECK-NEXT: v_add_co_u32_e32 v6, vcc, s4, v8 +; CHECK-NEXT: v_mov_b32_e32 v2, s4 +; CHECK-NEXT: v_mov_b32_e32 v3, s5 +; CHECK-NEXT: flat_load_dwordx4 v[8:11], v[2:3] +; CHECK-NEXT: v_add_co_u32_e32 v2, vcc, s4, v6 ; CHECK-NEXT: s_add_u32 s4, s4, 16 ; CHECK-NEXT: s_addc_u32 s5, s5, 0 ; CHECK-NEXT: v_cmp_lt_u64_e64 s[6:7], s[4:5], 32 -; CHECK-NEXT: v_addc_co_u32_e32 v7, vcc, v9, v7, vcc +; CHECK-NEXT: v_addc_co_u32_e32 v3, vcc, v7, v3, vcc ; CHECK-NEXT: s_and_b64 vcc, exec, s[6:7] ; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; CHECK-NEXT: flat_store_dwordx4 v[6:7], v[10:13] +; CHECK-NEXT: flat_store_dwordx4 v[2:3], v[8:11] ; CHECK-NEXT: s_cbranch_vccnz .LBB0_1 ; CHECK-NEXT: ; %bb.2: ; %dynamic-memcpy-expansion-residual-cond -; CHECK-NEXT: s_branch .LBB0_4 -; CHECK-NEXT: ; %bb.3: -; CHECK-NEXT: s_mov_b64 s[4:5], -1 -; CHECK-NEXT: ; implicit-def: $vgpr6_vgpr7 -; CHECK-NEXT: s_and_b64 s[4:5], s[4:5], exec -; CHECK-NEXT: s_cselect_b32 s4, 1, 0 -; CHECK-NEXT: s_cmp_lg_u32 s4, 1 -; CHECK-NEXT: s_cbranch_scc0 .LBB0_5 -; CHECK-NEXT: s_branch .LBB0_8 -; CHECK-NEXT: .LBB0_4: ; %dynamic-memcpy-expansion-residual-cond.dynamic-memcpy-post-expansion_crit_edge -; CHECK-NEXT: v_lshlrev_b64 v[6:7], 6, v[2:3] -; CHECK-NEXT: s_mov_b64 s[4:5], 0 -; CHECK-NEXT: s_and_b64 s[4:5], s[4:5], exec -; CHECK-NEXT: s_cselect_b32 s4, 1, 0 -; CHECK-NEXT: s_cmp_lg_u32 s4, 1 -; CHECK-NEXT: s_cbranch_scc1 .LBB0_8 -; CHECK-NEXT: .LBB0_5: ; %dynamic-memcpy-expansion-residual-body.preheader +; CHECK-NEXT: s_cbranch_execnz .LBB0_5 +; CHECK-NEXT: ; %bb.3: ; %dynamic-memcpy-expansion-residual-body.preheader ; CHECK-NEXT: s_add_u32 s4, s16, 32 ; CHECK-NEXT: s_addc_u32 s5, s17, 0 ; CHECK-NEXT: v_mov_b32_e32 v3, s5 ; CHECK-NEXT: v_add_co_u32_e32 v2, vcc, s4, v4 ; CHECK-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v5, vcc ; CHECK-NEXT: s_mov_b64 s[4:5], 0 -; CHECK-NEXT: ; %bb.6: ; %dynamic-memcpy-expansion-residual-body +; CHECK-NEXT: ; %bb.4: ; %dynamic-memcpy-expansion-residual-body ; CHECK-NEXT: s_add_u32 s6, 32, s4 ; CHECK-NEXT: s_addc_u32 s7, 0, s5 -; CHECK-NEXT: v_mov_b32_e32 v6, s6 -; CHECK-NEXT: v_mov_b32_e32 v7, s7 -; CHECK-NEXT: flat_load_ubyte v10, v[6:7] -; CHECK-NEXT: v_mov_b32_e32 v7, s5 -; CHECK-NEXT: v_add_co_u32_e32 v6, vcc, s4, v2 -; CHECK-NEXT: v_addc_co_u32_e32 v7, vcc, v3, v7, vcc +; CHECK-NEXT: v_mov_b32_e32 v9, s7 +; CHECK-NEXT: v_mov_b32_e32 v8, s6 +; CHECK-NEXT: flat_load_ubyte v10, v[8:9] +; CHECK-NEXT: v_mov_b32_e32 v9, s5 +; CHECK-NEXT: v_add_co_u32_e32 v8, vcc, s4, v2 +; CHECK-NEXT: v_addc_co_u32_e32 v9, vcc, v3, v9, vcc ; CHECK-NEXT: s_add_u32 s4, s4, 1 ; CHECK-NEXT: s_addc_u32 s5, 0, s5 ; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; CHECK-NEXT: flat_store_byte v[6:7], v10 -; CHECK-NEXT: ; %bb.7: -; CHECK-NEXT: v_mov_b32_e32 v7, v5 -; CHECK-NEXT: v_mov_b32_e32 v6, v4 -; CHECK-NEXT: .LBB0_8: ; %dynamic-memcpy-post-expansion +; CHECK-NEXT: flat_store_byte v[8:9], v10 +; CHECK-NEXT: .LBB0_5: ; %dynamic-memcpy-post-expansion ; CHECK-NEXT: v_and_b32_e32 v2, 15, v0 ; CHECK-NEXT: v_and_b32_e32 v0, -16, v0 -; CHECK-NEXT: v_add_co_u32_e32 v4, vcc, v6, v0 +; CHECK-NEXT: v_add_co_u32_e32 v4, vcc, v4, v0 ; CHECK-NEXT: v_mov_b32_e32 v3, 0 -; CHECK-NEXT: v_addc_co_u32_e32 v5, vcc, v7, v1, vcc +; CHECK-NEXT: v_addc_co_u32_e32 v5, vcc, v5, v1, vcc ; CHECK-NEXT: v_cmp_ne_u64_e64 s[4:5], 0, v[0:1] ; CHECK-NEXT: v_cmp_ne_u64_e64 s[6:7], 0, v[2:3] -; CHECK-NEXT: v_mov_b32_e32 v6, s17 +; CHECK-NEXT: v_mov_b32_e32 v8, s17 ; CHECK-NEXT: v_add_co_u32_e32 v4, vcc, s16, v4 -; CHECK-NEXT: v_addc_co_u32_e32 v5, vcc, v6, v5, vcc -; CHECK-NEXT: s_branch .LBB0_11 -; CHECK-NEXT: .LBB0_9: ; %Flow14 -; CHECK-NEXT: ; in Loop: Header=BB0_11 Depth=1 +; CHECK-NEXT: v_addc_co_u32_e32 v5, vcc, v8, v5, vcc +; CHECK-NEXT: s_branch .LBB0_8 +; CHECK-NEXT: .LBB0_6: ; %Flow14 +; CHECK-NEXT: ; in Loop: Header=BB0_8 Depth=1 ; CHECK-NEXT: s_or_b64 exec, exec, s[10:11] ; CHECK-NEXT: s_mov_b64 s[8:9], 0 -; CHECK-NEXT: .LBB0_10: ; %Flow16 -; CHECK-NEXT: ; in Loop: Header=BB0_11 Depth=1 +; CHECK-NEXT: .LBB0_7: ; %Flow16 +; CHECK-NEXT: ; in Loop: Header=BB0_8 Depth=1 ; CHECK-NEXT: s_and_b64 s[8:9], s[8:9], exec ; CHECK-NEXT: s_cselect_b32 s8, 1, 0 ; CHECK-NEXT: s_cmp_lg_u32 s8, 1 -; CHECK-NEXT: s_cbranch_scc0 .LBB0_18 -; CHECK-NEXT: .LBB0_11: ; %while.cond +; CHECK-NEXT: s_cbranch_scc0 .LBB0_15 +; CHECK-NEXT: .LBB0_8: ; %while.cond ; CHECK-NEXT: ; =>This Loop Header: Depth=1 -; CHECK-NEXT: ; Child Loop BB0_13 Depth 2 -; CHECK-NEXT: ; Child Loop BB0_17 Depth 2 +; CHECK-NEXT: ; Child Loop BB0_10 Depth 2 +; CHECK-NEXT: ; Child Loop BB0_14 Depth 2 ; CHECK-NEXT: s_and_saveexec_b64 s[8:9], s[4:5] -; CHECK-NEXT: s_cbranch_execz .LBB0_14 -; CHECK-NEXT: ; %bb.12: ; %dynamic-memcpy-expansion-main-body2.preheader -; CHECK-NEXT: ; in Loop: Header=BB0_11 Depth=1 +; CHECK-NEXT: s_cbranch_execz .LBB0_11 +; CHECK-NEXT: ; %bb.9: ; %dynamic-memcpy-expansion-main-body2.preheader +; CHECK-NEXT: ; in Loop: Header=BB0_8 Depth=1 ; CHECK-NEXT: s_mov_b64 s[10:11], 0 ; CHECK-NEXT: s_mov_b64 s[12:13], 0 -; CHECK-NEXT: .LBB0_13: ; %dynamic-memcpy-expansion-main-body2 -; CHECK-NEXT: ; Parent Loop BB0_11 Depth=1 +; CHECK-NEXT: .LBB0_10: ; %dynamic-memcpy-expansion-main-body2 +; CHECK-NEXT: ; Parent Loop BB0_8 Depth=1 ; CHECK-NEXT: ; => This Inner Loop Header: Depth=2 -; CHECK-NEXT: v_mov_b32_e32 v6, s10 -; CHECK-NEXT: v_mov_b32_e32 v7, s11 -; CHECK-NEXT: flat_load_dwordx4 v[10:13], v[6:7] -; CHECK-NEXT: v_add_co_u32_e32 v6, vcc, s10, v8 +; CHECK-NEXT: v_mov_b32_e32 v8, s10 +; CHECK-NEXT: v_mov_b32_e32 v9, s11 +; CHECK-NEXT: flat_load_dwordx4 v[8:11], v[8:9] +; CHECK-NEXT: v_mov_b32_e32 v13, s11 +; CHECK-NEXT: v_add_co_u32_e32 v12, vcc, s10, v6 ; CHECK-NEXT: s_add_u32 s10, s10, 16 -; CHECK-NEXT: v_addc_co_u32_e32 v7, vcc, v9, v7, vcc +; CHECK-NEXT: v_addc_co_u32_e32 v13, vcc, v7, v13, vcc ; CHECK-NEXT: s_addc_u32 s11, s11, 0 ; CHECK-NEXT: v_cmp_ge_u64_e32 vcc, s[10:11], v[0:1] ; CHECK-NEXT: s_or_b64 s[12:13], vcc, s[12:13] ; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; CHECK-NEXT: flat_store_dwordx4 v[6:7], v[10:13] +; CHECK-NEXT: flat_store_dwordx4 v[12:13], v[8:11] ; CHECK-NEXT: s_andn2_b64 exec, exec, s[12:13] -; CHECK-NEXT: s_cbranch_execnz .LBB0_13 -; CHECK-NEXT: .LBB0_14: ; %Flow15 -; CHECK-NEXT: ; in Loop: Header=BB0_11 Depth=1 +; CHECK-NEXT: s_cbranch_execnz .LBB0_10 +; CHECK-NEXT: .LBB0_11: ; %Flow15 +; CHECK-NEXT: ; in Loop: Header=BB0_8 Depth=1 ; CHECK-NEXT: s_or_b64 exec, exec, s[8:9] ; CHECK-NEXT: s_mov_b64 s[8:9], -1 -; CHECK-NEXT: s_cbranch_execz .LBB0_10 -; CHECK-NEXT: ; %bb.15: ; %dynamic-memcpy-expansion-residual-cond5 -; CHECK-NEXT: ; in Loop: Header=BB0_11 Depth=1 +; CHECK-NEXT: s_cbranch_execz .LBB0_7 +; CHECK-NEXT: ; %bb.12: ; %dynamic-memcpy-expansion-residual-cond5 +; CHECK-NEXT: ; in Loop: Header=BB0_8 Depth=1 ; CHECK-NEXT: s_and_saveexec_b64 s[10:11], s[6:7] -; CHECK-NEXT: s_cbranch_execz .LBB0_9 -; CHECK-NEXT: ; %bb.16: ; %dynamic-memcpy-expansion-residual-body4.preheader -; CHECK-NEXT: ; in Loop: Header=BB0_11 Depth=1 +; CHECK-NEXT: s_cbranch_execz .LBB0_6 +; CHECK-NEXT: ; %bb.13: ; %dynamic-memcpy-expansion-residual-body4.preheader +; CHECK-NEXT: ; in Loop: Header=BB0_8 Depth=1 ; CHECK-NEXT: s_mov_b64 s[12:13], 0 ; CHECK-NEXT: s_mov_b64 s[14:15], 0 -; CHECK-NEXT: .LBB0_17: ; %dynamic-memcpy-expansion-residual-body4 -; CHECK-NEXT: ; Parent Loop BB0_11 Depth=1 +; CHECK-NEXT: .LBB0_14: ; %dynamic-memcpy-expansion-residual-body4 +; CHECK-NEXT: ; Parent Loop BB0_8 Depth=1 ; CHECK-NEXT: ; => This Inner Loop Header: Depth=2 ; CHECK-NEXT: v_mov_b32_e32 v10, s13 -; CHECK-NEXT: v_add_co_u32_e32 v6, vcc, s12, v0 -; CHECK-NEXT: v_addc_co_u32_e32 v7, vcc, v1, v10, vcc -; CHECK-NEXT: flat_load_ubyte v11, v[6:7] -; CHECK-NEXT: v_add_co_u32_e32 v6, vcc, s12, v4 +; CHECK-NEXT: v_add_co_u32_e32 v8, vcc, s12, v0 +; CHECK-NEXT: v_addc_co_u32_e32 v9, vcc, v1, v10, vcc +; CHECK-NEXT: flat_load_ubyte v11, v[8:9] +; CHECK-NEXT: v_add_co_u32_e32 v8, vcc, s12, v4 ; CHECK-NEXT: s_add_u32 s12, s12, 1 ; CHECK-NEXT: s_addc_u32 s13, s13, 0 ; CHECK-NEXT: v_cmp_ge_u64_e64 s[8:9], s[12:13], v[2:3] -; CHECK-NEXT: v_addc_co_u32_e32 v7, vcc, v5, v10, vcc +; CHECK-NEXT: v_addc_co_u32_e32 v9, vcc, v5, v10, vcc ; CHECK-NEXT: s_or_b64 s[14:15], s[8:9], s[14:15] ; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; CHECK-NEXT: flat_store_byte v[6:7], v11 +; CHECK-NEXT: flat_store_byte v[8:9], v11 ; CHECK-NEXT: s_andn2_b64 exec, exec, s[14:15] -; CHECK-NEXT: s_cbranch_execnz .LBB0_17 -; CHECK-NEXT: s_branch .LBB0_9 -; CHECK-NEXT: .LBB0_18: ; %DummyReturnBlock +; CHECK-NEXT: s_cbranch_execnz .LBB0_14 +; CHECK-NEXT: s_branch .LBB0_6 +; CHECK-NEXT: .LBB0_15: ; %DummyReturnBlock ; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; CHECK-NEXT: s_setpc_b64 s[30:31] entry: diff --git a/llvm/test/CodeGen/NVPTX/gvn-scalar-pre-reg-pressure.ll b/llvm/test/CodeGen/NVPTX/gvn-scalar-pre-reg-pressure.ll index 5b7f893889744..a265f737deb34 100644 --- a/llvm/test/CodeGen/NVPTX/gvn-scalar-pre-reg-pressure.ll +++ b/llvm/test/CodeGen/NVPTX/gvn-scalar-pre-reg-pressure.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_100 -O3 | FileCheck %s --check-prefix=PIPELINE -; RUN: opt < %s -passes='gvn<no-scalar-pre>' -S | llc -mtriple=nvptx64 -mcpu=sm_100 -O0 | FileCheck %s --check-prefix=NO-SCALAR-PRE -; RUN: opt < %s -passes='gvn<scalar-pre>' -S | llc -mtriple=nvptx64 -mcpu=sm_100 -O0 | FileCheck %s --check-prefix=SCALAR-PRE +; RUN: opt < %s -passes='gvn<no-scalar-pre>' --enable-simple-gvn-hoist=false -S | llc -mtriple=nvptx64 -mcpu=sm_100 -O0 | FileCheck %s --check-prefix=NO-SCALAR-PRE +; RUN: opt < %s -passes='gvn<scalar-pre>' --enable-simple-gvn-hoist=false -S | llc -mtriple=nvptx64 -mcpu=sm_100 -O0 | FileCheck %s --check-prefix=SCALAR-PRE ; Scalar PRE inserts a critical-edge computation and a PHI for the common add. ; That shape needs more NVPTX virtual registers than keeping the duplicated adds. diff --git a/llvm/test/Transforms/GVN/2012-05-22-PreCrash.ll b/llvm/test/Transforms/GVN/2012-05-22-PreCrash.ll index 205dff7968018..a8eae380b9033 100644 --- a/llvm/test/Transforms/GVN/2012-05-22-PreCrash.ll +++ b/llvm/test/Transforms/GVN/2012-05-22-PreCrash.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 -; RUN: opt < %s -passes=gvn -S | FileCheck %s +; RUN: opt < %s -passes=gvn --enable-simple-gvn-hoist=false -S | FileCheck %s ; PR12858 diff --git a/llvm/test/Transforms/GVN/PRE/load-pre-across-backedge.ll b/llvm/test/Transforms/GVN/PRE/load-pre-across-backedge.ll index b6772725d2a88..4eb06478a55c1 100644 --- a/llvm/test/Transforms/GVN/PRE/load-pre-across-backedge.ll +++ b/llvm/test/Transforms/GVN/PRE/load-pre-across-backedge.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes=gvn -S < %s | FileCheck %s --check-prefixes=CHECK,MDEP -; RUN: opt -passes='gvn<memoryssa>' -S < %s | FileCheck %s --check-prefixes=CHECK,MSSA +; RUN: opt -passes=gvn --enable-simple-gvn-hoist=false -S < %s | FileCheck %s --check-prefixes=CHECK,MDEP +; RUN: opt -passes='gvn<memoryssa>' --enable-simple-gvn-hoist=false -S < %s | FileCheck %s --check-prefixes=CHECK,MSSA ; Check that PRE-LOAD across backedge does not ; result in invalid dominator tree. diff --git a/llvm/test/Transforms/GVN/PRE/local-pre.ll b/llvm/test/Transforms/GVN/PRE/local-pre.ll index c67a5f1549f80..5d0914b52aa9c 100644 --- a/llvm/test/Transforms/GVN/PRE/local-pre.ll +++ b/llvm/test/Transforms/GVN/PRE/local-pre.ll @@ -1,5 +1,5 @@ -; RUN: opt < %s -passes=gvn -enable-scalar-pre -S | FileCheck %s -; RUN: opt < %s -passes="gvn<scalar-pre>" -enable-scalar-pre=false -S | FileCheck %s +; RUN: opt < %s -passes=gvn -enable-scalar-pre --enable-simple-gvn-hoist=false -S | FileCheck %s +; RUN: opt < %s -passes="gvn<scalar-pre>" -enable-scalar-pre=false --enable-simple-gvn-hoist=false -S | FileCheck %s declare void @may_exit() nounwind diff --git a/llvm/test/Transforms/GVN/PRE/no-scalar-pre.ll b/llvm/test/Transforms/GVN/PRE/no-scalar-pre.ll index c817ac5d1a131..b682e594ad222 100644 --- a/llvm/test/Transforms/GVN/PRE/no-scalar-pre.ll +++ b/llvm/test/Transforms/GVN/PRE/no-scalar-pre.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 -; RUN: opt -enable-scalar-pre=false -passes=gvn -S < %s | FileCheck %s -; RUN: opt -enable-scalar-pre=true -passes=gvn -S < %s | FileCheck %s --check-prefixes=CHECK-ENABLED -; RUN: opt -passes='gvn<no-scalar-pre>' -S < %s | FileCheck %s -; RUN: opt -passes='gvn<scalar-pre>' -S < %s | FileCheck %s --check-prefixes=CHECK-ENABLED +; RUN: opt -enable-scalar-pre=false -passes=gvn --enable-simple-gvn-hoist=false -S < %s | FileCheck %s +; RUN: opt -enable-scalar-pre=true -passes=gvn --enable-simple-gvn-hoist=false -S < %s | FileCheck %s --check-prefixes=CHECK-ENABLED +; RUN: opt -passes='gvn<no-scalar-pre>' --enable-simple-gvn-hoist=false -S < %s | FileCheck %s +; RUN: opt -passes='gvn<scalar-pre>' --enable-simple-gvn-hoist=false -S < %s | FileCheck %s --check-prefixes=CHECK-ENABLED define void @test_scalar_pre_option(ptr %arr, i8 %cond) { ; CHECK-LABEL: define void @test_scalar_pre_option( diff --git a/llvm/test/Transforms/GVN/PRE/phi-translate.ll b/llvm/test/Transforms/GVN/PRE/phi-translate.ll index 1915244e9490e..5931735ac5cba 100644 --- a/llvm/test/Transforms/GVN/PRE/phi-translate.ll +++ b/llvm/test/Transforms/GVN/PRE/phi-translate.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 -; RUN: opt -passes=gvn -S < %s | FileCheck %s --check-prefixes=CHECK,MDEP -; RUN: opt -passes='gvn<memoryssa>' -S < %s | FileCheck %s --check-prefixes=CHECK,MSSA +; RUN: opt -passes=gvn --enable-simple-gvn-hoist=false -S < %s | FileCheck %s --check-prefixes=CHECK,MDEP +; RUN: opt -passes='gvn<memoryssa>' --enable-simple-gvn-hoist=false -S < %s | FileCheck %s --check-prefixes=CHECK,MSSA target datalayout = "e-p:64:64:64" diff --git a/llvm/test/Transforms/GVN/PRE/pre-basic-add.ll b/llvm/test/Transforms/GVN/PRE/pre-basic-add.ll index 92306015378cb..239e1bc5c0d3e 100644 --- a/llvm/test/Transforms/GVN/PRE/pre-basic-add.ll +++ b/llvm/test/Transforms/GVN/PRE/pre-basic-add.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 -; RUN: opt < %s -passes=gvn -enable-scalar-pre -S | FileCheck %s --check-prefixes=CHECK,MDEP -; RUN: opt < %s -passes='gvn<memoryssa>' -enable-scalar-pre -S | FileCheck %s --check-prefixes=CHECK,MSSA -; RUN: opt < %s -passes="gvn<scalar-pre>" -enable-scalar-pre=false -S | FileCheck %s +; RUN: opt < %s -passes=gvn -enable-scalar-pre --enable-simple-gvn-hoist=false -S | FileCheck %s --check-prefixes=CHECK,MDEP +; RUN: opt < %s -passes='gvn<memoryssa>' -enable-scalar-pre --enable-simple-gvn-hoist=false -S | FileCheck %s --check-prefixes=CHECK,MSSA +; RUN: opt < %s -passes="gvn<scalar-pre>" -enable-scalar-pre=false --enable-simple-gvn-hoist=false -S | FileCheck %s @H = common global i32 0 ; <ptr> [#uses=2] @G = common global i32 0 ; <ptr> [#uses=1] diff --git a/llvm/test/Transforms/GVN/PRE/pre-load-through-select.ll b/llvm/test/Transforms/GVN/PRE/pre-load-through-select.ll index 2a2019e80323e..ccc760ddc085f 100644 --- a/llvm/test/Transforms/GVN/PRE/pre-load-through-select.ll +++ b/llvm/test/Transforms/GVN/PRE/pre-load-through-select.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='require<domtree>,loop(loop-simplifycfg),gvn' -S %s | FileCheck %s --check-prefixes=CHECK,MDEP -; RUN: opt -passes='require<domtree>,loop(loop-simplifycfg),gvn<memoryssa>' -S %s | FileCheck %s --check-prefixes=CHECK,MSSA +; RUN: opt -passes='require<domtree>,loop(loop-simplifycfg),gvn' --enable-simple-gvn-hoist=false -S %s | FileCheck %s --check-prefixes=CHECK,MDEP +; RUN: opt -passes='require<domtree>,loop(loop-simplifycfg),gvn<memoryssa>' --enable-simple-gvn-hoist=false -S %s | FileCheck %s --check-prefixes=CHECK,MSSA define i32 @test_pointer_phi_select_simp_1(ptr %a, ptr %b, i1 %cond) { ; MDEP-LABEL: @test_pointer_phi_select_simp_1( diff --git a/llvm/test/Transforms/GVN/PRE/pre-no-cost-phi.ll b/llvm/test/Transforms/GVN/PRE/pre-no-cost-phi.ll index 22c628bb35464..329c6aee0e5e9 100644 --- a/llvm/test/Transforms/GVN/PRE/pre-no-cost-phi.ll +++ b/llvm/test/Transforms/GVN/PRE/pre-no-cost-phi.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 -; RUN: opt < %s -passes=gvn -S | FileCheck %s --check-prefixes=CHECK,MDEP -; RUN: opt < %s -passes='gvn<memoryssa>' -S | FileCheck %s --check-prefixes=CHECK,MSSA +; RUN: opt < %s -passes=gvn --enable-simple-gvn-hoist=false -S | FileCheck %s --check-prefixes=CHECK,MDEP +; RUN: opt < %s -passes='gvn<memoryssa>' --enable-simple-gvn-hoist=false -S | FileCheck %s --check-prefixes=CHECK,MSSA ; This testcase tests insertion of no-cost phis. That is, ; when the value is already available in every predecessor, ; and we just need to insert a phi node to merge the available values. diff --git a/llvm/test/Transforms/GVN/PRE/pre-poison-add.ll b/llvm/test/Transforms/GVN/PRE/pre-poison-add.ll index a4ee356628f11..c328826f0bbb1 100644 --- a/llvm/test/Transforms/GVN/PRE/pre-poison-add.ll +++ b/llvm/test/Transforms/GVN/PRE/pre-poison-add.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 -; RUN: opt < %s -passes=gvn -enable-scalar-pre -S | FileCheck %s --check-prefixes=CHECK,MDEP -; RUN: opt < %s -passes='gvn<memoryssa>' -enable-scalar-pre -S | FileCheck %s --check-prefixes=CHECK,MSSA +; RUN: opt < %s -passes=gvn -enable-scalar-pre --enable-simple-gvn-hoist=false -S | FileCheck %s --check-prefixes=CHECK,MDEP +; RUN: opt < %s -passes='gvn<memoryssa>' -enable-scalar-pre --enable-simple-gvn-hoist=false -S | FileCheck %s --check-prefixes=CHECK,MSSA @H = common global i32 0 @G = common global i32 0 diff --git a/llvm/test/Transforms/GVN/freeze.ll b/llvm/test/Transforms/GVN/freeze.ll index de079fddb0dac..577cd017db22a 100644 --- a/llvm/test/Transforms/GVN/freeze.ll +++ b/llvm/test/Transforms/GVN/freeze.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt < %s -passes=gvn -S | FileCheck %s +; RUN: opt < %s -passes=gvn --enable-simple-gvn-hoist=false -S | FileCheck %s define i1 @f(i1 %a) { ; CHECK-LABEL: @f( diff --git a/llvm/test/Transforms/GVN/gc_relocate.ll b/llvm/test/Transforms/GVN/gc_relocate.ll index 6bc71f5da53fb..131b157c26d73 100644 --- a/llvm/test/Transforms/GVN/gc_relocate.ll +++ b/llvm/test/Transforms/GVN/gc_relocate.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes=gvn -S < %s | FileCheck %s +; RUN: opt -passes=gvn --enable-simple-gvn-hoist=false -S < %s | FileCheck %s declare void @func() declare i32 @"personality_function"() diff --git a/llvm/test/Transforms/GVN/simple-gvn-hoist-limits.ll b/llvm/test/Transforms/GVN/simple-gvn-hoist-limits.ll index 0c0789e6e4144..25dbe8b80d5a6 100644 --- a/llvm/test/Transforms/GVN/simple-gvn-hoist-limits.ll +++ b/llvm/test/Transforms/GVN/simple-gvn-hoist-limits.ll @@ -49,12 +49,12 @@ define i32 @f(i1 %c, i32 %a, i32 %b, i32 %d) { ; ; MAX-DEPTH4-LABEL: @f( ; MAX-DEPTH4-NEXT: entry: +; MAX-DEPTH4-NEXT: [[D0:%.*]] = add i32 [[D:%.*]], 1 ; MAX-DEPTH4-NEXT: br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] ; MAX-DEPTH4: if.then: ; MAX-DEPTH4-NEXT: [[R0:%.*]] = add i32 [[B:%.*]], 1 ; MAX-DEPTH4-NEXT: [[AND0:%.*]] = and i32 [[A:%.*]], 1 ; MAX-DEPTH4-NEXT: [[TOBOOL_AND0:%.*]] = icmp eq i32 [[AND0]], 0 -; MAX-DEPTH4-NEXT: [[D0:%.*]] = add i32 [[D:%.*]], 1 ; MAX-DEPTH4-NEXT: [[S0:%.*]] = add i32 [[D0]], [[B]] ; MAX-DEPTH4-NEXT: br i1 [[TOBOOL_AND0]], label [[IF_THEN1:%.*]], label [[IF_ELSE1:%.*]] ; MAX-DEPTH4: if.then1: @@ -67,8 +67,7 @@ define i32 @f(i1 %c, i32 %a, i32 %b, i32 %d) { ; MAX-DEPTH4-NEXT: [[R1:%.*]] = add i32 [[B]], 2 ; MAX-DEPTH4-NEXT: [[AND1:%.*]] = and i32 [[A]], 1 ; MAX-DEPTH4-NEXT: [[TOBOOL_AND1:%.*]] = icmp eq i32 [[AND1]], 0 -; MAX-DEPTH4-NEXT: [[D1:%.*]] = add i32 [[D]], 1 -; MAX-DEPTH4-NEXT: [[S1:%.*]] = add i32 [[D1]], [[B]] +; MAX-DEPTH4-NEXT: [[S1:%.*]] = add i32 [[D0]], [[B]] ; MAX-DEPTH4-NEXT: br i1 [[TOBOOL_AND1]], label [[IF_THEN2:%.*]], label [[IF_ELSE2:%.*]] ; MAX-DEPTH4: if.then2: ; MAX-DEPTH4-NEXT: [[OR2:%.*]] = or i32 [[R1]], 4 @@ -84,13 +83,13 @@ define i32 @f(i1 %c, i32 %a, i32 %b, i32 %d) { ; ; MAX-DEPTH5-LABEL: @f( ; MAX-DEPTH5-NEXT: entry: +; MAX-DEPTH5-NEXT: [[D0:%.*]] = add i32 [[D:%.*]], 1 +; MAX-DEPTH5-NEXT: [[S0:%.*]] = add i32 [[D0]], [[B:%.*]] ; MAX-DEPTH5-NEXT: br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] ; MAX-DEPTH5: if.then: -; MAX-DEPTH5-NEXT: [[R0:%.*]] = add i32 [[B:%.*]], 1 +; MAX-DEPTH5-NEXT: [[R0:%.*]] = add i32 [[B]], 1 ; MAX-DEPTH5-NEXT: [[AND0:%.*]] = and i32 [[A:%.*]], 1 ; MAX-DEPTH5-NEXT: [[TOBOOL_AND0:%.*]] = icmp eq i32 [[AND0]], 0 -; MAX-DEPTH5-NEXT: [[D0:%.*]] = add i32 [[D:%.*]], 1 -; MAX-DEPTH5-NEXT: [[S0:%.*]] = add i32 [[D0]], [[B]] ; MAX-DEPTH5-NEXT: br i1 [[TOBOOL_AND0]], label [[IF_THEN1:%.*]], label [[IF_ELSE1:%.*]] ; MAX-DEPTH5: if.then1: ; MAX-DEPTH5-NEXT: [[OR0:%.*]] = or i32 [[R0]], 1 @@ -102,8 +101,6 @@ define i32 @f(i1 %c, i32 %a, i32 %b, i32 %d) { ; MAX-DEPTH5-NEXT: [[R1:%.*]] = add i32 [[B]], 2 ; MAX-DEPTH5-NEXT: [[AND1:%.*]] = and i32 [[A]], 1 ; MAX-DEPTH5-NEXT: [[TOBOOL_AND1:%.*]] = icmp eq i32 [[AND1]], 0 -; MAX-DEPTH5-NEXT: [[D1:%.*]] = add i32 [[D]], 1 -; MAX-DEPTH5-NEXT: [[S1:%.*]] = add i32 [[D1]], [[B]] ; MAX-DEPTH5-NEXT: br i1 [[TOBOOL_AND1]], label [[IF_THEN2:%.*]], label [[IF_ELSE2:%.*]] ; MAX-DEPTH5: if.then2: ; MAX-DEPTH5-NEXT: [[OR2:%.*]] = or i32 [[R1]], 4 @@ -113,7 +110,7 @@ define i32 @f(i1 %c, i32 %a, i32 %b, i32 %d) { ; MAX-DEPTH5-NEXT: br label [[EXIT]] ; MAX-DEPTH5: exit: ; MAX-DEPTH5-NEXT: [[OR:%.*]] = phi i32 [ [[OR0]], [[IF_THEN1]] ], [ [[OR1]], [[IF_ELSE1]] ], [ [[OR2]], [[IF_THEN2]] ], [ [[OR3]], [[IF_ELSE2]] ] -; MAX-DEPTH5-NEXT: [[S:%.*]] = phi i32 [ [[S0]], [[IF_THEN1]] ], [ [[S0]], [[IF_ELSE1]] ], [ [[S1]], [[IF_THEN2]] ], [ [[S1]], [[IF_ELSE2]] ] +; MAX-DEPTH5-NEXT: [[S:%.*]] = phi i32 [ [[S0]], [[IF_THEN1]] ], [ [[S0]], [[IF_ELSE1]] ], [ [[S0]], [[IF_THEN2]] ], [ [[S0]], [[IF_ELSE2]] ] ; MAX-DEPTH5-NEXT: [[R:%.*]] = add i32 [[OR]], [[S]] ; MAX-DEPTH5-NEXT: ret i32 [[R]] ; diff --git a/llvm/test/Transforms/GVN/simple-gvn-hoist-scalars.ll b/llvm/test/Transforms/GVN/simple-gvn-hoist-scalars.ll index e774f64d4a4e5..d35ca95cb87c5 100644 --- a/llvm/test/Transforms/GVN/simple-gvn-hoist-scalars.ll +++ b/llvm/test/Transforms/GVN/simple-gvn-hoist-scalars.ll @@ -7,19 +7,16 @@ target triple = "aarch64-unknown-linux" define dso_local i32 @everything_hoisted(i1 %cc, i32 %a, i32 %b, i32 %c) { ; CHECK-LABEL: @everything_hoisted( ; CHECK-NEXT: entry: -; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] -; CHECK: if.then: ; CHECK-NEXT: [[TMP0:%.*]] = call i32 @barrier(i32 [[A:%.*]]) ; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], [[B:%.*]] ; CHECK-NEXT: [[TMP2:%.*]] = sdiv i32 [[TMP1]], [[C:%.*]] +; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] +; CHECK: if.then: ; CHECK-NEXT: br label [[IF_END:%.*]] ; CHECK: if.else: -; CHECK-NEXT: [[TMP3:%.*]] = call i32 @barrier(i32 [[A]]) -; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], [[B]] -; CHECK-NEXT: [[TMP5:%.*]] = sdiv i32 [[TMP4]], [[C]] ; CHECK-NEXT: br label [[IF_END]] ; CHECK: if.end: -; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP2]], [[IF_THEN]] ], [ [[TMP5]], [[IF_ELSE]] ] +; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP2]], [[IF_THEN]] ], [ [[TMP2]], [[IF_ELSE]] ] ; CHECK-NEXT: ret i32 [[R]] ; entry: @@ -46,15 +43,14 @@ if.end: define dso_local i32 @spec_barrier_short_side(i1 %cc, i32 %a, i32 %b, i32 %c) { ; CHECK-LABEL: @spec_barrier_short_side( ; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] ; CHECK: if.then: -; CHECK-NEXT: [[TMP2:%.*]] = call i32 @barrier(i32 [[A:%.*]]) -; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[A]], [[B:%.*]] +; CHECK-NEXT: [[TMP2:%.*]] = call i32 @barrier(i32 [[A]]) ; CHECK-NEXT: [[TMP1:%.*]] = sdiv i32 [[C:%.*]], [[TMP0]] ; CHECK-NEXT: br label [[IF_END:%.*]] ; CHECK: if.else: -; CHECK-NEXT: [[TMP6:%.*]] = add nsw i32 [[A]], [[B]] -; CHECK-NEXT: [[TMP3:%.*]] = sdiv i32 [[C]], [[TMP6]] +; CHECK-NEXT: [[TMP3:%.*]] = sdiv i32 [[C]], [[TMP0]] ; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[TMP3]], 1 ; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP5]], 2 ; CHECK-NEXT: br label [[IF_END]] @@ -87,15 +83,14 @@ if.end: define dso_local i32 @spec_barrier_long_side(i1 %cc, i32 %a, i32 %b, i32 %c) { ; CHECK-LABEL: @spec_barrier_long_side( ; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] ; CHECK: if.then: -; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: [[TMP1:%.*]] = sdiv i32 [[C:%.*]], [[TMP0]] ; CHECK-NEXT: br label [[IF_END:%.*]] ; CHECK: if.else: ; CHECK-NEXT: [[TMP2:%.*]] = call i32 @barrier(i32 [[A]]) -; CHECK-NEXT: [[TMP6:%.*]] = add nsw i32 [[A]], [[B]] -; CHECK-NEXT: [[TMP3:%.*]] = sdiv i32 [[C]], [[TMP6]] +; CHECK-NEXT: [[TMP3:%.*]] = sdiv i32 [[C]], [[TMP0]] ; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[TMP3]], 1 ; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP5]], 2 ; CHECK-NEXT: br label [[IF_END]] @@ -127,15 +122,14 @@ if.end: define dso_local i32 @no_reorder_across_volatile(i1 %cc, i32 %a, i32 %b, i32 %c, ptr %p) { ; CHECK-LABEL: @no_reorder_across_volatile( ; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] ; CHECK: if.then: -; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: [[TMP1:%.*]] = sdiv i32 [[C:%.*]], [[TMP0]] ; CHECK-NEXT: br label [[IF_END:%.*]] ; CHECK: if.else: ; CHECK-NEXT: store volatile i32 0, ptr [[P:%.*]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = add nsw i32 [[A]], [[B]] -; CHECK-NEXT: [[TMP2:%.*]] = sdiv i32 [[C]], [[TMP3]] +; CHECK-NEXT: [[TMP2:%.*]] = sdiv i32 [[C]], [[TMP0]] ; CHECK-NEXT: br label [[IF_END]] ; CHECK: if.end: ; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP1]], [[IF_THEN]] ], [ [[TMP2]], [[IF_ELSE]] ] @@ -163,19 +157,16 @@ if.end: define dso_local i32 @no_barrier_call(i1 %cc, i32 %a, i32 %b, i32 %c) { ; CHECK-LABEL: @no_barrier_call( ; CHECK-NEXT: entry: -; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] -; CHECK: if.then: ; CHECK-NEXT: call void @will_return() ; CHECK-NEXT: [[TMP0:%.*]] = sdiv i32 [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: [[TMP1:%.*]] = add nsw i32 [[C:%.*]], [[TMP0]] +; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] +; CHECK: if.then: ; CHECK-NEXT: br label [[IF_END:%.*]] ; CHECK: if.else: -; CHECK-NEXT: call void @will_return() -; CHECK-NEXT: [[TMP2:%.*]] = sdiv i32 [[A]], [[B]] -; CHECK-NEXT: [[TMP3:%.*]] = add nsw i32 [[C]], [[TMP2]] ; CHECK-NEXT: br label [[IF_END]] ; CHECK: if.end: -; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP1]], [[IF_THEN]] ], [ [[TMP3]], [[IF_ELSE]] ] +; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP1]], [[IF_THEN]] ], [ [[TMP1]], [[IF_ELSE]] ] ; CHECK-NEXT: ret i32 [[R]] ; entry: @@ -243,19 +234,16 @@ if.end: define dso_local i32 @multiple_use(i1 %cc, i32 %a, i32 %b, i32 %c) { ; CHECK-LABEL: @multiple_use( ; CHECK-NEXT: entry: -; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] -; CHECK: if.then: ; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: [[TMP1:%.*]] = mul nsw i32 [[TMP0]], [[C:%.*]] ; CHECK-NEXT: [[TMP2:%.*]] = add nsw i32 [[TMP0]], [[TMP1]] +; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] +; CHECK: if.then: ; CHECK-NEXT: br label [[IF_END:%.*]] ; CHECK: if.else: -; CHECK-NEXT: [[TMP3:%.*]] = add nsw i32 [[A]], [[B]] -; CHECK-NEXT: [[TMP4:%.*]] = mul nsw i32 [[TMP3]], [[C]] -; CHECK-NEXT: [[TMP5:%.*]] = add nsw i32 [[TMP3]], [[TMP4]] ; CHECK-NEXT: br label [[IF_END]] ; CHECK: if.end: -; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP2]], [[IF_THEN]] ], [ [[TMP5]], [[IF_ELSE]] ] +; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP2]], [[IF_THEN]] ], [ [[TMP2]], [[IF_ELSE]] ] ; CHECK-NEXT: ret i32 [[R]] ; entry: @@ -282,19 +270,16 @@ if.end: define dso_local i32 @commutative_ops(i1 %cc, i32 %a, i32 %b, i32 %c) { ; CHECK-LABEL: @commutative_ops( ; CHECK-NEXT: entry: -; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] -; CHECK: if.then: ; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: [[TMP1:%.*]] = add nsw i32 [[TMP0]], [[C:%.*]] ; CHECK-NEXT: [[TMP2:%.*]] = sdiv i32 [[TMP0]], [[TMP1]] +; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] +; CHECK: if.then: ; CHECK-NEXT: br label [[IF_END:%.*]] ; CHECK: if.else: -; CHECK-NEXT: [[TMP3:%.*]] = add nsw i32 [[A]], [[B]] -; CHECK-NEXT: [[TMP4:%.*]] = add nsw i32 [[C]], [[TMP3]] -; CHECK-NEXT: [[TMP5:%.*]] = sdiv i32 [[TMP3]], [[TMP4]] ; CHECK-NEXT: br label [[IF_END]] ; CHECK: if.end: -; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP2]], [[IF_THEN]] ], [ [[TMP5]], [[IF_ELSE]] ] +; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[TMP2]], [[IF_THEN]] ], [ [[TMP2]], [[IF_ELSE]] ] ; CHECK-NEXT: ret i32 [[R]] ; entry: @@ -378,5 +363,57 @@ if.else: ret i32 %1 } +define dso_local i32 @no_hoist_nomerge(i1 %cc, i32 %x, ptr %p) { +; CHECK-LABEL: @no_hoist_nomerge( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[P:%.*]], align 4 +; CHECK-NEXT: [[W:%.*]] = add i32 [[V]], [[X:%.*]] +; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] +; CHECK: if.then: +; CHECK-NEXT: [[TMP0:%.*]] = call i32 @cannot_merge() +; CHECK-NEXT: ret i32 [[TMP0]] +; CHECK: if.else: +; CHECK-NEXT: [[TMP1:%.*]] = call i32 @cannot_merge() +; CHECK-NEXT: ret i32 [[TMP1]] +; +entry: + %v = load i32, ptr %p + %w = add i32 %v, %x + br i1 %cc, label %if.then, label %if.else + +if.then: + %0 = call i32 @cannot_merge() + ret i32 %0 + +if.else: + %1 = call i32 @cannot_merge() + ret i32 %1 +} + + +define i32 @no_hoist_bundles(i1 %cc, i32 %x, ptr %p) { +; CHECK-LABEL: @no_hoist_bundles( +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[CC:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] +; CHECK: if.then: +; CHECK-NEXT: [[U:%.*]] = call i32 @will_return(i1 true, i32 [[X:%.*]], ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] [ "foo"(i32 [[X]], ptr [[P]]) ] +; CHECK-NEXT: ret i32 [[U]] +; CHECK: if.else: +; CHECK-NEXT: [[V:%.*]] = call i32 @will_return(i1 false, i32 [[X]], ptr [[P]]) #[[ATTR0]] [ "bar"(i32 [[X]], ptr [[P]]) ] +; CHECK-NEXT: ret i32 [[V]] +; +entry: + br i1 %cc, label %if.then, label %if.else + +if.then: + %u = call i32 @will_return(i1 %cc, i32 %x, ptr %p) memory(none) [ "foo"(i32 %x, ptr %p) ] + ret i32 %u + +if.else: + %v = call i32 @will_return(i1 %cc, i32 %x, ptr %p) memory(none) [ "bar"(i32 %x, ptr %p) ] + ret i32 %v +} + declare i32 @barrier(i32) memory(none) -declare void @will_return() nounwind willreturn +declare void @will_return(i1, i32, ptr) memory(none) nounwind willreturn +declare void @cannot_merge() memory(none) nounwind willreturn nomerge _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
