https://github.com/mtrofin updated https://github.com/llvm/llvm-project/pull/164271
>From 0ecbc8eb923c12bc31708d5d5664a0dedf64ffec Mon Sep 17 00:00:00 2001 From: Mircea Trofin <[email protected]> Date: Mon, 20 Oct 2025 08:21:26 -0700 Subject: [PATCH] [SLU][profcheck] create likely branch weights for guard->branch --- .../Transforms/Scalar/SimpleLoopUnswitch.cpp | 12 +- .../Transforms/SimpleLoopUnswitch/guards.ll | 181 +++++++++++++----- 2 files changed, 139 insertions(+), 54 deletions(-) diff --git a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp index dd36a63c1564b..88a19d3683991 100644 --- a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp +++ b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp @@ -40,6 +40,7 @@ #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/MDBuilder.h" #include "llvm/IR/Module.h" #include "llvm/IR/PatternMatch.h" #include "llvm/IR/ProfDataUtils.h" @@ -2829,9 +2830,14 @@ static BranchInst *turnGuardIntoBranch(IntrinsicInst *GI, Loop &L, MSSAU->getMemorySSA()->verifyMemorySSA(); DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager); - Instruction *DeoptBlockTerm = - SplitBlockAndInsertIfThen(GI->getArgOperand(0), GI, true, - GI->getMetadata(LLVMContext::MD_prof), &DTU, &LI); + // llvm.experimental.guard doesn't have branch weights. We can assume, + // however, that the deopt path is unlikely. + Instruction *DeoptBlockTerm = SplitBlockAndInsertIfThen( + GI->getArgOperand(0), GI, true, + !ProfcheckDisableMetadataFixes && EstimateProfile + ? MDBuilder(GI->getContext()).createUnlikelyBranchWeights() + : nullptr, + &DTU, &LI); BranchInst *CheckBI = cast<BranchInst>(CheckBB->getTerminator()); // SplitBlockAndInsertIfThen inserts control flow that branches to // DeoptBlockTerm if the condition is true. We want the opposite. diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll b/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll index 533b1f691f5ad..e83047e397d3d 100644 --- a/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll +++ b/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll @@ -1,26 +1,34 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: -p --version 5 +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: -p --check-globals all --version 5 ; RUN: opt -passes='loop(simple-loop-unswitch<nontrivial>),verify<loops>' -simple-loop-unswitch-guards -S < %s | FileCheck %s ; RUN: opt -passes='simple-loop-unswitch<nontrivial>' -simple-loop-unswitch-guards -S < %s | FileCheck %s ; RUN: opt -passes='loop-mssa(simple-loop-unswitch<nontrivial>),verify<loops>' -simple-loop-unswitch-guards -verify-memoryssa -verify-loop-info -S < %s | FileCheck %s declare void @llvm.experimental.guard(i1, ...) -define void @test_simple_case(i1 %cond, i32 %N) { -; CHECK-LABEL: @test_simple_case( +define void @test_simple_case(i1 %cond, i32 %N) !prof !0 { +; CHECK-LABEL: define void @test_simple_case(i1 %cond, i32 %N) !prof !0 { ; CHECK-NEXT: entry: -; CHECK-NEXT: br i1 [[COND:%.*]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]] +; CHECK-NEXT: br i1 %cond, label %entry.split.us, label %entry.split, !prof !1 ; CHECK: entry.split.us: -; CHECK-NEXT: br label [[LOOP_US:%.*]] +; CHECK-NEXT: br label %loop.us ; CHECK: loop.us: -; CHECK-NEXT: [[IV_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US]] ], [ [[IV_NEXT_US:%.*]], [[GUARDED_US:%.*]] ] -; CHECK-NEXT: br label [[GUARDED_US]] +; CHECK-NEXT: %iv.us = phi i32 [ 0, %entry.split.us ], [ %iv.next.us, %guarded.us ] +; CHECK-NEXT: br label %guarded.us ; CHECK: guarded.us: -; CHECK-NEXT: [[IV_NEXT_US]] = add i32 [[IV_US]], 1 -; CHECK-NEXT: [[LOOP_COND_US:%.*]] = icmp slt i32 [[IV_NEXT_US]], [[N:%.*]] -; CHECK-NEXT: br i1 [[LOOP_COND_US]], label [[LOOP_US]], label [[EXIT_SPLIT_US:%.*]] +; CHECK-NEXT: %iv.next.us = add i32 %iv.us, 1 +; CHECK-NEXT: %loop.cond.us = icmp slt i32 %iv.next.us, %N +; CHECK-NEXT: br i1 %loop.cond.us, label %loop.us, label %exit.split.us +; CHECK: exit.split.us: +; CHECK-NEXT: br label %exit +; CHECK: entry.split: +; CHECK-NEXT: br label %loop +; CHECK: loop: +; CHECK-NEXT: br label %deopt ; CHECK: deopt: ; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ] ; CHECK-NEXT: unreachable +; CHECK: exit: +; CHECK-NEXT: ret void ; entry: @@ -38,25 +46,39 @@ exit: } define void @test_two_guards(i1 %cond1, i1 %cond2, i32 %N) { -; CHECK-LABEL: @test_two_guards( +; CHECK-LABEL: define void @test_two_guards(i1 %cond1, i1 %cond2, i32 %N) { ; CHECK-NEXT: entry: -; CHECK-NEXT: br i1 [[COND1:%.*]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]] +; CHECK-NEXT: br i1 %cond1, label %entry.split.us, label %entry.split, !prof !1 ; CHECK: entry.split.us: -; CHECK-NEXT: br i1 [[COND2:%.*]], label [[ENTRY_SPLIT_US_SPLIT_US:%.*]], label [[ENTRY_SPLIT_US_SPLIT:%.*]] +; CHECK-NEXT: br i1 %cond2, label %entry.split.us.split.us, label %entry.split.us.split, !prof !1 ; CHECK: entry.split.us.split.us: -; CHECK-NEXT: br label [[LOOP_US_US:%.*]] +; CHECK-NEXT: br label %loop.us.us ; CHECK: loop.us.us: -; CHECK-NEXT: [[IV_US_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US_SPLIT_US]] ], [ [[IV_NEXT_US_US:%.*]], [[GUARDED_US2:%.*]] ] -; CHECK-NEXT: br label [[GUARDED_US_US:%.*]] +; CHECK-NEXT: %iv.us.us = phi i32 [ 0, %entry.split.us.split.us ], [ %iv.next.us.us, %guarded.us2 ] +; CHECK-NEXT: br label %guarded.us.us ; CHECK: guarded.us.us: -; CHECK-NEXT: br label [[GUARDED_US2]] +; CHECK-NEXT: br label %guarded.us2 ; CHECK: guarded.us2: -; CHECK-NEXT: [[IV_NEXT_US_US]] = add i32 [[IV_US_US]], 1 -; CHECK-NEXT: [[LOOP_COND_US_US:%.*]] = icmp slt i32 [[IV_NEXT_US_US]], [[N:%.*]] -; CHECK-NEXT: br i1 [[LOOP_COND_US_US]], label [[LOOP_US_US]], label [[EXIT_SPLIT_US_SPLIT_US:%.*]] +; CHECK-NEXT: %iv.next.us.us = add i32 %iv.us.us, 1 +; CHECK-NEXT: %loop.cond.us.us = icmp slt i32 %iv.next.us.us, %N +; CHECK-NEXT: br i1 %loop.cond.us.us, label %loop.us.us, label %exit.split.us.split.us +; CHECK: exit.split.us.split.us: +; CHECK-NEXT: br label %exit.split.us +; CHECK: entry.split.us.split: +; CHECK-NEXT: br label %loop.us +; CHECK: loop.us: +; CHECK-NEXT: br label %guarded.us +; CHECK: guarded.us: +; CHECK-NEXT: br label %deopt1 ; CHECK: deopt1: ; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ] ; CHECK-NEXT: unreachable +; CHECK: exit.split.us: +; CHECK-NEXT: br label %exit +; CHECK: entry.split: +; CHECK-NEXT: br label %loop +; CHECK: loop: +; CHECK-NEXT: br label %deopt ; CHECK: deopt: ; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ] ; CHECK-NEXT: unreachable @@ -80,35 +102,45 @@ exit: } define void @test_conditional_guards(i1 %cond, i32 %N) { -; CHECK-LABEL: @test_conditional_guards( +; CHECK-LABEL: define void @test_conditional_guards(i1 %cond, i32 %N) { ; CHECK-NEXT: entry: -; CHECK-NEXT: [[FROZEN:%.+]] = freeze i1 [[COND:%.*]] -; CHECK-NEXT: br i1 [[FROZEN]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]] +; CHECK-NEXT: %cond.fr = freeze i1 %cond +; CHECK-NEXT: br i1 %cond.fr, label %entry.split.us, label %entry.split, !prof !1 ; CHECK: entry.split.us: -; CHECK-NEXT: br label [[LOOP_US:%.*]] +; CHECK-NEXT: br label %loop.us ; CHECK: loop.us: -; CHECK-NEXT: [[IV_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US]] ], [ [[IV_NEXT_US:%.*]], [[BACKEDGE_US:%.*]] ] -; CHECK-NEXT: [[CONDITION_US:%.*]] = icmp eq i32 [[IV_US]], 123 -; CHECK-NEXT: br i1 [[CONDITION_US]], label [[GUARD_US:%.*]], label [[BACKEDGE_US]] +; CHECK-NEXT: %iv.us = phi i32 [ 0, %entry.split.us ], [ %iv.next.us, %backedge.us ] +; CHECK-NEXT: %condition.us = icmp eq i32 %iv.us, 123 +; CHECK-NEXT: br i1 %condition.us, label %guard.us, label %backedge.us ; CHECK: guard.us: -; CHECK-NEXT: br label [[GUARDED_US:%.*]] +; CHECK-NEXT: br label %guarded.us ; CHECK: backedge.us: -; CHECK-NEXT: [[IV_NEXT_US]] = add i32 [[IV_US]], 1 -; CHECK-NEXT: [[LOOP_COND_US:%.*]] = icmp slt i32 [[IV_NEXT_US]], [[N:%.*]] -; CHECK-NEXT: br i1 [[LOOP_COND_US]], label [[LOOP_US]], label [[EXIT_SPLIT_US:%.*]] +; CHECK-NEXT: %iv.next.us = add i32 %iv.us, 1 +; CHECK-NEXT: %loop.cond.us = icmp slt i32 %iv.next.us, %N +; CHECK-NEXT: br i1 %loop.cond.us, label %loop.us, label %exit.split.us +; CHECK: guarded.us: +; CHECK-NEXT: br label %backedge.us +; CHECK: exit.split.us: +; CHECK-NEXT: br label %exit +; CHECK: entry.split: +; CHECK-NEXT: br label %loop ; CHECK: loop: -; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ] -; CHECK-NEXT: [[CONDITION:%.*]] = icmp eq i32 [[IV]], 123 -; CHECK-NEXT: br i1 [[CONDITION]], label [[GUARD:%.*]], label [[BACKEDGE]] +; CHECK-NEXT: %iv = phi i32 [ 0, %entry.split ], [ %iv.next, %backedge ] +; CHECK-NEXT: %condition = icmp eq i32 %iv, 123 +; CHECK-NEXT: br i1 %condition, label %guard, label %backedge ; CHECK: guard: -; CHECK-NEXT: br label [[DEOPT:%.*]] +; CHECK-NEXT: br label %deopt ; CHECK: deopt: ; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ] ; CHECK-NEXT: unreachable ; CHECK: backedge: -; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1 -; CHECK-NEXT: [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], [[N]] -; CHECK-NEXT: br i1 [[LOOP_COND]], label %loop, label [[EXIT_SPLIT:%.*]] +; CHECK-NEXT: %iv.next = add i32 %iv, 1 +; CHECK-NEXT: %loop.cond = icmp slt i32 %iv.next, %N +; CHECK-NEXT: br i1 %loop.cond, label %loop, label %exit.split +; CHECK: exit.split: +; CHECK-NEXT: br label %exit +; CHECK: exit: +; CHECK-NEXT: ret void ; entry: @@ -135,7 +167,7 @@ exit: define void @test_nested_loop(i1 %cond, i32 %N, i1 %arg) { ; CHECK-LABEL: define void @test_nested_loop(i1 %cond, i32 %N, i1 %arg) { ; CHECK-NEXT: entry: -; CHECK-NEXT: br i1 %cond, label %entry.split, label %outer_loop.split +; CHECK-NEXT: br i1 %cond, label %entry.split, label %outer_loop.split, !prof !1 ; CHECK: entry.split: ; CHECK-NEXT: br i1 %arg, label %entry.split.split.us, label %entry.split.split ; CHECK: entry.split.split.us: @@ -204,17 +236,49 @@ exit: } define void @test_sibling_loops(i1 %cond1, i1 %cond2, i32 %N) { -; CHECK-LABEL: @test_sibling_loops( +; CHECK-LABEL: define void @test_sibling_loops(i1 %cond1, i1 %cond2, i32 %N) { ; CHECK-NEXT: entry: -; CHECK-NEXT: br i1 [[COND1:%.*]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]] -; CHECK: [[IV1_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US]] ], [ [[IV1_NEXT_US:%.*]], [[GUARDED_US:%.*]] ] -; CHECK-NEXT: br label [[GUARDED_US]] -; CHECK: call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ] +; CHECK-NEXT: br i1 %cond1, label %entry.split.us, label %entry.split, !prof !1 +; CHECK: entry.split.us: +; CHECK-NEXT: br label %loop1.us +; CHECK: loop1.us: +; CHECK-NEXT: %iv1.us = phi i32 [ 0, %entry.split.us ], [ %iv1.next.us, %guarded.us ] +; CHECK-NEXT: br label %guarded.us +; CHECK: guarded.us: +; CHECK-NEXT: %iv1.next.us = add i32 %iv1.us, 1 +; CHECK-NEXT: %loop1.cond.us = icmp slt i32 %iv1.next.us, %N +; CHECK-NEXT: br i1 %loop1.cond.us, label %loop1.us, label %between.split.us +; CHECK: between.split.us: +; CHECK-NEXT: br label %between +; CHECK: entry.split: +; CHECK-NEXT: br label %loop1 +; CHECK: loop1: +; CHECK-NEXT: br label %deopt +; CHECK: deopt: +; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ] ; CHECK-NEXT: unreachable -; CHECK: [[IV2_US:%.*]] = phi i32 [ 0, [[BETWEEN:%.*]] ], [ [[IV1_NEXT_US2:%.*]], [[GUARDED_US2:%.*]] ] -; CHECK-NEXT: br label [[GUARDED_US2]] -; CHECK: call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ] +; CHECK: between: +; CHECK-NEXT: br i1 %cond2, label %between.split.us2, label %between.split, !prof !1 +; CHECK: between.split.us2: +; CHECK-NEXT: br label %loop2.us +; CHECK: loop2.us: +; CHECK-NEXT: %iv2.us = phi i32 [ 0, %between.split.us2 ], [ %iv2.next.us, %guarded.us3 ] +; CHECK-NEXT: br label %guarded.us3 +; CHECK: guarded.us3: +; CHECK-NEXT: %iv2.next.us = add i32 %iv2.us, 1 +; CHECK-NEXT: %loop2.cond.us = icmp slt i32 %iv2.next.us, %N +; CHECK-NEXT: br i1 %loop2.cond.us, label %loop2.us, label %exit.split.us +; CHECK: exit.split.us: +; CHECK-NEXT: br label %exit +; CHECK: between.split: +; CHECK-NEXT: br label %loop2 +; CHECK: loop2: +; CHECK-NEXT: br label %deopt1 +; CHECK: deopt1: +; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ] ; CHECK-NEXT: unreachable +; CHECK: exit: +; CHECK-NEXT: ret void ; entry: @@ -242,11 +306,20 @@ exit: } ; Check that we don't do anything because of cleanuppad. -; CHECK-LABEL: @test_cleanuppad( -; CHECK: call void (i1, ...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ] -; CHECK-NOT: call void (i1, ...) @llvm.experimental.guard( define void @test_cleanuppad(i1 %cond, i32 %N) personality ptr @__CxxFrameHandler3 { - +; CHECK-LABEL: define void @test_cleanuppad(i1 %cond, i32 %N) personality ptr @__CxxFrameHandler3 { +; CHECK-NEXT: entry: +; CHECK-NEXT: br label %loop +; CHECK: loop: +; CHECK-NEXT: %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] +; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ] +; CHECK-NEXT: %iv.next = add i32 %iv, 1 +; CHECK-NEXT: invoke void @may_throw(i32 %iv) +; CHECK-NEXT: to label %loop unwind label %exit +; CHECK: exit: +; CHECK-NEXT: %cp = cleanuppad within none [] +; CHECK-NEXT: cleanupret from %cp unwind to caller +; entry: br label %loop @@ -264,3 +337,9 @@ exit: declare void @may_throw(i32 %i) declare i32 @__CxxFrameHandler3(...) + +!0 = !{!"function_entry_count", i32 10} +;. +; CHECK: !0 = !{!"function_entry_count", i32 10} +; CHECK: !1 = !{!"branch_weights", i32 1048575, i32 1} +;. _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
