https://github.com/yxsamliu updated https://github.com/llvm/llvm-project/pull/221517
>From b5fc30b4d6bf6fd2dcdef7bf2aeb872706544cb7 Mon Sep 17 00:00:00 2001 From: "Yaxun (Sam) Liu" <[email protected]> Date: Wed, 28 Oct 2026 16:47:54 +0000 Subject: [PATCH] [CHR] Use branch uniformity profiles to select scopes Lane counts can make a divergent GPU branch look strongly biased. For example, if 63 of 64 lanes take one path in every wave, that path has a high count even though every wave also executes the other path. CHR may duplicate control flow and increase register pressure without avoiding the work on the other path. When a function has uniformity.profile, require branch.uniformity.profile on each biased branch used to form a candidate CHR scope. Skip the scope if any such branch lacks the hint. Missing hints can also mean missing observations, so this is a conservative profitability decision. Functions without uniformity data and scopes containing only selects retain their existing behavior. --- .../ControlHeightReduction.cpp | 37 ++++ .../chr-uniformity-profile-amdgpu.ll | 5 + .../PGOProfile/chr-uniformity-profile.ll | 174 ++++++++++++++++++ 3 files changed, 216 insertions(+) create mode 100644 llvm/test/Transforms/PGOProfile/chr-uniformity-profile-amdgpu.ll create mode 100644 llvm/test/Transforms/PGOProfile/chr-uniformity-profile.ll diff --git a/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp b/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp index 8984e6cbdc6f62..67f4ffa543b9c4 100644 --- a/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp +++ b/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp @@ -356,6 +356,11 @@ class CHR { OptimizationRemarkEmitter &ORE; CHRStats Stats; + // A function attachment indicates that uniformity profile data is available. + // This preserves CHR's existing behavior when no uniformity profile was + // collected. + bool HasUniformityProfile = F.getMetadata(LLVMContext::MD_uniformity_profile); + // All the true-biased regions in the function DenseSet<Region *> TrueBiasedRegionsGlobal; // All the false-biased regions in the function @@ -1307,9 +1312,41 @@ static bool hasAtLeastTwoBiasedBranches(const InstrumentationOptions &Opts, return NumBiased >= Opts.chr_merge_threshold; } +static Instruction * +findBranchWithoutUniformityProfile(const DenseSet<Region *> &Regions) { + for (Region *R : Regions) { + Instruction *Branch = R->getEntry()->getTerminator(); + if (!Branch->getMetadata(LLVMContext::MD_branch_uniformity_profile)) + return Branch; + } + return nullptr; +} + void CHR::filterScopes(SmallVectorImpl<CHRScope *> &Input, SmallVectorImpl<CHRScope *> &Output) { for (CHRScope *Scope : Input) { + // Branch weights count individual lanes. A divergent branch can therefore + // look strongly biased even when every wave executes both paths. Avoid + // applying CHR to such a scope because duplicating its control flow may + // increase live ranges and register pressure without eliminating control + // flow for the wave. + if (HasUniformityProfile) { + Instruction *BranchWithoutUniformityProfile = + findBranchWithoutUniformityProfile(Scope->TrueBiasedRegions); + if (!BranchWithoutUniformityProfile) + BranchWithoutUniformityProfile = + findBranchWithoutUniformityProfile(Scope->FalseBiasedRegions); + if (BranchWithoutUniformityProfile) { + ORE.emit([&]() { + return OptimizationRemarkMissed(DEBUG_TYPE, "BranchNotKnownUniform", + BranchWithoutUniformityProfile) + << "drop scope containing a branch that is not known to be " + "uniform"; + }); + continue; + } + } + // Filter out the ones with only one region and no subs. if (!hasAtLeastTwoBiasedBranches(Opts, Scope)) { CHR_DEBUG(dbgs() << "Filtered out by biased branches truthy-regions " diff --git a/llvm/test/Transforms/PGOProfile/chr-uniformity-profile-amdgpu.ll b/llvm/test/Transforms/PGOProfile/chr-uniformity-profile-amdgpu.ll new file mode 100644 index 00000000000000..f8552e2ce64196 --- /dev/null +++ b/llvm/test/Transforms/PGOProfile/chr-uniformity-profile-amdgpu.ll @@ -0,0 +1,5 @@ +; REQUIRES: amdgpu-registered-target +; RUN: opt < %S/chr-uniformity-profile.ll -mtriple=amdgpu9.50-amd-amdhsa -passes='default<O3>' -S | FileCheck %S/chr-uniformity-profile.ll + +; Run the shared scope checks with the AMDGPU O3 pipeline. Keep this separately +; gated so the target-independent checks also run in builds without AMDGPU. diff --git a/llvm/test/Transforms/PGOProfile/chr-uniformity-profile.ll b/llvm/test/Transforms/PGOProfile/chr-uniformity-profile.ll new file mode 100644 index 00000000000000..056637ffca88b0 --- /dev/null +++ b/llvm/test/Transforms/PGOProfile/chr-uniformity-profile.ll @@ -0,0 +1,174 @@ +; RUN: opt < %s -passes='require<profile-summary>,function(chr,instcombine,simplifycfg)' -S | FileCheck %s +; RUN: opt < %s -passes='default<O3>' -S | FileCheck %s + +declare void @foo() + +; Preserve the existing behavior when a function has no uniformity profile. +define void @no_profile(ptr %ptr) !prof !14 { +; CHECK-LABEL: define void @no_profile( +; CHECK: entry.split.nonchr: +entry: + %value = load i32, ptr %ptr + %bit0 = and i32 %value, 1 + %cond0 = icmp eq i32 %bit0, 0 + br i1 %cond0, label %bb1, label %bb0, !prof !15 + +bb0: + call void @foo() + br label %bb1 + +bb1: + %bit1 = and i32 %value, 2 + %cond1 = icmp eq i32 %bit1, 0 + br i1 %cond1, label %exit, label %bb2, !prof !15 + +bb2: + call void @foo() + br label %exit + +exit: + ret void +} + +; Uniform branches remain eligible for CHR when uniformity profile is +; present. +define void @uniform(ptr %ptr) !prof !14 !uniformity.profile !16 { +; CHECK-LABEL: define void @uniform( +; CHECK: entry.split.nonchr: +entry: + %value = load i32, ptr %ptr + %bit0 = and i32 %value, 1 + %cond0 = icmp eq i32 %bit0, 0 + br i1 %cond0, label %bb1, label %bb0, !prof !15, + !branch.uniformity.profile !16 + +bb0: + call void @foo() + br label %bb1 + +bb1: + %bit1 = and i32 %value, 2 + %cond1 = icmp eq i32 %bit1, 0 + br i1 %cond1, label %exit, label %bb2, !prof !15, + !branch.uniformity.profile !16 + +bb2: + call void @foo() + br label %exit + +exit: + ret void +} + +; With uniformity profile, a branch without branch-uniformity metadata +; is not known to be uniform. Do not apply CHR to a scope containing one. +define void @not_known_uniform(ptr %ptr) !prof !14 !uniformity.profile !16 { +; CHECK-LABEL: define void @not_known_uniform( +; CHECK-NOT: split +; CHECK: ret void +entry: + %value = load i32, ptr %ptr + %bit0 = and i32 %value, 1 + %cond0 = icmp eq i32 %bit0, 0 + br i1 %cond0, label %bb1, label %bb0, !prof !15, + !branch.uniformity.profile !16 + +bb0: + call void @foo() + br label %bb1 + +bb1: + %bit1 = and i32 %value, 2 + %cond1 = icmp eq i32 %bit1, 0 + br i1 %cond1, label %exit, label %bb2, !prof !15 + +bb2: + call void @foo() + br label %exit + +exit: + ret void +} + +; A scope with a branch that is not known uniform does not disable a separate +; uniform scope in the same function. +define void @per_scope(ptr %uniform_ptr, ptr %divergent_ptr) !prof !14 !uniformity.profile !16 { +; CHECK-LABEL: define void @per_scope( +; CHECK: entry.split.nonchr: +; CHECK-NOT: after.uniform.split.nonchr: +; CHECK: after.uniform: +entry: + %uniform_value = load i32, ptr %uniform_ptr + %uniform_bit0 = and i32 %uniform_value, 1 + %uniform_cond0 = icmp eq i32 %uniform_bit0, 0 + br i1 %uniform_cond0, label %uniform.bb1, label %uniform.bb0, !prof !15, + !branch.uniformity.profile !16 + +uniform.bb0: + call void @foo() + br label %uniform.bb1 + +uniform.bb1: + %uniform_bit1 = and i32 %uniform_value, 2 + %uniform_cond1 = icmp eq i32 %uniform_bit1, 0 + br i1 %uniform_cond1, label %after.uniform, label %uniform.bb2, !prof !15, + !branch.uniformity.profile !16 + +uniform.bb2: + call void @foo() + br label %after.uniform + +after.uniform: + %divergent_value = load i32, ptr %divergent_ptr + %divergent_bit0 = and i32 %divergent_value, 1 + %divergent_cond0 = icmp eq i32 %divergent_bit0, 0 + br i1 %divergent_cond0, label %divergent.bb1, label %divergent.bb0, + !prof !15 + +divergent.bb0: + call void @foo() + br label %divergent.bb1 + +divergent.bb1: + %divergent_bit1 = and i32 %divergent_value, 2 + %divergent_cond1 = icmp eq i32 %divergent_bit1, 0 + br i1 %divergent_cond1, label %exit, label %divergent.bb2, !prof !15 + +divergent.bb2: + call void @foo() + br label %exit + +exit: + ret void +} + +; Scopes containing only non-Boolean selects keep their existing behavior. +define i32 @select_only(i32 %value) !prof !14 !uniformity.profile !16 { +; CHECK-LABEL: define i32 @select_only( +; CHECK: entry.split.nonchr: +entry: + %bit0 = and i32 %value, 1 + %cond0 = icmp eq i32 %bit0, 0 + %sum0 = select i1 %cond0, i32 %value, i32 42, !prof !15 + %bit1 = and i32 %value, 2 + %cond1 = icmp eq i32 %bit1, 0 + %sum1 = select i1 %cond1, i32 %sum0, i32 43, !prof !15 + ret i32 %sum1 +} + +!llvm.module.flags = !{!0} +!0 = !{i32 1, !"ProfileSummary", !1} +!1 = !{!2, !3, !4, !5, !6, !7, !8, !9} +!2 = !{!"ProfileFormat", !"InstrProf"} +!3 = !{!"TotalCount", i64 10000} +!4 = !{!"MaxCount", i64 10} +!5 = !{!"MaxInternalCount", i64 1} +!6 = !{!"MaxFunctionCount", i64 1000} +!7 = !{!"NumCounts", i64 1} +!8 = !{!"NumFunctions", i64 1} +!9 = !{!"DetailedSummary", !10} +!10 = !{!11} +!11 = !{i32 999999, i64 1, i32 1} +!14 = !{!"function_entry_count", i64 100} +!15 = !{!"branch_weights", i32 0, i32 1} +!16 = !{} _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
