https://github.com/kubamracek created https://github.com/llvm/llvm-project/pull/229402
This is being discussed in an RFC: https://discourse.llvm.org/t/rfc-optimizing-linkonce-odr-functions-frontend-hints-inlining-and-internal-cloning/92423 Some more context is in this PR: https://github.com/llvm/llvm-project/pull/228074 This adds, a (currently off-by-default) pass that can perform internalization of linkonce_odr functions. Three modes: - off ... default - likely-module-local ... only when the frontend (Clang) annotates the function as coming from a .cpp file and not a .h file - see https://github.com/llvm/llvm-project/pull/228074 and the RFC detailing this idea - all ... for experimentation and for reference, it's useful to be able to assess how much full internalization shifts performance and codesize Currently, this PR includes the commits from https://github.com/llvm/llvm-project/pull/228074. I don't know how to make a stacked PR 😄 >From 8c5319cf5ccae6701f4da1163494281249744638 Mon Sep 17 00:00:00 2001 From: Kuba Mracek <[email protected]> Date: Thu, 1 Oct 2026 11:57:38 +0100 Subject: [PATCH 1/3] [clang][Inliner] Add opt-in inliner hints for lambdas and main-file functions Clang now marks function definitions with "clang-lambda" (the function is a lambda) and "clang-main-file" (the function is defined in the main source file, as opposed to in a header-file) with the value of "inline-or-template" if it is declared inline or is a template instantiation. With -mllvm -inline-use-clang-hints (off by default), the inliner raises the thresholds for such callees, and makes lambdas, template/inline code defined in main source, and especially the combination of both. Also, allows sole-call bonus for the lambda-in-a-main-file situation (we can reasonably expect the original lambda to be completely unused after inlining, even across TUs). This makes this specific subset of typically linkonce_odr functions more likely to get inlined and optimized. No bonus applies to minsize callers or to cold call sites or callees. --- clang/lib/CodeGen/CodeGenModule.cpp | 16 +++++ clang/test/CodeGenCXX/inline-origin-hints.cpp | 29 +++++++++ llvm/lib/Analysis/InlineCost.cpp | 55 +++++++++++++++- llvm/test/Transforms/Inline/clang-hints.ll | 63 +++++++++++++++++++ 4 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 clang/test/CodeGenCXX/inline-origin-hints.cpp create mode 100644 llvm/test/Transforms/Inline/clang-hints.ll diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index d19d64e2e1c60..a2756988464a1 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -3296,6 +3296,22 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D, if (CodeGenOpts.DisableOutlining || D->hasAttr<NoOutlineAttr>()) B.addAttribute(llvm::Attribute::NoOutline); + // Hints for the optimizer (see -mllvm -inline-use-clang-hints) helping with + // linkonce_odr C++ functions: is this a lambda, is it defined in the main + // source file, is it declared inline or a template instantiation. + if (isLambdaCallOperator(dyn_cast<DeclContext>(D))) + B.addAttribute("clang-lambda"); + // D->getLocation(), for a template instantiation, is the location of the + // template's definition, not instantiation. + const SourceManager &SM = getContext().getSourceManager(); + if (SM.isInMainFile(D->getLocation())) { + const auto *FD = dyn_cast<FunctionDecl>(D); + bool InlineOrTemplate = + FD && (FD->isInlineSpecified() || FD->isTemplateInstantiation()); + B.addAttribute("clang-main-file", + InlineOrTemplate ? "inline-or-template" : ""); + } + F->addFnAttrs(B); llvm::MaybeAlign ExplicitAlignment; diff --git a/clang/test/CodeGenCXX/inline-origin-hints.cpp b/clang/test/CodeGenCXX/inline-origin-hints.cpp new file mode 100644 index 0000000000000..f73708a8309c0 --- /dev/null +++ b/clang/test/CodeGenCXX/inline-origin-hints.cpp @@ -0,0 +1,29 @@ +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: %clang_cc1 -triple arm64-apple-macosx14.0.0 -std=c++20 -emit-llvm -o - %t/main.cpp | FileCheck %s + +//--- hdr.h +inline int hdr_inline(int x) { return x * 3; } +template <class F> int hdr_call(F f) { return f(); } + +//--- main.cpp +#include "hdr.h" + +int main_plain(int x) { return x + 1; } +inline int main_inline(int x) { return x * 7; } +template <class T> int main_tmpl(T x) { return hdr_call([&] { return x * 2; }); } + +int use(int x) { return main_plain(x) + main_inline(x) + main_tmpl(x) + hdr_inline(x); } + +// CHECK: define{{.*}} i32 @_Z10main_plaini({{.*}}) #[[PLAIN:[0-9]+]] +// CHECK: define{{.*}} i32 @_Z3usei({{.*}}) #[[PLAIN]] +// CHECK: define{{.*}} i32 @_Z11main_inlinei({{.*}}) #[[INL:[0-9]+]] +// CHECK: define{{.*}} i32 @_Z9main_tmplIiEiT_({{.*}}) #[[INL]] +// CHECK: define{{.*}} i32 @_Z10hdr_inlinei({{.*}}) #[[HDR:[0-9]+]] +// CHECK: define{{.*}} i32 @_ZZ9main_tmplIiEiT_ENKUlvE_clEv({{.*}}) #[[LAMBDA:[0-9]+]] + +// CHECK: attributes #[[PLAIN]] = { {{.*}}"clang-main-file" {{.*}} } +// CHECK: attributes #[[INL]] = { {{.*}}"clang-main-file"="inline-or-template"{{.*}} } +// CHECK: attributes #[[HDR]] = { +// CHECK-NOT: clang- +// CHECK: attributes #[[LAMBDA]] = { {{.*}}"clang-lambda" "clang-main-file"="inline-or-template"{{.*}} } diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp index 0a540c1b8fb67..53bcde650c65c 100644 --- a/llvm/lib/Analysis/InlineCost.cpp +++ b/llvm/lib/Analysis/InlineCost.cpp @@ -183,6 +183,27 @@ static cl::opt<bool> InlineAllViableCalls( "inline-all-viable-calls", cl::Hidden, cl::init(false), cl::desc("Inline all viable calls, even if they exceed the inlining " "threshold")); + +static cl::opt<bool> UseClangHints( + "inline-use-clang-hints", cl::Hidden, cl::init(false), + cl::desc("Use clang's lambda/main-file hints to raise thresholds")); + +static cl::opt<int> ClangHintsLambdaBonus( + "clang-hints-lambda-bonus", cl::Hidden, cl::init(500), + cl::desc("Threshold bonus for lambdas")); + +static cl::opt<int> ClangHintsMainFileTemplateBonus( + "clang-hints-main-file-template-bonus", cl::Hidden, cl::init(150), + cl::desc("Threshold bonus for main-file inline/template functions")); + +static cl::opt<int> ClangHintsLambdaMainFileBonus( + "clang-hints-lambda-main-file-bonus", cl::Hidden, cl::init(3000), + cl::desc("Threshold bonus for lambdas in the main file")); + +static cl::opt<bool> ClangHintsLocalLambdas( + "clang-hints-local-lambdas", cl::Hidden, cl::init(true), + cl::desc("Treat main-file linkonce_odr lambdas as local functions")); + namespace llvm { std::optional<int> getStringFnAttrAsInt(const Attribute &Attr) { if (Attr.isValid()) { @@ -1247,10 +1268,20 @@ class InlineCostCallAnalyzer final : public CallAnalyzer { }; // Return true if CB is the sole call to local function Callee. +// +// With UseClangHints and ClangHintsLocalLambdas, a linkonce_odr function +// that's (1) a lambda, and (2) defined in the main source file, we count as +// local too. Ignoring very unusual setups (#include-ing a .cpp file), no other +// translation unit can reuse such lambda, so we expect inlining it turns the +// original dead. static bool isSoleCallToLocalFunction(const CallBase &CB, const Function &Callee) { - return Callee.hasLocalLinkage() && Callee.hasOneLiveUse() && - &Callee == CB.getCalledFunction(); + bool Local = Callee.hasLocalLinkage() || + (UseClangHints && ClangHintsLocalLambdas && + Callee.hasLinkOnceODRLinkage() && + Callee.hasFnAttribute("clang-lambda") && + Callee.hasFnAttribute("clang-main-file")); + return Local && Callee.hasOneLiveUse() && &Callee == CB.getCalledFunction(); } class InlineCostFeaturesAnalyzer final : public CallAnalyzer { @@ -2104,12 +2135,14 @@ void InlineCostCallAnalyzer::updateThreshold(CallBase &Call, Function &Callee) { int SingleBBBonusPercent = 50; int VectorBonusPercent = TTI.getInlinerVectorBonusPercent(); int LastCallToStaticBonus = TTI.getInliningLastCallToStaticBonus(); + bool ClangHintBonusAllowed = true; // Lambda to set all the above bonus and bonus percentages to 0. auto DisallowAllBonuses = [&]() { SingleBBBonusPercent = 0; VectorBonusPercent = 0; LastCallToStaticBonus = 0; + ClangHintBonusAllowed = false; }; // Use the OptMinSizeThreshold or OptSizeThreshold knob if they are available @@ -2122,6 +2155,7 @@ void InlineCostCallAnalyzer::updateThreshold(CallBase &Call, Function &Callee) { // call/return instructions. SingleBBBonusPercent = 0; VectorBonusPercent = 0; + ClangHintBonusAllowed = false; } else if (Caller->hasOptSize()) Threshold = MinIfValid(Threshold, Params.OptSizeThreshold); @@ -2194,6 +2228,23 @@ void InlineCostCallAnalyzer::updateThreshold(CallBase &Call, Function &Callee) { SingleBBBonus = Threshold * SingleBBBonusPercent / 100; VectorBonus = Threshold * VectorBonusPercent / 100; + // A lambda is usually specific to its call sites, very often there's only + // one call site, and inlining is especially benefitial: Captured variables + // turn back into the caller's locals. A function defined in the main source + // file (very typically) isn't used in other translation unit, inlining it + // cannot multiply copies across them (that's what linkonce_odr tries to + // prevent). Boost those situations. + if (UseClangHints && ClangHintBonusAllowed) { + bool IsLambda = Callee.hasFnAttribute("clang-lambda"); + Attribute MainFile = Callee.getFnAttribute("clang-main-file"); + if (IsLambda && MainFile.isValid()) + Threshold += ClangHintsLambdaMainFileBonus; + else if (IsLambda) + Threshold += ClangHintsLambdaBonus; + else if (MainFile.getValueAsString() == "inline-or-template") + Threshold += ClangHintsMainFileTemplateBonus; + } + // If there is only one call of the function, and it has internal linkage, // the cost of inlining it drops dramatically. It may seem odd to update // Cost in updateThreshold, but the bonus depends on the logic in this method. diff --git a/llvm/test/Transforms/Inline/clang-hints.ll b/llvm/test/Transforms/Inline/clang-hints.ll new file mode 100644 index 0000000000000..01b3719ce31f8 --- /dev/null +++ b/llvm/test/Transforms/Inline/clang-hints.ll @@ -0,0 +1,63 @@ +; RUN: opt -passes=inline -inline-threshold=0 -S < %s | FileCheck %s --check-prefix=OFF +; RUN: opt -passes=inline -inline-threshold=0 -inline-use-clang-hints -S < %s | FileCheck %s --check-prefix=HINTS +; RUN: opt -passes=inline -inline-threshold=0 -inline-use-clang-hints -clang-hints-lambda-main-file-bonus=0 -S < %s | FileCheck %s --check-prefix=LOCAL +; RUN: opt -passes=inline -inline-threshold=0 -inline-use-clang-hints -clang-hints-lambda-main-file-bonus=0 -clang-hints-local-lambdas=false -S < %s | FileCheck %s --check-prefix=NOLOCAL + +; OFF-LABEL: define i32 @caller( +; OFF-COUNT-2: call i32 @lambda( +; OFF-COUNT-2: call i32 @main_template( +; OFF-COUNT-2: call i32 @main_plain( +; OFF: call i32 @sole_main_inline( + +; HINTS-LABEL: define i32 @caller( +; HINTS-NOT: call i32 @lambda( +; HINTS-COUNT-2: call i32 @main_template( +; HINTS-COUNT-2: call i32 @main_plain( +; HINTS-NOT: call i32 @sole_main_inline( +; HINTS: ret i32 + +; LOCAL-LABEL: define i32 @caller( +; LOCAL-NOT: call i32 @sole_main_inline( +; LOCAL: ret i32 + +; NOLOCAL-LABEL: define i32 @caller( +; NOLOCAL: call i32 @sole_main_inline( +define i32 @caller(i32 %x) { + %a1 = call i32 @lambda(i32 %x) + %a2 = call i32 @lambda(i32 %a1) + %b1 = call i32 @main_template(i32 %a2) + %b2 = call i32 @main_template(i32 %b1) + %c1 = call i32 @main_plain(i32 %b2) + %c2 = call i32 @main_plain(i32 %c1) + %d = call i32 @sole_main_inline(i32 %c2) + ret i32 %d +} + +define linkonce_odr i32 @lambda(i32 %x) "clang-lambda" "function-inline-cost"="300" { + %r = add i32 %x, 1 + ret i32 %r +} + +define linkonce_odr i32 @main_template(i32 %x) "clang-main-file"="inline-or-template" "function-inline-cost"="300" { + %r = add i32 %x, 2 + ret i32 %r +} + +define linkonce_odr i32 @main_plain(i32 %x) "clang-main-file" "function-inline-cost"="300" { + %r = add i32 %x, 3 + ret i32 %r +} + +define linkonce_odr i32 @sole_main_inline(i32 %x) "clang-lambda" "clang-main-file" { + %v1 = mul i32 %x, 3 + %v2 = mul i32 %v1, 5 + %v3 = mul i32 %v2, 7 + %v4 = mul i32 %v3, 11 + %v5 = mul i32 %v4, 13 + %v6 = mul i32 %v5, 17 + %v7 = mul i32 %v6, 19 + %v8 = mul i32 %v7, 23 + %v9 = mul i32 %v8, 29 + %v10 = mul i32 %v9, 31 + ret i32 %v10 +} >From 6208ab1389dd48b11e05fdf9e22a4c9d96e3c675 Mon Sep 17 00:00:00 2001 From: Kuba Mracek <[email protected]> Date: Mon, 5 Oct 2026 22:24:48 +0100 Subject: [PATCH 2/3] [clang][Inliner] Simplify module-local hints to a single "frontend-hint-likely-module-local" attribute --- clang/lib/CodeGen/CodeGenModule.cpp | 29 +++--- .../frontend-hint-likely-module-local.cpp | 56 +++++++++++ clang/test/CodeGenCXX/inline-origin-hints.cpp | 29 ------ llvm/lib/Analysis/InlineCost.cpp | 55 +---------- llvm/test/Transforms/Inline/clang-hints.ll | 63 ------------ .../frontend-hint-likely-module-local.ll | 95 +++++++++++++++++++ 6 files changed, 172 insertions(+), 155 deletions(-) create mode 100644 clang/test/CodeGenCXX/frontend-hint-likely-module-local.cpp delete mode 100644 clang/test/CodeGenCXX/inline-origin-hints.cpp delete mode 100644 llvm/test/Transforms/Inline/clang-hints.ll create mode 100644 llvm/test/Transforms/Inline/frontend-hint-likely-module-local.ll diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index a2756988464a1..011aae06effef 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -94,6 +94,12 @@ static llvm::cl::opt<bool> LimitedCoverage( "limited-coverage-experimental", llvm::cl::Hidden, llvm::cl::desc("Emit limited coverage mapping information (experimental)")); +static llvm::cl::opt<bool> EmitModuleLocalHints( + "clang-emit-module-local-hints", llvm::cl::Hidden, + llvm::cl::init(false), + llvm::cl::desc("Mark inline and template functions defined in the main " + "source file with \"frontend-hint-likely-module-local\"")); + static const char AnnotationSection[] = "llvm.metadata"; static constexpr auto ErrnoTBAAMDName = "llvm.errno.tbaa"; @@ -3296,20 +3302,17 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D, if (CodeGenOpts.DisableOutlining || D->hasAttr<NoOutlineAttr>()) B.addAttribute(llvm::Attribute::NoOutline); - // Hints for the optimizer (see -mllvm -inline-use-clang-hints) helping with - // linkonce_odr C++ functions: is this a lambda, is it defined in the main - // source file, is it declared inline or a template instantiation. - if (isLambdaCallOperator(dyn_cast<DeclContext>(D))) - B.addAttribute("clang-lambda"); - // D->getLocation(), for a template instantiation, is the location of the - // template's definition, not instantiation. - const SourceManager &SM = getContext().getSourceManager(); - if (SM.isInMainFile(D->getLocation())) { + // An inline function or template instantiation defined in the main source + // file is (ignoring unusual setups like #include-ing a .cpp file) not + // referenced or emitted by any other translation unit. + if (EmitModuleLocalHints) { const auto *FD = dyn_cast<FunctionDecl>(D); - bool InlineOrTemplate = - FD && (FD->isInlineSpecified() || FD->isTemplateInstantiation()); - B.addAttribute("clang-main-file", - InlineOrTemplate ? "inline-or-template" : ""); + const SourceManager &SM = getContext().getSourceManager(); + // D->getLocation(), for a template instantiation, is the location of the + // template's definition, not instantiation. + bool InMainFile = SM.isInMainFile(D->getLocation()); + if (FD && (FD->isInlined() || FD->isTemplateInstantiation()) && InMainFile) + B.addAttribute("frontend-hint-likely-module-local"); } F->addFnAttrs(B); diff --git a/clang/test/CodeGenCXX/frontend-hint-likely-module-local.cpp b/clang/test/CodeGenCXX/frontend-hint-likely-module-local.cpp new file mode 100644 index 0000000000000..3bbea1ce840e2 --- /dev/null +++ b/clang/test/CodeGenCXX/frontend-hint-likely-module-local.cpp @@ -0,0 +1,56 @@ +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: %clang_cc1 -triple arm64-apple-macosx14.0.0 -std=c++20 -emit-llvm -mllvm -clang-emit-module-local-hints -o %t/on.ll %t/main.cpp +// RUN: FileCheck %s < %t/on.ll +// RUN: FileCheck %s --check-prefix=PLAIN < %t/on.ll +// RUN: FileCheck %s --check-prefix=HDR < %t/on.ll +// RUN: FileCheck %s --check-prefix=HDRTMPL < %t/on.ll +// RUN: %clang_cc1 -triple arm64-apple-macosx14.0.0 -std=c++20 -emit-llvm -o - %t/main.cpp | FileCheck %s --check-prefix=OFF + +//--- hdr.h +inline int hdr_inline(int x) { return x * 3; } +template <class F> int hdr_call(F f) { return f(); } + +//--- main.cpp +#include "hdr.h" + +struct S { int m(int x) { return x - 1; } }; +int main_plain(int x) { return x + 1; } +inline int main_inline(int x) { return x * 7; } +template <class T> int main_tmpl(T x) { return hdr_call([&] { return x * 2; }); } + +int use(int x) { + return main_plain(x) + main_inline(x) + main_tmpl(x) + hdr_inline(x) + + S().m(x); +} + +// Inline functions, template instantiations and lambdas defined in main.cpp. +// CHECK-DAG: define{{.*}} i32 @_Z11main_inlinei({{.*}}) #[[INL:[0-9]+]] +// CHECK-DAG: define{{.*}} i32 @_Z9main_tmplIiEiT_({{.*}}) #[[TMPL:[0-9]+]] +// CHECK-DAG: define{{.*}} i32 @_ZZ9main_tmplIiEiT_ENKUlvE_clEv({{.*}}) #[[LAMBDA:[0-9]+]] +// CHECK-DAG: define{{.*}} i32 @_ZN1S1mEi({{.*}}) #[[MEMBER:[0-9]+]] +// CHECK-DAG: attributes #[[INL]] = { {{.*}}"frontend-hint-likely-module-local"{{.*}} } +// CHECK-DAG: attributes #[[TMPL]] = { {{.*}}"frontend-hint-likely-module-local"{{.*}} } +// CHECK-DAG: attributes #[[LAMBDA]] = { {{.*}}"frontend-hint-likely-module-local"{{.*}} } +// CHECK-DAG: attributes #[[MEMBER]] = { {{.*}}"frontend-hint-likely-module-local"{{.*}} } + +// Non-inline functions in main.cpp. +// PLAIN-DAG: define{{.*}} i32 @_Z10main_plaini({{.*}}) #[[PLAIN:[0-9]+]] +// PLAIN-DAG: define{{.*}} i32 @_Z3usei({{.*}}) #[[PLAIN]] +// PLAIN: attributes #[[PLAIN]] = { +// PLAIN-NOT: frontend-hint-likely-module-local +// PLAIN-SAME: } + +// Inline function defined in hdr.h. +// HDR: define{{.*}} i32 @_Z10hdr_inlinei({{.*}}) #[[HDR:[0-9]+]] +// HDR: attributes #[[HDR]] = { +// HDR-NOT: frontend-hint-likely-module-local +// HDR-SAME: } + +// Template defined in hdr.h, instantiated (with a main.cpp lambda) in main.cpp. +// HDRTMPL: define{{.*}} i32 @_Z8hdr_call{{.*}}) #[[HDRTMPL:[0-9]+]] +// HDRTMPL: attributes #[[HDRTMPL]] = { +// HDRTMPL-NOT: frontend-hint-likely-module-local +// HDRTMPL-SAME: } + +// OFF-NOT: frontend-hint-likely-module-local diff --git a/clang/test/CodeGenCXX/inline-origin-hints.cpp b/clang/test/CodeGenCXX/inline-origin-hints.cpp deleted file mode 100644 index f73708a8309c0..0000000000000 --- a/clang/test/CodeGenCXX/inline-origin-hints.cpp +++ /dev/null @@ -1,29 +0,0 @@ -// RUN: rm -rf %t -// RUN: split-file %s %t -// RUN: %clang_cc1 -triple arm64-apple-macosx14.0.0 -std=c++20 -emit-llvm -o - %t/main.cpp | FileCheck %s - -//--- hdr.h -inline int hdr_inline(int x) { return x * 3; } -template <class F> int hdr_call(F f) { return f(); } - -//--- main.cpp -#include "hdr.h" - -int main_plain(int x) { return x + 1; } -inline int main_inline(int x) { return x * 7; } -template <class T> int main_tmpl(T x) { return hdr_call([&] { return x * 2; }); } - -int use(int x) { return main_plain(x) + main_inline(x) + main_tmpl(x) + hdr_inline(x); } - -// CHECK: define{{.*}} i32 @_Z10main_plaini({{.*}}) #[[PLAIN:[0-9]+]] -// CHECK: define{{.*}} i32 @_Z3usei({{.*}}) #[[PLAIN]] -// CHECK: define{{.*}} i32 @_Z11main_inlinei({{.*}}) #[[INL:[0-9]+]] -// CHECK: define{{.*}} i32 @_Z9main_tmplIiEiT_({{.*}}) #[[INL]] -// CHECK: define{{.*}} i32 @_Z10hdr_inlinei({{.*}}) #[[HDR:[0-9]+]] -// CHECK: define{{.*}} i32 @_ZZ9main_tmplIiEiT_ENKUlvE_clEv({{.*}}) #[[LAMBDA:[0-9]+]] - -// CHECK: attributes #[[PLAIN]] = { {{.*}}"clang-main-file" {{.*}} } -// CHECK: attributes #[[INL]] = { {{.*}}"clang-main-file"="inline-or-template"{{.*}} } -// CHECK: attributes #[[HDR]] = { -// CHECK-NOT: clang- -// CHECK: attributes #[[LAMBDA]] = { {{.*}}"clang-lambda" "clang-main-file"="inline-or-template"{{.*}} } diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp index 53bcde650c65c..d9d9b9401be01 100644 --- a/llvm/lib/Analysis/InlineCost.cpp +++ b/llvm/lib/Analysis/InlineCost.cpp @@ -183,27 +183,6 @@ static cl::opt<bool> InlineAllViableCalls( "inline-all-viable-calls", cl::Hidden, cl::init(false), cl::desc("Inline all viable calls, even if they exceed the inlining " "threshold")); - -static cl::opt<bool> UseClangHints( - "inline-use-clang-hints", cl::Hidden, cl::init(false), - cl::desc("Use clang's lambda/main-file hints to raise thresholds")); - -static cl::opt<int> ClangHintsLambdaBonus( - "clang-hints-lambda-bonus", cl::Hidden, cl::init(500), - cl::desc("Threshold bonus for lambdas")); - -static cl::opt<int> ClangHintsMainFileTemplateBonus( - "clang-hints-main-file-template-bonus", cl::Hidden, cl::init(150), - cl::desc("Threshold bonus for main-file inline/template functions")); - -static cl::opt<int> ClangHintsLambdaMainFileBonus( - "clang-hints-lambda-main-file-bonus", cl::Hidden, cl::init(3000), - cl::desc("Threshold bonus for lambdas in the main file")); - -static cl::opt<bool> ClangHintsLocalLambdas( - "clang-hints-local-lambdas", cl::Hidden, cl::init(true), - cl::desc("Treat main-file linkonce_odr lambdas as local functions")); - namespace llvm { std::optional<int> getStringFnAttrAsInt(const Attribute &Attr) { if (Attr.isValid()) { @@ -1269,18 +1248,14 @@ class InlineCostCallAnalyzer final : public CallAnalyzer { // Return true if CB is the sole call to local function Callee. // -// With UseClangHints and ClangHintsLocalLambdas, a linkonce_odr function -// that's (1) a lambda, and (2) defined in the main source file, we count as -// local too. Ignoring very unusual setups (#include-ing a .cpp file), no other -// translation unit can reuse such lambda, so we expect inlining it turns the -// original dead. +// A linkonce_odr function marked "frontend-hint-likely-module-local" counts as +// local too: no other module is expected to reference it, so we expect +// inlining the sole call to turn the original dead. static bool isSoleCallToLocalFunction(const CallBase &CB, const Function &Callee) { bool Local = Callee.hasLocalLinkage() || - (UseClangHints && ClangHintsLocalLambdas && - Callee.hasLinkOnceODRLinkage() && - Callee.hasFnAttribute("clang-lambda") && - Callee.hasFnAttribute("clang-main-file")); + (Callee.hasLinkOnceODRLinkage() && + Callee.hasFnAttribute("frontend-hint-likely-module-local")); return Local && Callee.hasOneLiveUse() && &Callee == CB.getCalledFunction(); } @@ -2135,14 +2110,12 @@ void InlineCostCallAnalyzer::updateThreshold(CallBase &Call, Function &Callee) { int SingleBBBonusPercent = 50; int VectorBonusPercent = TTI.getInlinerVectorBonusPercent(); int LastCallToStaticBonus = TTI.getInliningLastCallToStaticBonus(); - bool ClangHintBonusAllowed = true; // Lambda to set all the above bonus and bonus percentages to 0. auto DisallowAllBonuses = [&]() { SingleBBBonusPercent = 0; VectorBonusPercent = 0; LastCallToStaticBonus = 0; - ClangHintBonusAllowed = false; }; // Use the OptMinSizeThreshold or OptSizeThreshold knob if they are available @@ -2155,7 +2128,6 @@ void InlineCostCallAnalyzer::updateThreshold(CallBase &Call, Function &Callee) { // call/return instructions. SingleBBBonusPercent = 0; VectorBonusPercent = 0; - ClangHintBonusAllowed = false; } else if (Caller->hasOptSize()) Threshold = MinIfValid(Threshold, Params.OptSizeThreshold); @@ -2228,23 +2200,6 @@ void InlineCostCallAnalyzer::updateThreshold(CallBase &Call, Function &Callee) { SingleBBBonus = Threshold * SingleBBBonusPercent / 100; VectorBonus = Threshold * VectorBonusPercent / 100; - // A lambda is usually specific to its call sites, very often there's only - // one call site, and inlining is especially benefitial: Captured variables - // turn back into the caller's locals. A function defined in the main source - // file (very typically) isn't used in other translation unit, inlining it - // cannot multiply copies across them (that's what linkonce_odr tries to - // prevent). Boost those situations. - if (UseClangHints && ClangHintBonusAllowed) { - bool IsLambda = Callee.hasFnAttribute("clang-lambda"); - Attribute MainFile = Callee.getFnAttribute("clang-main-file"); - if (IsLambda && MainFile.isValid()) - Threshold += ClangHintsLambdaMainFileBonus; - else if (IsLambda) - Threshold += ClangHintsLambdaBonus; - else if (MainFile.getValueAsString() == "inline-or-template") - Threshold += ClangHintsMainFileTemplateBonus; - } - // If there is only one call of the function, and it has internal linkage, // the cost of inlining it drops dramatically. It may seem odd to update // Cost in updateThreshold, but the bonus depends on the logic in this method. diff --git a/llvm/test/Transforms/Inline/clang-hints.ll b/llvm/test/Transforms/Inline/clang-hints.ll deleted file mode 100644 index 01b3719ce31f8..0000000000000 --- a/llvm/test/Transforms/Inline/clang-hints.ll +++ /dev/null @@ -1,63 +0,0 @@ -; RUN: opt -passes=inline -inline-threshold=0 -S < %s | FileCheck %s --check-prefix=OFF -; RUN: opt -passes=inline -inline-threshold=0 -inline-use-clang-hints -S < %s | FileCheck %s --check-prefix=HINTS -; RUN: opt -passes=inline -inline-threshold=0 -inline-use-clang-hints -clang-hints-lambda-main-file-bonus=0 -S < %s | FileCheck %s --check-prefix=LOCAL -; RUN: opt -passes=inline -inline-threshold=0 -inline-use-clang-hints -clang-hints-lambda-main-file-bonus=0 -clang-hints-local-lambdas=false -S < %s | FileCheck %s --check-prefix=NOLOCAL - -; OFF-LABEL: define i32 @caller( -; OFF-COUNT-2: call i32 @lambda( -; OFF-COUNT-2: call i32 @main_template( -; OFF-COUNT-2: call i32 @main_plain( -; OFF: call i32 @sole_main_inline( - -; HINTS-LABEL: define i32 @caller( -; HINTS-NOT: call i32 @lambda( -; HINTS-COUNT-2: call i32 @main_template( -; HINTS-COUNT-2: call i32 @main_plain( -; HINTS-NOT: call i32 @sole_main_inline( -; HINTS: ret i32 - -; LOCAL-LABEL: define i32 @caller( -; LOCAL-NOT: call i32 @sole_main_inline( -; LOCAL: ret i32 - -; NOLOCAL-LABEL: define i32 @caller( -; NOLOCAL: call i32 @sole_main_inline( -define i32 @caller(i32 %x) { - %a1 = call i32 @lambda(i32 %x) - %a2 = call i32 @lambda(i32 %a1) - %b1 = call i32 @main_template(i32 %a2) - %b2 = call i32 @main_template(i32 %b1) - %c1 = call i32 @main_plain(i32 %b2) - %c2 = call i32 @main_plain(i32 %c1) - %d = call i32 @sole_main_inline(i32 %c2) - ret i32 %d -} - -define linkonce_odr i32 @lambda(i32 %x) "clang-lambda" "function-inline-cost"="300" { - %r = add i32 %x, 1 - ret i32 %r -} - -define linkonce_odr i32 @main_template(i32 %x) "clang-main-file"="inline-or-template" "function-inline-cost"="300" { - %r = add i32 %x, 2 - ret i32 %r -} - -define linkonce_odr i32 @main_plain(i32 %x) "clang-main-file" "function-inline-cost"="300" { - %r = add i32 %x, 3 - ret i32 %r -} - -define linkonce_odr i32 @sole_main_inline(i32 %x) "clang-lambda" "clang-main-file" { - %v1 = mul i32 %x, 3 - %v2 = mul i32 %v1, 5 - %v3 = mul i32 %v2, 7 - %v4 = mul i32 %v3, 11 - %v5 = mul i32 %v4, 13 - %v6 = mul i32 %v5, 17 - %v7 = mul i32 %v6, 19 - %v8 = mul i32 %v7, 23 - %v9 = mul i32 %v8, 29 - %v10 = mul i32 %v9, 31 - ret i32 %v10 -} diff --git a/llvm/test/Transforms/Inline/frontend-hint-likely-module-local.ll b/llvm/test/Transforms/Inline/frontend-hint-likely-module-local.ll new file mode 100644 index 0000000000000..de9aa101b2615 --- /dev/null +++ b/llvm/test/Transforms/Inline/frontend-hint-likely-module-local.ll @@ -0,0 +1,95 @@ +; RUN: opt -passes=inline -inline-threshold=0 -S < %s | FileCheck %s + +; A linkonce_odr function marked "frontend-hint-likely-module-local" gets the +; last-call-to-static bonus for its sole call, like a local function would. + +; CHECK-LABEL: define i32 @caller( +; CHECK-NOT: call i32 @hint_once( +; CHECK-COUNT-2: call i32 @hint_twice( +; CHECK: call i32 @nohint_once( +; CHECK: call i32 @hint_external_once( +; CHECK: ret i32 +define i32 @caller(i32 %x) { + %a = call i32 @hint_once(i32 %x) + %b1 = call i32 @hint_twice(i32 %a) + %b2 = call i32 @hint_twice(i32 %b1) + %c = call i32 @nohint_once(i32 %b2) + %d = call i32 @hint_external_once(i32 %c) + ret i32 %d +} + +define linkonce_odr i32 @hint_once(i32 %x) "frontend-hint-likely-module-local" { + %v1 = mul i32 %x, 3 + %v2 = mul i32 %v1, 5 + %v3 = mul i32 %v2, 7 + %v4 = mul i32 %v3, 11 + %v5 = mul i32 %v4, 13 + %v6 = mul i32 %v5, 17 + %v7 = mul i32 %v6, 19 + %v8 = mul i32 %v7, 23 + %v9 = mul i32 %v8, 29 + %v10 = mul i32 %v9, 31 + %v11 = mul i32 %v10, 37 + %v12 = mul i32 %v11, 41 + %v13 = mul i32 %v12, 43 + %v14 = mul i32 %v13, 47 + %v15 = mul i32 %v14, 53 + ret i32 %v15 +} + +define linkonce_odr i32 @hint_twice(i32 %x) "frontend-hint-likely-module-local" { + %v1 = mul i32 %x, 3 + %v2 = mul i32 %v1, 5 + %v3 = mul i32 %v2, 7 + %v4 = mul i32 %v3, 11 + %v5 = mul i32 %v4, 13 + %v6 = mul i32 %v5, 17 + %v7 = mul i32 %v6, 19 + %v8 = mul i32 %v7, 23 + %v9 = mul i32 %v8, 29 + %v10 = mul i32 %v9, 31 + %v11 = mul i32 %v10, 37 + %v12 = mul i32 %v11, 41 + %v13 = mul i32 %v12, 43 + %v14 = mul i32 %v13, 47 + %v15 = mul i32 %v14, 53 + ret i32 %v15 +} + +define linkonce_odr i32 @nohint_once(i32 %x) { + %v1 = mul i32 %x, 3 + %v2 = mul i32 %v1, 5 + %v3 = mul i32 %v2, 7 + %v4 = mul i32 %v3, 11 + %v5 = mul i32 %v4, 13 + %v6 = mul i32 %v5, 17 + %v7 = mul i32 %v6, 19 + %v8 = mul i32 %v7, 23 + %v9 = mul i32 %v8, 29 + %v10 = mul i32 %v9, 31 + %v11 = mul i32 %v10, 37 + %v12 = mul i32 %v11, 41 + %v13 = mul i32 %v12, 43 + %v14 = mul i32 %v13, 47 + %v15 = mul i32 %v14, 53 + ret i32 %v15 +} + +define i32 @hint_external_once(i32 %x) "frontend-hint-likely-module-local" { + %v1 = mul i32 %x, 3 + %v2 = mul i32 %v1, 5 + %v3 = mul i32 %v2, 7 + %v4 = mul i32 %v3, 11 + %v5 = mul i32 %v4, 13 + %v6 = mul i32 %v5, 17 + %v7 = mul i32 %v6, 19 + %v8 = mul i32 %v7, 23 + %v9 = mul i32 %v8, 29 + %v10 = mul i32 %v9, 31 + %v11 = mul i32 %v10, 37 + %v12 = mul i32 %v11, 41 + %v13 = mul i32 %v12, 43 + %v14 = mul i32 %v13, 47 + %v15 = mul i32 %v14, 53 + ret i32 %v15 +} >From 29a26ffbcfd3c9a4b35508ab6f0f2d2b60f333a6 Mon Sep 17 00:00:00 2001 From: Kuba Mracek <[email protected]> Date: Tue, 6 Oct 2026 13:24:44 +0100 Subject: [PATCH 3/3] [IPO] Add InternalizeLinkOnceODR pass A function with linkonce_odr linkage is not subject to most IPO, to avoid breaking other TU's usage of our TU's optimized version. However, a copy/clone is a correct implementation, and calling a private copy instead is allowed as long as the function's address keeps being unique. The new pass, when enabled, internalizes linkonce_odr functions and redirects direct calls to them, either: - (1) in-place, if every use of the function is a direct call and the function is alone in its comdat - (2) by making a unnamed_addr clone "<name>.internal", and the original stays for all other uses. The pass is controlled by -enable-linkonce-odr-internalization and internalizes all or "likely module local" functions: - -enable-linkonce-odr-internalization=off (default): do nothing. - -enable-linkonce-odr-internalization=likely-module-local: only functions marked by the frontend with "frontend-hint-likely-module-local", which another translation unit is unlikely to also emit. - -enable-linkonce-odr-internalization=all: every linkonce_odr function. The pass is added to the module simplification pipeline before IPSCCP. --- .../Transforms/IPO/InternalizeLinkOnceODR.h | 38 ++++++ llvm/lib/Passes/PassBuilder.cpp | 1 + llvm/lib/Passes/PassBuilderPipelines.cpp | 5 + llvm/lib/Passes/PassRegistry.def | 1 + llvm/lib/Transforms/IPO/CMakeLists.txt | 1 + .../Transforms/IPO/InternalizeLinkOnceODR.cpp | 121 ++++++++++++++++++ .../InternalizeLinkOnceODR/basic.ll | 82 ++++++++++++ .../InternalizeLinkOnceODR/pipeline.ll | 23 ++++ 8 files changed, 272 insertions(+) create mode 100644 llvm/include/llvm/Transforms/IPO/InternalizeLinkOnceODR.h create mode 100644 llvm/lib/Transforms/IPO/InternalizeLinkOnceODR.cpp create mode 100644 llvm/test/Transforms/InternalizeLinkOnceODR/basic.ll create mode 100644 llvm/test/Transforms/InternalizeLinkOnceODR/pipeline.ll diff --git a/llvm/include/llvm/Transforms/IPO/InternalizeLinkOnceODR.h b/llvm/include/llvm/Transforms/IPO/InternalizeLinkOnceODR.h new file mode 100644 index 0000000000000..e2df918b66f2c --- /dev/null +++ b/llvm/include/llvm/Transforms/IPO/InternalizeLinkOnceODR.h @@ -0,0 +1,38 @@ +//===- InternalizeLinkOnceODR.h - Clone linkonce_odr functions ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_IPO_INTERNALIZELINKONCEODR_H +#define LLVM_TRANSFORMS_IPO_INTERNALIZELINKONCEODR_H + +#include "llvm/IR/PassManager.h" +#include "llvm/Support/Compiler.h" + +namespace llvm { + +class Module; + +/// Clone and internalize linkonce_odr functions in the current TU (when asked): +/// * -enable-linkonce-odr-internalization=off ... default, nothing +/// * -enable-linkonce-odr-internalization=likely-module-local ... when marked +/// with "frontend-hint-likely-module-local" function attribute +/// * -enable-linkonce-odr-internalization=all ... on all linkonce_odr functions +/// +/// Changing calls in the current TU to instead use a private copy of a +/// linkonce_odr function is allowed, as long as nothing relies on the address +/// of the function, so only direct calls are redirected, other uses keep +/// referring to the original. When all uses of a function are direct calls, the +/// internalization is done in-place. +class InternalizeLinkOnceODRPass + : public OptionalPassInfoMixin<InternalizeLinkOnceODRPass> { +public: + LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &); +}; + +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_IPO_INTERNALIZELINKONCEODR_H diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 30a6f75a1b86d..602870246d0dc 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -258,6 +258,7 @@ #include "llvm/Transforms/IPO/InferFunctionAttrs.h" #include "llvm/Transforms/IPO/Instrumentor.h" #include "llvm/Transforms/IPO/Internalize.h" +#include "llvm/Transforms/IPO/InternalizeLinkOnceODR.h" #include "llvm/Transforms/IPO/LowerTypeTests.h" #include "llvm/Transforms/IPO/MemProfContextDisambiguation.h" #include "llvm/Transforms/IPO/MergeFunctions.h" diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index bd37fffc8e991..73c3d7540fe2c 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -64,6 +64,7 @@ #include "llvm/Transforms/IPO/InferFunctionAttrs.h" #include "llvm/Transforms/IPO/Inliner.h" #include "llvm/Transforms/IPO/Instrumentor.h" +#include "llvm/Transforms/IPO/InternalizeLinkOnceODR.h" #include "llvm/Transforms/IPO/LowerTypeTests.h" #include "llvm/Transforms/IPO/MemProfContextDisambiguation.h" #include "llvm/Transforms/IPO/MergeFunctions.h" @@ -1222,6 +1223,10 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, invokePipelineEarlySimplificationEPCallbacks(MPM, Level, Phase); + // Give linkonce_odr functions internal copies before the interprocedural + // passes that require local linkage (IPSCCP, GlobalOpt, DeadArgElim, etc.). + MPM.addPass(InternalizeLinkOnceODRPass()); + // Interprocedural constant propagation now that basic cleanup has occurred // and prior to optimizing globals. // FIXME: This position in the pipeline hasn't been carefully considered in diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index af4ce5029551d..8cd25201f76f8 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -107,6 +107,7 @@ MODULE_PASS("ctx-instr-lower", PGOCtxProfLoweringPass()) MODULE_PASS("print<ctx-prof-analysis>", CtxProfAnalysisPrinterPass(errs())) MODULE_PASS("invalidate<all>", InvalidateAllAnalysesPass()) MODULE_PASS("instrumentor", InstrumentorPass()) +MODULE_PASS("internalize-linkonce-odr", InternalizeLinkOnceODRPass()) MODULE_PASS("jmc-instrumenter", JMCInstrumenterPass()) MODULE_PASS("lower-emutls", LowerEmuTLSPass()) MODULE_PASS("lower-global-dtors", LowerGlobalDtorsPass()) diff --git a/llvm/lib/Transforms/IPO/CMakeLists.txt b/llvm/lib/Transforms/IPO/CMakeLists.txt index fd791f9fe86f8..e67a25e755014 100644 --- a/llvm/lib/Transforms/IPO/CMakeLists.txt +++ b/llvm/lib/Transforms/IPO/CMakeLists.txt @@ -31,6 +31,7 @@ add_llvm_component_library(LLVMipo InstrumentorConfigFile.cpp InstrumentorStubPrinter.cpp Internalize.cpp + InternalizeLinkOnceODR.cpp LowerTypeTests.cpp MemProfContextDisambiguation.cpp MergeFunctions.cpp diff --git a/llvm/lib/Transforms/IPO/InternalizeLinkOnceODR.cpp b/llvm/lib/Transforms/IPO/InternalizeLinkOnceODR.cpp new file mode 100644 index 0000000000000..b176abbdeea9d --- /dev/null +++ b/llvm/lib/Transforms/IPO/InternalizeLinkOnceODR.cpp @@ -0,0 +1,121 @@ +//===- InternalizeLinkOnceODR.cpp - Clone linkonce_odr functions ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/IPO/InternalizeLinkOnceODR.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/IR/Comdat.h" +#include "llvm/IR/InstIterator.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Transforms/Utils/Cloning.h" +#include "llvm/Transforms/Utils/ValueMapper.h" + +using namespace llvm; + +#define DEBUG_TYPE "internalize-linkonce-odr" + +STATISTIC(NumInternalized, "Number of linkonce_odr functions made internal"); +STATISTIC(NumCloned, "Number of internal clones of linkonce_odr functions"); +STATISTIC(NumCallsRedirected, "Number of direct calls redirected to a clone"); + +enum class LinkOnceODRInternalization { Off, LikelyModuleLocal, All }; + +static cl::opt<LinkOnceODRInternalization> EnableLinkOnceODRInternalization( + "enable-linkonce-odr-internalization", cl::Hidden, + cl::init(LinkOnceODRInternalization::Off), + cl::desc("Clone internal copies of linkonce_odr functions"), + cl::values(clEnumValN(LinkOnceODRInternalization::Off, "off", "never"), + clEnumValN(LinkOnceODRInternalization::LikelyModuleLocal, + "likely-module-local", + "when TU-local hint is present"), + clEnumValN(LinkOnceODRInternalization::All, "all", + "all linkonce_odr functions"))); + +static bool isCandidate(const Function &F) { + if (F.isDeclaration() || !F.hasLinkOnceODRLinkage()) + return false; + return EnableLinkOnceODRInternalization == LinkOnceODRInternalization::All || + F.hasFnAttribute("frontend-hint-likely-module-local"); +} + +static bool isDirectCall(const Use &U) { + const auto *CB = dyn_cast<CallBase>(U.getUser()); + return CB && CB->isCallee(&U); +} + +// Whether a copy of F can be made. Duplicating it must not duplicate anything +// that has to be unique, like labels of block addresses or of inline asm. +static bool isCloneable(const Function &F) { + if (F.hasFnAttribute(Attribute::NoDuplicate)) + return false; + for (const BasicBlock &BB : F) + if (BB.hasAddressTaken()) + return false; + for (const Instruction &I : instructions(F)) + if (const auto *CB = dyn_cast<CallBase>(&I)) + if (CB->isInlineAsm() || CB->cannotDuplicate()) + return false; + return true; +} + +PreservedAnalyses InternalizeLinkOnceODRPass::run(Module &M, + ModuleAnalysisManager &) { + if (EnableLinkOnceODRInternalization == LinkOnceODRInternalization::Off) + return PreservedAnalyses::all(); + + SmallVector<Function *, 16> Candidates; + for (Function &F : M) + if (isCandidate(F)) + Candidates.push_back(&F); + + bool Changed = false; + for (Function *F : Candidates) { + bool HasDirectCalls = false, HasOtherUses = false; + for (const Use &U : F->uses()) { + if (isDirectCall(U)) + HasDirectCalls = true; + else + HasOtherUses = true; + } + if (!HasDirectCalls) + continue; + + // (1) Make local in-place when allowed (only direct calls, alone in comdat) + const Comdat *C = F->getComdat(); + if (!HasOtherUses && (!C || C->getUsers().size() == 1)) { + F->setLinkage(GlobalValue::InternalLinkage); + F->setComdat(nullptr); + ++NumInternalized; + Changed = true; + continue; + } + + if (!isCloneable(*F)) + continue; + + // (2) Clone and redirect (direct calls only) + ValueToValueMapTy VMap; + Function *Clone = CloneFunction(F, VMap); + Clone->setName(F->getName() + ".internal"); + Clone->setLinkage(GlobalValue::InternalLinkage); + Clone->setComdat(nullptr); + Clone->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); + for (Use &U : make_early_inc_range(F->uses())) { + if (isDirectCall(U)) { + U.set(Clone); + ++NumCallsRedirected; + } + } + ++NumCloned; + Changed = true; + } + + return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); +} diff --git a/llvm/test/Transforms/InternalizeLinkOnceODR/basic.ll b/llvm/test/Transforms/InternalizeLinkOnceODR/basic.ll new file mode 100644 index 0000000000000..ef6599d3f8c86 --- /dev/null +++ b/llvm/test/Transforms/InternalizeLinkOnceODR/basic.ll @@ -0,0 +1,82 @@ +; RUN: opt -passes=internalize-linkonce-odr -enable-linkonce-odr-internalization=likely-module-local -S < %s | FileCheck %s --check-prefixes=CHECK,MAIN +; RUN: opt -passes=internalize-linkonce-odr -enable-linkonce-odr-internalization=all -S < %s | FileCheck %s --check-prefixes=CHECK,ALL + +$only_calls = comdat any +$shared = comdat any + +; Only direct calls: made internal in place, removed from its comdat. +; CHECK: define internal i32 @only_calls(i32 %x) #0 { +define linkonce_odr hidden i32 @only_calls(i32 %x) #0 comdat { + %r = add i32 %x, 1 + ret i32 %r +} + +; Address taken: the original stays for that use, all direct calls (including +; the recursive ones in the original and in the clone) go to an internal clone. +; CHECK: define linkonce_odr i32 @address_taken(i32 %x) #0 { +; CHECK-NEXT: call i32 @address_taken.internal( +define linkonce_odr i32 @address_taken(i32 %x) #0 { + %r = call i32 @address_taken(i32 %x) + ret i32 %r +} + +; Shares its comdat with a global: cannot leave the comdat, so it is cloned. +; CHECK: define linkonce_odr i32 @in_shared_comdat(i32 %x) #0 comdat($shared) { +@shared_var = linkonce_odr global i32 0, comdat($shared) +define linkonce_odr i32 @in_shared_comdat(i32 %x) #0 comdat($shared) { + ret i32 %x +} + +; Not marked likely module-local: only with =all. +; MAIN: define linkonce_odr i32 @from_header(i32 %x) { +; ALL: define internal i32 @from_header(i32 %x) { +define linkonce_odr i32 @from_header(i32 %x) { + ret i32 %x +} + +; Not linkonce_odr: untouched. +; CHECK: define weak_odr i32 @explicit_instantiation(i32 %x) #0 { +define weak_odr i32 @explicit_instantiation(i32 %x) #0 { + ret i32 %x +} + +; Address taken and not cloneable (inline asm may define labels): untouched. +; CHECK: define linkonce_odr void @has_asm() #0 { +; CHECK-NOT: @has_asm.internal +define linkonce_odr void @has_asm() #0 { + call void asm sideeffect "nop", ""() + ret void +} + +; No calls: nothing to do. +; CHECK: define linkonce_odr void @never_called() #0 { +define linkonce_odr void @never_called() #0 { + ret void +} + +@fptrs = global [3 x ptr] [ptr @address_taken, ptr @has_asm, ptr @never_called] + +; CHECK-LABEL: define i32 @caller( +; CHECK: call i32 @only_calls( +; CHECK: call i32 @address_taken.internal( +; CHECK: call i32 @in_shared_comdat.internal( +; MAIN: call i32 @from_header( +; ALL: call i32 @from_header( +; CHECK: call i32 @explicit_instantiation( +; CHECK: call void @has_asm( +define i32 @caller(i32 %x) { + %a = call i32 @only_calls(i32 %x) + %b = call i32 @address_taken(i32 %a) + %c = call i32 @in_shared_comdat(i32 %b) + %d = call i32 @from_header(i32 %c) + %e = call i32 @explicit_instantiation(i32 %d) + call void @has_asm() + ret i32 %e +} + +; Clones are appended to the module. +; CHECK: define internal i32 @address_taken.internal(i32 %x) unnamed_addr #0 { +; CHECK-NEXT: call i32 @address_taken.internal( +; CHECK: define internal i32 @in_shared_comdat.internal(i32 %x) unnamed_addr #0 { + +attributes #0 = { "frontend-hint-likely-module-local" } diff --git a/llvm/test/Transforms/InternalizeLinkOnceODR/pipeline.ll b/llvm/test/Transforms/InternalizeLinkOnceODR/pipeline.ll new file mode 100644 index 0000000000000..712171cac32c4 --- /dev/null +++ b/llvm/test/Transforms/InternalizeLinkOnceODR/pipeline.ll @@ -0,0 +1,23 @@ +; With the internal copy, IPSCCP sees every call of @scale and propagates the +; constant argument and the resulting return value into the callers. +; RUN: opt -passes='default<O2>' -S < %s | FileCheck %s --check-prefix=OFF +; RUN: opt -passes='default<O2>' -enable-linkonce-odr-internalization=likely-module-local -S < %s | FileCheck %s --check-prefix=ON + +; OFF-LABEL: define {{.*}}i32 @caller( +; OFF: call i32 @scale(i32 5) +; OFF: call i32 @scale(i32 5) + +; ON-LABEL: define {{.*}}i32 @caller( +; ON-NOT: call +; ON: ret i32 20 +define i32 @caller() { + %a = call i32 @scale(i32 5) + %b = call i32 @scale(i32 5) + %r = add i32 %a, %b + ret i32 %r +} + +define linkonce_odr i32 @scale(i32 %x) noinline "frontend-hint-likely-module-local" { + %r = mul i32 %x, 2 + ret i32 %r +} _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
