Author: Rahman Lavaee Date: 2026-10-08T08:07:33-07:00 New Revision: 3e6acc87c58843e551d6e8e4bf06c26712b04622
URL: https://github.com/llvm/llvm-project/commit/3e6acc87c58843e551d6e8e4bf06c26712b04622 DIFF: https://github.com/llvm/llvm-project/commit/3e6acc87c58843e551d6e8e4bf06c26712b04622.diff LOG: [CodeGen] Add a FunctionSplittingMode target option (#226575) Add TargetOptions::FunctionSplitting and a -function-splitting={none,bbsections,all} flag, replacing TargetOptions::EnableMachineFunctionSplitter and the -split-machine-functions / -enable-split-machine-functions flags. MachineFunctionSplitter runs when the mode is 'all'. clang and flang map -fsplit-machine-functions to 'all', including through the LTO plugin-opt. This is the first step towards merging MachineFunctionSplitter into BasicBlockSections. After that, 'none' and 'bbsections' will fully take effect, so splitting can be turned off for some or all functions whether a bbsections or PGO profile is used. Added: llvm/test/CodeGen/X86/function-splitting-mode.ll Modified: clang/lib/CodeGen/BackendUtil.cpp clang/lib/Driver/ToolChains/CommonArgs.cpp clang/test/Driver/fsplit-machine-functions.c flang/lib/Frontend/CompilerInstance.cpp flang/test/Driver/split-machine-function-pass.f90 llvm/docs/ReleaseNotes.md llvm/include/llvm/CodeGen/CommandFlags.h llvm/include/llvm/Target/CGPassBuilderOption.h llvm/include/llvm/Target/TargetOptions.h llvm/lib/CodeGen/CodeGenOptions.td llvm/lib/CodeGen/CommandFlags.cpp llvm/lib/CodeGen/TargetPassConfig.cpp llvm/test/CodeGen/Generic/machine-function-splitter-optnone.ll llvm/test/CodeGen/Generic/machine-function-splitter.ll llvm/test/CodeGen/X86/basic-block-address-map-with-mfs.ll llvm/test/CodeGen/X86/basic-block-address-map.ll llvm/test/CodeGen/X86/basic-block-sections.ll llvm/test/CodeGen/X86/load-sample-profile-2.ll llvm/test/CodeGen/X86/load-sample-profile.ll llvm/test/tools/llvm-profgen/X86/func-split.test Removed: ################################################################################ diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp index c754c75aa59de5..2cab30fad3d00a 100644 --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -419,7 +419,8 @@ static bool initTargetOptions(const CompilerInstance &CI, Options.BBSectionsFuncListBuf = std::move(*MBOrErr); } - Options.EnableMachineFunctionSplitter = CodeGenOpts.SplitMachineFunctions; + if (CodeGenOpts.SplitMachineFunctions) + Options.FunctionSplitting = llvm::FunctionSplittingMode::All; Options.EnableStaticDataPartitioning = CodeGenOpts.PartitionStaticDataSections; Options.FunctionSections = CodeGenOpts.FunctionSections; diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp index c54a0b1068f437..7c832ef405fa51 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -1303,7 +1303,7 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args, options::OPT_fno_split_machine_functions)) { if (A->getOption().matches(options::OPT_fsplit_machine_functions)) CmdArgs.push_back(Args.MakeArgString(Twine(PluginOptPrefix) + - "-split-machine-functions")); + "-function-splitting=all")); } if (auto *A = diff --git a/clang/test/Driver/fsplit-machine-functions.c b/clang/test/Driver/fsplit-machine-functions.c index 153aafba8fbd00..80485a32db3b66 100644 --- a/clang/test/Driver/fsplit-machine-functions.c +++ b/clang/test/Driver/fsplit-machine-functions.c @@ -14,8 +14,8 @@ // RUN: %clang -### --target=x86_64-linux -flto -fsplit-machine-functions %s 2>&1 | FileCheck %s --check-prefix=LTO // RUN: %clang -### --target=x86_64-linux -flto -fsplit-machine-functions -fno-split-machine-functions %s 2>&1 | FileCheck %s --check-prefix=LTO-NEG -// LTO: "-plugin-opt=-split-machine-functions" -// LTO-NEG-NOT: "-plugin-opt=-split-machine-functions" +// LTO: "-plugin-opt=-function-splitting=all" +// LTO-NEG-NOT: "-plugin-opt=-function-splitting=all" // RUN: not %clang -### -c --target=arm-unknown-linux -fsplit-machine-functions %s 2>&1 | FileCheck %s --check-prefix=ERR // ERR: error: unsupported option '-fsplit-machine-functions' for target diff --git a/flang/lib/Frontend/CompilerInstance.cpp b/flang/lib/Frontend/CompilerInstance.cpp index 1b09fcfd049afb..f63b7cb1580825 100644 --- a/flang/lib/Frontend/CompilerInstance.cpp +++ b/flang/lib/Frontend/CompilerInstance.cpp @@ -380,7 +380,8 @@ bool CompilerInstance::setUpTargetMachine() { std::optional<llvm::CodeModel::Model> cm = getCodeModel(CGOpts.CodeModel); llvm::TargetOptions tOpts = llvm::TargetOptions(); - tOpts.EnableMachineFunctionSplitter = targetOpts.SplitMachineFunctions; + if (targetOpts.SplitMachineFunctions) + tOpts.FunctionSplitting = llvm::FunctionSplittingMode::All; tOpts.VecLib = convertDriverVectorLibraryToVectorLibrary(CGOpts.getVecLib()); tOpts.MCOptions.DisableIntegratedAS = CGOpts.DisableIntegratedAS; tOpts.FunctionSections = CGOpts.FunctionSections; diff --git a/flang/test/Driver/split-machine-function-pass.f90 b/flang/test/Driver/split-machine-function-pass.f90 index cf8bd0eafb6241..b8659321c85d29 100644 --- a/flang/test/Driver/split-machine-function-pass.f90 +++ b/flang/test/Driver/split-machine-function-pass.f90 @@ -1,8 +1,10 @@ -! Verify that the MachineFunctionSplitter pass is enabled while passing -fsplit-machine-functions. +! Verify that the MachineFunctionSplitter pass is enabled while passing +! -fsplit-machine-functions or -mllvm -function-splitting=all. ! REQUIRES: x86-registered-target ! RUN: %flang_fc1 -S -fsplit-machine-functions %s -triple x86_64-unknown-linux-gnu -mllvm -debug-pass=Structure -o /dev/null 2>&1 | FileCheck %s --check-prefix=SPLIT +! RUN: %flang_fc1 -S %s -triple x86_64-unknown-linux-gnu -mllvm -function-splitting=all -mllvm -debug-pass=Structure -o /dev/null 2>&1 | FileCheck %s --check-prefix=SPLIT ! RUN: %flang_fc1 -S %s -triple x86_64-unknown-linux-gnu -mllvm -debug-pass=Structure -o /dev/null 2>&1 | FileCheck %s --check-prefix=NO-SPLIT ! SPLIT: Machine Function Splitter Transformation diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md index 1300ee1eb60d1f..792622a3cc5895 100644 --- a/llvm/docs/ReleaseNotes.md +++ b/llvm/docs/ReleaseNotes.md @@ -181,6 +181,17 @@ Makes programs 10x faster by doing Special New Thing. GNU-vs-EABI distinction is now derived entirely from the target triple's environment (e.g. `arm-none-gnueabi` vs `arm-none-eabi`). +* Replaced `TargetOptions::EnableMachineFunctionSplitter` with + `TargetOptions::FunctionSplitting`, and the `-split-machine-functions` and + `-enable-split-machine-functions` flags with + `-function-splitting={none,bbsections,all}`. + * Backend tools (`llc`, `opt`, LTO plugin options and libLTO debug options): + use `-function-splitting=all` for the previous behavior. + * Clang and Flang: `-fsplit-machine-functions` is unchanged and remains the + recommended spelling. `-mllvm -enable-split-machine-functions` should be + replaced with `-fsplit-machine-functions` or + `-mllvm -function-splitting=all`. + ### Changes to building LLVM * A new `LLVM_ENABLE_LZMA` option (`ON`, `OFF` or `FORCE_ON`; default `ON`) diff --git a/llvm/include/llvm/CodeGen/CommandFlags.h b/llvm/include/llvm/CodeGen/CommandFlags.h index 1a5d44570430fe..4e09a2c720c170 100644 --- a/llvm/include/llvm/CodeGen/CommandFlags.h +++ b/llvm/include/llvm/CodeGen/CommandFlags.h @@ -117,8 +117,6 @@ LLVM_ABI bool getEnableCallGraphSection(); LLVM_ABI bool getEmitCallSiteInfo(); -LLVM_ABI bool getEnableMachineFunctionSplitter(); - LLVM_ABI bool getEnableStaticDataPartitioning(); LLVM_ABI bool getEnableDebugEntryValues(); diff --git a/llvm/include/llvm/Target/CGPassBuilderOption.h b/llvm/include/llvm/Target/CGPassBuilderOption.h index b8bffb9e60209e..b143715baacb37 100644 --- a/llvm/include/llvm/Target/CGPassBuilderOption.h +++ b/llvm/include/llvm/Target/CGPassBuilderOption.h @@ -55,7 +55,6 @@ struct CGPassBuilderOption { bool EnableImplicitNullChecks = false; bool EnableBlockPlacementStats = false; bool EnableGlobalMergeFunc = false; - bool EnableMachineFunctionSplitter = false; bool EnableSinkAndFold = false; bool EnableTailMerge = true; /// Enable LoopTermFold immediately after LSR. diff --git a/llvm/include/llvm/Target/TargetOptions.h b/llvm/include/llvm/Target/TargetOptions.h index 6db8eca394a38c..75e855789dedb3 100644 --- a/llvm/include/llvm/Target/TargetOptions.h +++ b/llvm/include/llvm/Target/TargetOptions.h @@ -50,6 +50,22 @@ enum class BasicBlockSection { None // Do not use Basic Block Sections. }; +/// Late function splitting mode. Decides which functions are eligible to have +/// their cold blocks moved into a separate section. +enum class FunctionSplittingMode { + None, // Hard off. Never create a cold section, even for functions + // which have a basic block sections profile. Such functions + // are still laid out using the profile, but are emitted as a + // single contiguous section. + // NOTE: Not implemented yet. + BBSectionsOnly, // Split only functions that have a basic block sections + // profile. Functions without one are not split, even if + // they have PGO/SamplePGO data. This is the default, and + // matches the behavior without -fsplit-machine-functions. + All // Split functions using the basic block sections profile + // where it is available, and PGO/AutoFDO elsewhere. +}; + /// Identify a debugger for "tuning" the debug info. /// /// The "debugger tuning" concept allows us to present a more intuitive @@ -113,7 +129,6 @@ class TargetOptions { TrapUnreachable(false), NoTrapAfterNoreturn(false), TLSSize(0), EmulatedTLS(false), EnableTLSDESC(false), EnableIPRA(false), EmitStackSizeSection(false), EnableMachineOutliner(false), - EnableMachineFunctionSplitter(false), EnableStaticDataPartitioning(false), EnableDefaultMachineVerifier(true), EmitAddrsig(false), BBAddrMap(false), EmitCallGraphSection(false), EmitCallSiteInfo(false), EnableDebugEntryValues(false), @@ -208,9 +223,6 @@ class TargetOptions { /// Enables the MachineOutliner pass. unsigned EnableMachineOutliner : 1; - /// Enables the MachineFunctionSplitter pass. - unsigned EnableMachineFunctionSplitter : 1; - /// Enables the StaticDataSplitter pass. unsigned EnableStaticDataPartitioning : 1; @@ -228,6 +240,10 @@ class TargetOptions { /// Emit basic blocks into separate sections. BasicBlockSection BBSections = BasicBlockSection::None; + /// Which functions are eligible for late function splitting. + FunctionSplittingMode FunctionSplitting = + FunctionSplittingMode::BBSectionsOnly; + /// Memory Buffer that contains information on sampled basic blocks and used /// to selectively generate basic block sections. std::shared_ptr<MemoryBuffer> BBSectionsFuncListBuf; diff --git a/llvm/lib/CodeGen/CodeGenOptions.td b/llvm/lib/CodeGen/CodeGenOptions.td index 3a5c4bd5471fa0..2084cbae77346d 100644 --- a/llvm/lib/CodeGen/CodeGenOptions.td +++ b/llvm/lib/CodeGen/CodeGenOptions.td @@ -151,8 +151,11 @@ defm : BoolField<"enable-implicit-null-checks", "false", "Fold null checks into faulting memory operations">; defm : OptionalBoolField<"enable-ipra", "Enable interprocedural register allocation to reduce load/store at procedure calls">; -defm : BoolField<"enable-split-machine-functions", "false", - "Split out cold blocks from machine functions based on profile information">; +defm : EnumField<"function-splitting", "std::optional<FunctionSplittingMode>", "std::nullopt", + "none,bbsections,all", + ["FunctionSplittingMode::None", "FunctionSplittingMode::BBSectionsOnly", + "FunctionSplittingMode::All"], + "Which functions are eligible for late function splitting">; // -fast-isel selects FastISel at any optimization level, and -fast-isel=0 // disables it at -O0, where it is used unless GlobalISel is enabled. defm : OptionalBoolField<"fast-isel", diff --git a/llvm/lib/CodeGen/CommandFlags.cpp b/llvm/lib/CodeGen/CommandFlags.cpp index 1bebecda621664..46a294f3524b94 100644 --- a/llvm/lib/CodeGen/CommandFlags.cpp +++ b/llvm/lib/CodeGen/CommandFlags.cpp @@ -104,7 +104,6 @@ CGOPT(bool, EnableStackSizeSection) CGOPT(bool, EnableAddrsig) CGOPT(bool, EnableCallGraphSection) CGOPT(bool, EmitCallSiteInfo) -CGOPT(bool, EnableMachineFunctionSplitter) CGOPT(bool, EnableStaticDataPartitioning) CGOPT(bool, EnableDebugEntryValues) CGOPT(bool, ForceDwarfFrameSection) @@ -432,13 +431,6 @@ codegen::RegisterCodeGenFlags::RegisterCodeGenFlags() { cl::init(false)); CGBINDOPT(EnableDebugEntryValues); - static cl::opt<bool> EnableMachineFunctionSplitter( - "split-machine-functions", - cl::desc("Split out cold basic blocks from machine functions based on " - "profile information"), - cl::init(false)); - CGBINDOPT(EnableMachineFunctionSplitter); - static cl::opt<bool> EnableStaticDataPartitioning( "partition-static-data-sections", cl::desc("Partition data sections using profile information."), @@ -550,7 +542,6 @@ codegen::InitTargetOptionsFromCodeGenFlags(const Triple &TheTriple) { Options.ExceptionModel = getExceptionModel(); Options.VecLib = getVectorLibrary(); Options.EmitStackSizeSection = getEnableStackSizeSection(); - Options.EnableMachineFunctionSplitter = getEnableMachineFunctionSplitter(); Options.EnableStaticDataPartitioning = getEnableStaticDataPartitioning(); Options.EmitAddrsig = getEnableAddrsig(); Options.EmitCallGraphSection = getEnableCallGraphSection(); diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp index 6ee2ac359ae897..44846365580eed 100644 --- a/llvm/lib/CodeGen/TargetPassConfig.cpp +++ b/llvm/lib/CodeGen/TargetPassConfig.cpp @@ -347,7 +347,6 @@ CGPassBuilderOption llvm::getCGPassBuilderOption() { toBoolOrDefault(Opts.debugify_check_and_strip_all_safe); Opt.DisableRAFSProfileLoader = Opts.disable_ra_fsprofile_loader; Opt.DisableCFIFixup = Opts.disable_cfi_fixup; - Opt.EnableMachineFunctionSplitter = Opts.enable_split_machine_functions; return Opt; } @@ -450,6 +449,9 @@ TargetPassConfig::TargetPassConfig(TargetMachine &TM, PassManagerBase &PM) if (Opts.global_isel_abort) TM.Options.GlobalISelAbort = *Opts.global_isel_abort; + if (Opts.function_splitting) + TM.Options.FunctionSplitting = *Opts.function_splitting; + setStartStopPasses(); } @@ -1090,8 +1092,10 @@ void TargetPassConfig::addMachinePasses() { addPass(createMIRAddFSDiscriminatorsPass( sampleprof::FSDiscriminatorPass::PassLast)); - if (TM->Options.EnableMachineFunctionSplitter || - Opts.enable_split_machine_functions || Opts.split_static_data || + const bool SplitFunctions = + TM->Options.FunctionSplitting == FunctionSplittingMode::All; + + if (SplitFunctions || Opts.split_static_data || TM->Options.EnableStaticDataPartitioning) { const std::string ProfileFile = getFSProfileFile(TM); if (!ProfileFile.empty()) { @@ -1114,8 +1118,7 @@ void TargetPassConfig::addMachinePasses() { // feature takes precedence. This means functions eligible for // basic-block-sections optimizations (`=all`, or `=list=` with function // included in the list profile) will get that optimization instead. - if (TM->Options.EnableMachineFunctionSplitter || - Opts.enable_split_machine_functions) + if (SplitFunctions) addPass(createMachineFunctionSplitterPass()); if (Opts.split_static_data || TM->Options.EnableStaticDataPartitioning) { diff --git a/llvm/test/CodeGen/Generic/machine-function-splitter-optnone.ll b/llvm/test/CodeGen/Generic/machine-function-splitter-optnone.ll index 67d2ad72ee2f43..2f53e64adf48df 100644 --- a/llvm/test/CodeGen/Generic/machine-function-splitter-optnone.ll +++ b/llvm/test/CodeGen/Generic/machine-function-splitter-optnone.ll @@ -1,6 +1,6 @@ ; REQUIRES: x86-registered-target -; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -split-machine-functions -O0 -mfs-psi-cutoff=0 -mfs-count-threshold=10000 | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -function-splitting=all -O0 -mfs-psi-cutoff=0 -mfs-count-threshold=10000 | FileCheck %s ;; Check that functions with optnone attribute are not split. ; CHECK-LABEL: foo_optnone: diff --git a/llvm/test/CodeGen/Generic/machine-function-splitter.ll b/llvm/test/CodeGen/Generic/machine-function-splitter.ll index d798b2875645b4..cf8abe3a536b29 100644 --- a/llvm/test/CodeGen/Generic/machine-function-splitter.ll +++ b/llvm/test/CodeGen/Generic/machine-function-splitter.ll @@ -2,11 +2,11 @@ ; REQUIRES: x86-registered-target ; COM: Machine function splitting with FDO profiles -; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -split-machine-functions | FileCheck %s -check-prefixes=MFS-DEFAULTS,MFS-DEFAULTS-X86,MFS-NOBBSECTIONS -; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -split-machine-functions -mfs-psi-cutoff=0 -mfs-count-threshold=2000 | FileCheck %s --dump-input=always -check-prefixes=MFS-OPTS1,MFS-OPTS1-X86 -; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -split-machine-functions -mfs-psi-cutoff=950000 | FileCheck %s -check-prefixes=MFS-OPTS2,MFS-OPTS2-X86 -; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -split-machine-functions -mfs-split-ehcode | FileCheck %s -check-prefixes=MFS-EH-SPLIT,MFS-EH-SPLIT-X86 -; RUN: llc < %s -mtriple=x86_64 -split-machine-functions -O0 -mfs-psi-cutoff=0 -mfs-count-threshold=10000 | FileCheck %s -check-prefixes=MFS-O0,MFS-O0-X86 +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -function-splitting=all | FileCheck %s -check-prefixes=MFS-DEFAULTS,MFS-DEFAULTS-X86,MFS-NOBBSECTIONS +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -function-splitting=all -mfs-psi-cutoff=0 -mfs-count-threshold=2000 | FileCheck %s --dump-input=always -check-prefixes=MFS-OPTS1,MFS-OPTS1-X86 +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -function-splitting=all -mfs-psi-cutoff=950000 | FileCheck %s -check-prefixes=MFS-OPTS2,MFS-OPTS2-X86 +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -function-splitting=all -mfs-split-ehcode | FileCheck %s -check-prefixes=MFS-EH-SPLIT,MFS-EH-SPLIT-X86 +; RUN: llc < %s -mtriple=x86_64 -function-splitting=all -O0 -mfs-psi-cutoff=0 -mfs-count-threshold=10000 | FileCheck %s -check-prefixes=MFS-O0,MFS-O0-X86 ; COM: Machine function splitting along with -basic-block-sections profile ; RUN: echo 'v1' > %t @@ -15,19 +15,20 @@ ; RUN: echo 'ffoo22' >> %t ; RUN: echo 'c0 1' >> %t ; RUN: echo 'c2' >> %t -; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -basic-block-sections=%t -split-machine-functions | FileCheck %s --check-prefixes=MFS-BBSECTIONS +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -basic-block-sections=%t -function-splitting=all | FileCheck %s --check-prefixes=MFS-BBSECTIONS,MFS-BBSECTIONS-ALL +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -basic-block-sections=%t -function-splitting=bbsections | FileCheck %s --check-prefixes=MFS-BBSECTIONS,MFS-BBSECTIONS-ONLY -; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu -aarch64-min-jump-table-entries=4 -enable-split-machine-functions | FileCheck %s -check-prefixes=MFS-DEFAULTS,MFS-DEFAULTS-AARCH64 -; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu -aarch64-min-jump-table-entries=4 -enable-split-machine-functions -mfs-psi-cutoff=0 -mfs-count-threshold=2000 | FileCheck %s --dump-input=always -check-prefixes=MFS-OPTS1,MFS-OPTS1-AARCH64 -; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu -aarch64-min-jump-table-entries=4 -enable-split-machine-functions -mfs-psi-cutoff=950000 | FileCheck %s -check-prefixes=MFS-OPTS2,MFS-OPTS2-AARCH64 -; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu -aarch64-min-jump-table-entries=4 -enable-split-machine-functions -mfs-split-ehcode | FileCheck %s -check-prefixes=MFS-EH-SPLIT,MFS-EH-SPLIT-AARCH64 -; RUN: llc < %s -mtriple=aarch64 -split-machine-functions -O0 -mfs-psi-cutoff=0 -mfs-count-threshold=10000 | FileCheck %s -check-prefixes=MFS-O0,MFS-O0-AARCH64 -; RUN: llc < %s -mtriple=aarch64 -enable-split-machine-functions -aarch64-redzone | FileCheck %s -check-prefixes=MFS-REDZONE-AARCH64 +; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu -aarch64-min-jump-table-entries=4 -function-splitting=all | FileCheck %s -check-prefixes=MFS-DEFAULTS,MFS-DEFAULTS-AARCH64 +; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu -aarch64-min-jump-table-entries=4 -function-splitting=all -mfs-psi-cutoff=0 -mfs-count-threshold=2000 | FileCheck %s --dump-input=always -check-prefixes=MFS-OPTS1,MFS-OPTS1-AARCH64 +; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu -aarch64-min-jump-table-entries=4 -function-splitting=all -mfs-psi-cutoff=950000 | FileCheck %s -check-prefixes=MFS-OPTS2,MFS-OPTS2-AARCH64 +; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu -aarch64-min-jump-table-entries=4 -function-splitting=all -mfs-split-ehcode | FileCheck %s -check-prefixes=MFS-EH-SPLIT,MFS-EH-SPLIT-AARCH64 +; RUN: llc < %s -mtriple=aarch64 -function-splitting=all -O0 -mfs-psi-cutoff=0 -mfs-count-threshold=10000 | FileCheck %s -check-prefixes=MFS-O0,MFS-O0-AARCH64 +; RUN: llc < %s -mtriple=aarch64 -function-splitting=all -aarch64-redzone | FileCheck %s -check-prefixes=MFS-REDZONE-AARCH64 ; COM: Machine function splitting with AFDO profiles ; RUN: sed 's/InstrProf/SampleProfile/g' %s > %t.ll -; RUN: llc < %t.ll -mtriple=x86_64-unknown-linux-gnu -split-machine-functions | FileCheck %s --check-prefix=FSAFDO-MFS -; RUN: llc < %t.ll -mtriple=x86_64-unknown-linux-gnu -split-machine-functions | FileCheck %s --check-prefix=FSAFDO-MFS2 +; RUN: llc < %t.ll -mtriple=x86_64-unknown-linux-gnu -function-splitting=all | FileCheck %s --check-prefix=FSAFDO-MFS +; RUN: llc < %t.ll -mtriple=x86_64-unknown-linux-gnu -function-splitting=all | FileCheck %s --check-prefix=FSAFDO-MFS2 define void @foo1(i1 zeroext %0) nounwind !prof !14 !section_prefix !15 { ;; Check that cold block is moved to .text.split. @@ -38,6 +39,15 @@ define void @foo1(i1 zeroext %0) nounwind !prof !14 !section_prefix !15 { ; MFS-DEFAULTS-X86-NEXT: callq baz ; MFS-DEFAULTS-AARCH64-NOT: bl bar ; MFS-DEFAULTS-AARCH64-NEXT: bl baz +;; Check that with a basic-block-sections profile, a function outside the +;; profile is still split using PGO with -function-splitting=all. +; MFS-BBSECTIONS-ALL-LABEL: foo1: +; MFS-BBSECTIONS-ALL: .section .text.split.foo1 +; MFS-BBSECTIONS-ALL-NEXT: foo1.cold: +;; Check that with -function-splitting=bbsections, a function outside the +;; basic-block-sections profile is not split, even though it has PGO data. +; MFS-BBSECTIONS-ONLY-LABEL: foo1: +; MFS-BBSECTIONS-ONLY-NOT: foo1.cold: br i1 %0, label %2, label %4, !prof !17 2: ; preds = %1 diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-with-mfs.ll b/llvm/test/CodeGen/X86/basic-block-address-map-with-mfs.ll index f8146bb38f71eb..a03c2ff9ef3d4c 100644 --- a/llvm/test/CodeGen/X86/basic-block-address-map-with-mfs.ll +++ b/llvm/test/CodeGen/X86/basic-block-address-map-with-mfs.ll @@ -1,8 +1,8 @@ ; COM: Emitting basic-block-address-map when machine function splitting is enabled. -; RUN: llc < %s -mtriple=x86_64 -function-sections -split-machine-functions -basic-block-address-map | FileCheck %s --check-prefixes=CHECK,BASIC +; RUN: llc < %s -mtriple=x86_64 -function-sections -function-splitting=all -basic-block-address-map | FileCheck %s --check-prefixes=CHECK,BASIC ; COM: Emitting basic-block-address-map with PGO analysis with machine function splitting enabled. -; RUN: llc < %s -mtriple=x86_64 -function-sections -split-machine-functions -basic-block-address-map -pgo-analysis-map=func-entry-count,bb-freq,br-prob | FileCheck %s --check-prefixes=CHECK,PGO +; RUN: llc < %s -mtriple=x86_64 -function-sections -function-splitting=all -basic-block-address-map -pgo-analysis-map=func-entry-count,bb-freq,br-prob | FileCheck %s --check-prefixes=CHECK,PGO define void @foo(i1 zeroext %0) nounwind !prof !14 { br i1 %0, label %2, label %4, !prof !15 diff --git a/llvm/test/CodeGen/X86/basic-block-address-map.ll b/llvm/test/CodeGen/X86/basic-block-address-map.ll index c716e12a576462..21d9066d2a7db8 100644 --- a/llvm/test/CodeGen/X86/basic-block-address-map.ll +++ b/llvm/test/CodeGen/X86/basic-block-address-map.ll @@ -1,7 +1,7 @@ ; Check the basic block sections labels option ; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map | FileCheck %s --check-prefixes=CHECK,ELF,UNIQ ; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=false -basic-block-address-map | FileCheck %s --check-prefixes=CHECK,ELF,NOUNIQ -; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map -split-machine-functions | FileCheck %s --check-prefixes=CHECK,ELF,UNIQ +; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map -function-splitting=all | FileCheck %s --check-prefixes=CHECK,ELF,UNIQ ; RUN: llc < %s -mtriple=x86_64-pc-windows-msvc -function-sections -basic-block-address-map | FileCheck %s --check-prefixes=CHECK,COFF define void @_Z3bazb(i1 zeroext, i1 zeroext) personality ptr @__gxx_personality_v0 { diff --git a/llvm/test/CodeGen/X86/basic-block-sections.ll b/llvm/test/CodeGen/X86/basic-block-sections.ll index 8e50416554a0a5..fd1db1e29ef392 100644 --- a/llvm/test/CodeGen/X86/basic-block-sections.ll +++ b/llvm/test/CodeGen/X86/basic-block-sections.ll @@ -1,6 +1,6 @@ ; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=all -unique-basic-block-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS ; RUN: llc < %s -mtriple=x86_64-pc-linux -basic-block-sections=all -unique-basic-block-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS -; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=all -unique-basic-block-section-names -split-machine-functions | FileCheck %s -check-prefix=LINUX-SECTIONS +; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=all -unique-basic-block-section-names -function-splitting=all | FileCheck %s -check-prefix=LINUX-SECTIONS ; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -function-sections -basic-block-sections=all -unique-basic-block-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS ; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -basic-block-sections=all -unique-basic-block-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS diff --git a/llvm/test/CodeGen/X86/function-splitting-mode.ll b/llvm/test/CodeGen/X86/function-splitting-mode.ll new file mode 100644 index 00000000000000..6fc1f3ced84335 --- /dev/null +++ b/llvm/test/CodeGen/X86/function-splitting-mode.ll @@ -0,0 +1,61 @@ +;; Tests for the -function-splitting= late function splitting mode. +; REQUIRES: x86-registered-target + +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -function-splitting=all | FileCheck %s --check-prefix=SPLIT + +;; A function which only has PGO data is not split unless the mode is 'all'. +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --check-prefix=NOSPLIT +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -function-splitting=none | FileCheck %s --check-prefix=NOSPLIT +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -function-splitting=bbsections | FileCheck %s --check-prefix=NOSPLIT + +;; Invalid modes are rejected. +; RUN: not llc < %s -mtriple=x86_64-unknown-linux-gnu -function-splitting=bogus 2>&1 | FileCheck %s --check-prefix=ERR +; ERR: invalid value 'bogus' in '-function-splitting=bogus' + +define void @foo(i1 zeroext %0) nounwind !prof !14 !section_prefix !15 { +; SPLIT-LABEL: foo +; SPLIT: .section .text.split.foo +; SPLIT-NEXT: foo.cold: +; SPLIT-NOT: callq bar +; SPLIT-NEXT: callq baz +; +; NOSPLIT-LABEL: foo +; NOSPLIT-NOT: .section .text.split.foo +; NOSPLIT-NOT: foo.cold: + br i1 %0, label %2, label %4, !prof !17 + +2: ; preds = %1 + %3 = call i32 @bar() + br label %6 + +4: ; preds = %1 + %5 = call i32 @baz() + br label %6 + +6: ; preds = %4, %2 + %7 = tail call i32 @qux() + ret void +} + +declare i32 @bar() +declare i32 @baz() +declare i32 @qux() + +!llvm.module.flags = !{!0} +!0 = !{i32 1, !"ProfileSummary", !1} +!1 = !{!2, !3, !4, !5, !6, !7, !8, !9} +!2 = !{!"ProfileFormat", !"InstrProf"} +!3 = !{!"TotalCount", i64 10000} +!4 = !{!"MaxCount", i64 10} +!5 = !{!"MaxInternalCount", i64 1} +!6 = !{!"MaxFunctionCount", i64 1000} +!7 = !{!"NumCounts", i64 3} +!8 = !{!"NumFunctions", i64 5} +!9 = !{!"DetailedSummary", !10} +!10 = !{!11, !12, !13} +!11 = !{i32 10000, i64 100, i32 1} +!12 = !{i32 999900, i64 100, i32 1} +!13 = !{i32 999999, i64 1, i32 2} +!14 = !{!"function_entry_count", i64 7000} +!15 = !{!"function_section_prefix", !"hot"} +!17 = !{!"branch_weights", i32 7000, i32 0} diff --git a/llvm/test/CodeGen/X86/load-sample-profile-2.ll b/llvm/test/CodeGen/X86/load-sample-profile-2.ll index f608d22843d568..76cc02e7b35c04 100644 --- a/llvm/test/CodeGen/X86/load-sample-profile-2.ll +++ b/llvm/test/CodeGen/X86/load-sample-profile-2.ll @@ -1,5 +1,5 @@ ;;; MFS with sample profile fails when no -enable-fs-discriminator=true. -; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -fs-profile-file=%S/Inputs/fsloader-mfs.afdo -split-machine-functions -enable-fs-discriminator=false 2>&1 | FileCheck %s --check-prefix=NODISCRIMINATOR +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -fs-profile-file=%S/Inputs/fsloader-mfs.afdo -function-splitting=all -enable-fs-discriminator=false 2>&1 | FileCheck %s --check-prefix=NODISCRIMINATOR ; NODISCRIMINATOR: warning: Using AutoFDO without FSDiscriminator for MFS may regress performance. define void @foo4(i1 zeroext %0, i1 zeroext %1) nounwind { diff --git a/llvm/test/CodeGen/X86/load-sample-profile.ll b/llvm/test/CodeGen/X86/load-sample-profile.ll index 2e3f8809d3882e..3ac1f39b17e388 100644 --- a/llvm/test/CodeGen/X86/load-sample-profile.ll +++ b/llvm/test/CodeGen/X86/load-sample-profile.ll @@ -1,6 +1,6 @@ ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -debug-pass=Structure -enable-fs-discriminator=true -improved-fs-discriminator=true 2>&1 | FileCheck %s --check-prefix=NOPROFILE ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -debug-pass=Structure -fs-profile-file=%S/Inputs/fsloader-mfs.afdo -enable-fs-discriminator=true -improved-fs-discriminator=true 2>&1 | FileCheck %s --check-prefix=PROFILE-NOMFS -; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -debug-pass=Structure -fs-profile-file=%S/Inputs/fsloader-mfs.afdo -split-machine-functions -enable-fs-discriminator=true -improved-fs-discriminator=true 2>&1 | FileCheck %s --check-prefix=PROFILE-MFS +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -debug-pass=Structure -fs-profile-file=%S/Inputs/fsloader-mfs.afdo -function-splitting=all -enable-fs-discriminator=true -improved-fs-discriminator=true 2>&1 | FileCheck %s --check-prefix=PROFILE-MFS ;; No profile is specified, no load passes. ; NOPROFILE: Add FS discriminators in MIR diff --git a/llvm/test/tools/llvm-profgen/X86/func-split.test b/llvm/test/tools/llvm-profgen/X86/func-split.test index 734f39ea898ca1..3393da2e52dcb1 100644 --- a/llvm/test/tools/llvm-profgen/X86/func-split.test +++ b/llvm/test/tools/llvm-profgen/X86/func-split.test @@ -30,7 +30,7 @@ ; clang -g -O3 -fdebug-info-for-profiling func-split.c -mllvm -mfs-count-threshold=0 -; -fprofile-sample-use=profile.txt -fno-inline -mllvm --enable-split-machine-functions=1 +; -fprofile-sample-use=profile.txt -fno-inline -fsplit-machine-functions #include <stdio.h> _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
