https://github.com/KRM7 created https://github.com/llvm/llvm-project/pull/233380
Enable speculative single-target devirtualization with LTO (disabled by default, can be enabled using -fdevirtualize-speculatively). The DevirtSpeculatively member variable is also removed from WPD, so that the optimization is controlled entirely by the -devirtualize-speculatively flag. The -enable-devirtualize-speculatively flag is now only used to add the WPD pass to the non-LTO pipeline. >From 4be403204bd31c7d32502f9fe57817a2884625fe Mon Sep 17 00:00:00 2001 From: Krisztian Rugasi <[email protected]> Date: Thu, 15 Oct 2026 10:03:19 +0000 Subject: [PATCH] [WPD] Enable speculative devirtualization with LTO Enable speculative single-target devirtualization with LTO (disabled by default, can be enabled using -fdevirtualize-speculatively). The DevirtSpeculatively member variable is also removed from WPD, so that the optimization is controlled entirely by the -devirtualize-speculatively flag. The -enable-devirtualize-speculatively flag is now only used to add the WPD pass to the non-LTO pipeline. --- clang/docs/UsersManual.md | 2 - clang/lib/Driver/ToolChains/CommonArgs.cpp | 8 + clang/test/Driver/lto.c | 8 + llvm/include/llvm/AsmParser/LLToken.h | 1 + llvm/include/llvm/IR/ModuleSummaryIndex.h | 1 + llvm/include/llvm/IR/ModuleSummaryIndexYAML.h | 2 + .../llvm/Transforms/IPO/WholeProgramDevirt.h | 7 +- llvm/lib/AsmParser/LLLexer.cpp | 1 + llvm/lib/AsmParser/LLParser.cpp | 3 + llvm/lib/IR/AsmWriter.cpp | 5 +- llvm/lib/Passes/PassBuilderPipelines.cpp | 3 +- llvm/lib/Passes/PassesOptions.td | 2 +- .../lib/Transforms/IPO/WholeProgramDevirt.cpp | 143 +++++++++--------- llvm/test/Assembler/thinlto-summary.ll | 4 +- llvm/test/ThinLTO/X86/devirt_speculative.ll | 71 +++++++++ .../X86/devirt_speculative_mixed_vis.ll | 105 +++++++++++++ .../devirt-single-impl-check-cfi.ll | 68 +++++++++ .../speculative-devirt-cfi.ll | 68 +++++++++ .../speculative-devirt-mixed-vis.ll | 77 ++++++++++ .../speculative-devirt-single-impl.ll | 14 +- 20 files changed, 505 insertions(+), 88 deletions(-) create mode 100644 llvm/test/ThinLTO/X86/devirt_speculative.ll create mode 100644 llvm/test/ThinLTO/X86/devirt_speculative_mixed_vis.ll create mode 100644 llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check-cfi.ll create mode 100644 llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-cfi.ll create mode 100644 llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-mixed-vis.ll diff --git a/clang/docs/UsersManual.md b/clang/docs/UsersManual.md index 96d1c4e009c71a..2c2c131b656e03 100644 --- a/clang/docs/UsersManual.md +++ b/clang/docs/UsersManual.md @@ -2589,8 +2589,6 @@ if.end.icp: ; preds = %if.false.orig_indir ret void ``` -This feature is temporarily ignored at the LLVM side when LTO is enabled. -TODO: Update the comment when the LLVM side supports this feature for LTO. This feature is turned off by default. ::: diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp index 7c832ef405fa51..329f5317ea47a9 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -1222,6 +1222,14 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args, (A->getOption().matches(options::OPT_fglobal_isel) ? "1" : "0"))); } + if (Arg *A = Args.getLastArg(options::OPT_fdevirtualize_speculatively, + options::OPT_fno_devirtualize_speculatively)) + CmdArgs.push_back(Args.MakeArgString( + Twine(PluginOptPrefix) + "-devirtualize-speculatively=" + + (A->getOption().matches(options::OPT_fdevirtualize_speculatively) + ? "1" + : "0"))); + // If an explicit debugger tuning argument appeared, pass it along. if (Arg *A = Args.getLastArg(options::OPT_gTune_Group, options::OPT_ggdbN_Group)) { diff --git a/clang/test/Driver/lto.c b/clang/test/Driver/lto.c index 838eb740863752..be18231f5d335c 100644 --- a/clang/test/Driver/lto.c +++ b/clang/test/Driver/lto.c @@ -127,6 +127,14 @@ // CHECK-NO-SLP: "-plugin-opt=-vectorize-slp=0" // CHECK-SLP: "-plugin-opt=-vectorize-slp=1" +// RUN: %clang --target=x86_64-unknown-linux -### %s -flto -fno-devirtualize-speculatively 2> %t +// RUN: FileCheck --check-prefix=CHECK-NO-DEVIRTUALIZE-SPECULATIVELY < %t %s +// RUN: %clang --target=x86_64-unknown-linux -### %s -flto -fdevirtualize-speculatively 2> %t +// RUN: FileCheck --check-prefix=CHECK-DEVIRTUALIZE-SPECULATIVELY < %t %s +// +// CHECK-NO-DEVIRTUALIZE-SPECULATIVELY: "-plugin-opt=-devirtualize-speculatively=0" +// CHECK-DEVIRTUALIZE-SPECULATIVELY: "-plugin-opt=-devirtualize-speculatively=1" + // -flto passes -time-passes when -ftime-report is passed // RUN: %clang --target=x86_64-unknown-linux-gnu -### %s -flto -ftime-report 2> %t // RUN: FileCheck --check-prefix=CHECK-TIME-REPORT < %t %s diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h index 2679a630c2dd95..e5816ca301527e 100644 --- a/llvm/include/llvm/AsmParser/LLToken.h +++ b/llvm/include/llvm/AsmParser/LLToken.h @@ -467,6 +467,7 @@ enum Kind { kw_indir, kw_singleImpl, kw_branchFunnel, + kw_speculative, kw_singleImplName, kw_resByArg, kw_byArg, diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h index 4f68582b65c4eb..f9d6640f50852a 100644 --- a/llvm/include/llvm/IR/ModuleSummaryIndex.h +++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h @@ -1349,6 +1349,7 @@ struct WholeProgramDevirtResolution { BranchFunnel, ///< When retpoline mitigation is enabled, use a branch funnel ///< that is defined in the merged module. Otherwise same as ///< Indir. + Speculative, ///< Speculative devirtualization } TheKind = Indir; std::string SingleImplName; diff --git a/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h b/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h index 158c03e77306c9..441433eb381adc 100644 --- a/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h +++ b/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h @@ -102,6 +102,8 @@ template <> struct ScalarEnumerationTraits<WholeProgramDevirtResolution::Kind> { io.enumCase(value, "SingleImpl", WholeProgramDevirtResolution::SingleImpl); io.enumCase(value, "BranchFunnel", WholeProgramDevirtResolution::BranchFunnel); + io.enumCase(value, "Speculative", + WholeProgramDevirtResolution::Speculative); } }; diff --git a/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h b/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h index c3a394ab4fb720..0dd61bbd41a4ab 100644 --- a/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h +++ b/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h @@ -227,14 +227,11 @@ struct WholeProgramDevirtPass ModuleSummaryIndex *ExportSummary; const ModuleSummaryIndex *ImportSummary; bool UseCommandLine = false; - bool DevirtSpeculatively = false; WholeProgramDevirtPass() : ExportSummary(nullptr), ImportSummary(nullptr), UseCommandLine(true) {} WholeProgramDevirtPass(ModuleSummaryIndex *ExportSummary, - const ModuleSummaryIndex *ImportSummary, - bool DevirtSpeculatively = false) - : ExportSummary(ExportSummary), ImportSummary(ImportSummary), - DevirtSpeculatively(DevirtSpeculatively) { + const ModuleSummaryIndex *ImportSummary) + : ExportSummary(ExportSummary), ImportSummary(ImportSummary) { assert(!(ExportSummary && ImportSummary)); } LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &); diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp index 5e52dc8dcb2478..20212fc26451aa 100644 --- a/llvm/lib/AsmParser/LLLexer.cpp +++ b/llvm/lib/AsmParser/LLLexer.cpp @@ -884,6 +884,7 @@ lltok::Kind LLLexer::LexIdentifier() { KEYWORD(indir); KEYWORD(singleImpl); KEYWORD(branchFunnel); + KEYWORD(speculative); KEYWORD(singleImplName); KEYWORD(resByArg); KEYWORD(byArg); diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp index 60b7824342aa38..03a266931632cf 100644 --- a/llvm/lib/AsmParser/LLParser.cpp +++ b/llvm/lib/AsmParser/LLParser.cpp @@ -10047,6 +10047,9 @@ bool LLParser::parseWpdRes(WholeProgramDevirtResolution &WPDRes) { case lltok::kw_branchFunnel: WPDRes.TheKind = WholeProgramDevirtResolution::BranchFunnel; break; + case lltok::kw_speculative: + WPDRes.TheKind = WholeProgramDevirtResolution::Speculative; + break; default: return error(Lex.getLoc(), "unexpected WholeProgramDevirtResolution kind"); } diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp index 1cbcfb195764b3..a30ff85e33160b 100644 --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -3402,6 +3402,8 @@ getWholeProgDevirtResKindName(WholeProgramDevirtResolution::Kind K) { return "singleImpl"; case WholeProgramDevirtResolution::BranchFunnel: return "branchFunnel"; + case WholeProgramDevirtResolution::Speculative: + return "speculative"; } llvm_unreachable("invalid WholeProgramDevirtResolution kind"); } @@ -3496,7 +3498,8 @@ void AssemblyWriter::printWPDRes(const WholeProgramDevirtResolution &WPDRes) { Out << "wpdRes: (kind: "; Out << getWholeProgDevirtResKindName(WPDRes.TheKind); - if (WPDRes.TheKind == WholeProgramDevirtResolution::SingleImpl) + if (WPDRes.TheKind == WholeProgramDevirtResolution::SingleImpl || + WPDRes.TheKind == WholeProgramDevirtResolution::Speculative) Out << ", singleImplName: \"" << WPDRes.SingleImplName << "\""; if (!WPDRes.ResByArg.empty()) { diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index 125c8e442f27b6..b4ce758e314e94 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -1620,8 +1620,7 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level, MPM.addPass(AssignGUIDPass()); MPM.addPass(WholeProgramDevirtPass( /*ExportSummary*/ nullptr, - /*ImportSummary*/ nullptr, - /*DevirtSpeculatively*/ PTO.DevirtualizeSpeculatively)); + /*ImportSummary*/ nullptr)); MPM.addPass(DropTypeTestsPass()); // Given that the devirtualization creates more opportunities for inlining, // we run the Inliner again here to maximize the optimization gain we diff --git a/llvm/lib/Passes/PassesOptions.td b/llvm/lib/Passes/PassesOptions.td index 4398d5c06bff2b..d37930c7ca39d8 100644 --- a/llvm/lib/Passes/PassesOptions.td +++ b/llvm/lib/Passes/PassesOptions.td @@ -38,7 +38,7 @@ defm : BoolField<"enable-constraint-elimination", "true", // TODO: There is a similar flag in WPD pass, we should consolidate them by // parsing the option only once in PassBuilder and share it across both places. defm : BoolField<"enable-devirtualize-speculatively", "false", - "Enable speculative devirtualization optimization">; + "Enable speculative devirtualization without LTO">; defm : BoolField<"enable-dfa-jump-thread", "true", "Enable DFA jump threading">; defm : BoolField<"enable-global-analyses", "true", diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp index 88c57d2195bbda..ba0718882d77e0 100644 --- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp +++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp @@ -160,8 +160,6 @@ static cl::opt<std::string> ClWriteSummary( "bitcode, otherwise YAML"), cl::Hidden); -// TODO: This option eventually should support any public visibility vtables -// with/out LTO. static cl::opt<bool> ClDevirtualizeSpeculatively( "devirtualize-speculatively", cl::desc("Enable speculative devirtualization optimization"), @@ -611,11 +609,9 @@ struct DevirtModule { std::map<CallInst *, unsigned> NumUnsafeUsesForTypeTest; PatternList FunctionsToSkip; - const bool DevirtSpeculatively; DevirtModule(Module &M, ModuleAnalysisManager &MAM, ModuleSummaryIndex *ExportSummary, - const ModuleSummaryIndex *ImportSummary, - bool DevirtSpeculatively) + const ModuleSummaryIndex *ImportSummary) : M(M), MAM(MAM), FAM(MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager()), ExportSummary(ExportSummary), ImportSummary(ImportSummary), @@ -628,8 +624,7 @@ struct DevirtModule { RemarksEnabled(areRemarksEnabled()), OREGetter([&](Function &F) -> OptimizationRemarkEmitter & { return FAM.getResult<OptimizationRemarkEmitterAnalysis>(F); - }), - DevirtSpeculatively(DevirtSpeculatively) { + }) { assert(!(ExportSummary && ImportSummary)); FunctionsToSkip.init(SkipFunctionNames); } @@ -645,18 +640,18 @@ struct DevirtModule { std::vector<VTableBits> &Bits, DenseMap<Metadata *, std::set<TypeMemberInfo>> &TypeIdMap); - bool - tryFindVirtualCallTargets(std::vector<VirtualCallTarget> &TargetsForSlot, - const std::set<TypeMemberInfo> &TypeMemberInfos, - uint64_t ByteOffset, - ModuleSummaryIndex *ExportSummary); + bool tryFindVirtualCallTargets( + std::vector<VirtualCallTarget> &TargetsForSlot, + const std::set<TypeMemberInfo> &TypeMemberInfos, uint64_t ByteOffset, + ModuleSummaryIndex *ExportSummary, bool &HasPublicTarget); void applySingleImplDevirt(VTableSlotInfo &SlotInfo, Constant *TheFn, - bool &IsExported); + bool HasPublicTarget, bool &IsExported); bool trySingleImplDevirt(ModuleSummaryIndex *ExportSummary, MutableArrayRef<VirtualCallTarget> TargetsForSlot, VTableSlotInfo &SlotInfo, - WholeProgramDevirtResolution *Res); + WholeProgramDevirtResolution *Res, + bool HasPublicTarget); void applyICallBranchFunnel(VTableSlotInfo &SlotInfo, Function &JT, bool &IsExported); @@ -743,8 +738,7 @@ struct DevirtModule { // Lower the module using the action and summary passed as command line // arguments. For testing purposes only. - static bool runForTesting(Module &M, ModuleAnalysisManager &MAM, - bool DevirtSpeculatively); + static bool runForTesting(Module &M, ModuleAnalysisManager &MAM); }; struct DevirtIndex { @@ -779,13 +773,14 @@ struct DevirtIndex { bool tryFindVirtualCallTargets(std::vector<ValueInfo> &TargetsForSlot, const TypeIdCompatibleVtableInfo TIdInfo, - uint64_t ByteOffset); + uint64_t ByteOffset, bool &HasPublicTarget); bool trySingleImplDevirt(MutableArrayRef<ValueInfo> TargetsForSlot, VTableSlotSummary &SlotSummary, VTableSlotInfo &SlotInfo, WholeProgramDevirtResolution *Res, - std::set<ValueInfo> &DevirtTargets); + std::set<ValueInfo> &DevirtTargets, + bool HasPublicTarget); void run(); }; @@ -794,22 +789,17 @@ struct DevirtIndex { PreservedAnalyses WholeProgramDevirtPass::run(Module &M, ModuleAnalysisManager &MAM) { if (UseCommandLine) { - if (!DevirtModule::runForTesting(M, MAM, ClDevirtualizeSpeculatively)) + if (!DevirtModule::runForTesting(M, MAM)) return PreservedAnalyses::all(); return PreservedAnalyses::none(); } std::optional<ModuleSummaryIndex> Index; - if (!ExportSummary && !ImportSummary && DevirtSpeculatively) { - // Build the ExportSummary from the module. - assert(!ExportSummary && - "ExportSummary is expected to be empty in non-LTO mode"); + if (!ExportSummary && !ImportSummary && ClDevirtualizeSpeculatively) { ProfileSummaryInfo PSI(M); - Index.emplace(buildModuleSummaryIndex(M, nullptr, &PSI)); - ExportSummary = Index.has_value() ? &Index.value() : nullptr; + ExportSummary = &Index.emplace(buildModuleSummaryIndex(M, nullptr, &PSI)); } - if (!DevirtModule(M, MAM, ExportSummary, ImportSummary, DevirtSpeculatively) - .run()) + if (!DevirtModule(M, MAM, ExportSummary, ImportSummary).run()) return PreservedAnalyses::all(); return PreservedAnalyses::none(); } @@ -907,9 +897,7 @@ void llvm::updatePublicTypeTestCalls(Module &M, CI->replaceAllUsesWith(NewCI); CI->eraseFromParent(); } - } else { - // TODO: Don't replace public type tests when speculative devirtualization - // gets enabled in LTO mode. + } else if (!ClDevirtualizeSpeculatively) { auto *True = ConstantInt::getTrue(M.getContext()); for (Use &U : make_early_inc_range(PublicTypeTestFunc->uses())) { auto *CI = cast<CallInst>(U.getUser()); @@ -1014,8 +1002,7 @@ static Error checkCombinedSummaryForTesting(ModuleSummaryIndex *Summary) { return ErrorSuccess(); } -bool DevirtModule::runForTesting(Module &M, ModuleAnalysisManager &MAM, - bool DevirtSpeculatively) { +bool DevirtModule::runForTesting(Module &M, ModuleAnalysisManager &MAM) { std::unique_ptr<ModuleSummaryIndex> Summary = std::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/false); @@ -1044,8 +1031,7 @@ bool DevirtModule::runForTesting(Module &M, ModuleAnalysisManager &MAM, ClSummaryAction == PassSummaryAction::Export ? Summary.get() : nullptr, ClSummaryAction == PassSummaryAction::Import ? Summary.get() - : nullptr, - DevirtSpeculatively) + : nullptr) .run(); if (!ClWriteSummary.empty()) { @@ -1104,16 +1090,19 @@ void DevirtModule::buildTypeIdentifierMap( bool DevirtModule::tryFindVirtualCallTargets( std::vector<VirtualCallTarget> &TargetsForSlot, const std::set<TypeMemberInfo> &TypeMemberInfos, uint64_t ByteOffset, - ModuleSummaryIndex *ExportSummary) { + ModuleSummaryIndex *ExportSummary, bool &HasPublicTarget) { + HasPublicTarget = false; + for (const TypeMemberInfo &TM : TypeMemberInfos) { if (!TM.Bits->GV->isConstant()) return false; - // Without DevirtSpeculatively, we cannot perform whole program - // devirtualization analysis on a vtable with public LTO visibility. - if (!DevirtSpeculatively && TM.Bits->GV->getVCallVisibility() == - GlobalObject::VCallVisibilityPublic) - return false; + if (TM.Bits->GV->getVCallVisibility() == + GlobalObject::VCallVisibilityPublic) { + HasPublicTarget = true; + if (!ClDevirtualizeSpeculatively) + return false; + } Function *Fn = nullptr; Constant *C = nullptr; @@ -1131,12 +1120,6 @@ bool DevirtModule::tryFindVirtualCallTargets( if (Fn->getName() == "__cxa_pure_virtual") continue; - // In most cases empty functions will be overridden by the - // implementation of the derived class, so we can skip them. - if (DevirtSpeculatively && Fn->getReturnType()->isVoidTy() && - Fn->getInstructionCount() <= 1) - continue; - // We can disregard unreachable functions as possible call targets, as // unreachable functions shouldn't be called. if (mustBeUnreachableFunction(Fn, ExportSummary)) @@ -1158,7 +1141,10 @@ bool DevirtModule::tryFindVirtualCallTargets( bool DevirtIndex::tryFindVirtualCallTargets( std::vector<ValueInfo> &TargetsForSlot, - const TypeIdCompatibleVtableInfo TIdInfo, uint64_t ByteOffset) { + const TypeIdCompatibleVtableInfo TIdInfo, uint64_t ByteOffset, + bool &HasPublicTarget) { + HasPublicTarget = false; + for (const TypeIdOffsetVtableInfo &P : TIdInfo) { // Find a representative copy of the vtable initializer. // We can have multiple available_externally, linkonce_odr and weak_odr @@ -1187,8 +1173,11 @@ bool DevirtIndex::tryFindVirtualCallTargets( VS = CurVS; // We cannot perform whole program devirtualization analysis on a vtable // with public LTO visibility. - if (VS->getVCallVisibility() == GlobalObject::VCallVisibilityPublic) - return false; + if (VS->getVCallVisibility() == GlobalObject::VCallVisibilityPublic) { + HasPublicTarget = true; + if (!ClDevirtualizeSpeculatively) + return false; + } break; } } @@ -1214,7 +1203,8 @@ bool DevirtIndex::tryFindVirtualCallTargets( } void DevirtModule::applySingleImplDevirt(VTableSlotInfo &SlotInfo, - Constant *TheFn, bool &IsExported) { + Constant *TheFn, bool HasPublicTarget, + bool &IsExported) { // Don't devirtualize function if we're told to skip it // in -wholeprogramdevirt-skip. if (FunctionsToSkip.match(TheFn->stripPointerCasts()->getName())) @@ -1257,7 +1247,7 @@ void DevirtModule::applySingleImplDevirt(VTableSlotInfo &SlotInfo, // add support to compare the virtual function pointer to the // devirtualized target. In case of a mismatch, fall back to indirect // call. - if (DevirtCheckMode == WPDCheckMode::Fallback || DevirtSpeculatively) { + if (DevirtCheckMode == WPDCheckMode::Fallback || HasPublicTarget) { MDNode *Weights = MDBuilder(M.getContext()).createLikelyBranchWeights(); // Version the indirect call site. If the called value is equal to the // given callee, 'NewInst' will be executed, otherwise the original call @@ -1278,6 +1268,7 @@ void DevirtModule::applySingleImplDevirt(VTableSlotInfo &SlotInfo, // In either trapping or non-checking mode, devirtualize original call. else { // Devirtualize unconditionally. + assert(!HasPublicTarget); CB.setCalledOperand(Callee); // Since the call site is now direct, we must clear metadata that // is only appropriate for indirect calls. This includes !prof and @@ -1295,7 +1286,7 @@ void DevirtModule::applySingleImplDevirt(VTableSlotInfo &SlotInfo, } // This use is no longer unsafe. - if (VCallSite.NumUnsafeUses) + if (!HasPublicTarget && VCallSite.NumUnsafeUses) --*VCallSite.NumUnsafeUses; } if (CSInfo.isExported()) @@ -1338,7 +1329,7 @@ static bool addCalls(VTableSlotInfo &SlotInfo, const ValueInfo &Callee) { bool DevirtModule::trySingleImplDevirt( ModuleSummaryIndex *ExportSummary, MutableArrayRef<VirtualCallTarget> TargetsForSlot, VTableSlotInfo &SlotInfo, - WholeProgramDevirtResolution *Res) { + WholeProgramDevirtResolution *Res, bool HasPublicTarget) { // See if the program contains a single implementation of this virtual // function. auto *TheFn = TargetsForSlot[0].Fn; @@ -1351,7 +1342,7 @@ bool DevirtModule::trySingleImplDevirt( TargetsForSlot[0].WasDevirt = true; bool IsExported = false; - applySingleImplDevirt(SlotInfo, TheFn, IsExported); + applySingleImplDevirt(SlotInfo, TheFn, HasPublicTarget, IsExported); if (!IsExported) return false; @@ -1383,7 +1374,11 @@ bool DevirtModule::trySingleImplDevirt( // LTO unit split, so we can ignore return value of AddCalls. addCalls(SlotInfo, TheFnVI); - Res->TheKind = WholeProgramDevirtResolution::SingleImpl; + if (HasPublicTarget) + Res->TheKind = WholeProgramDevirtResolution::Speculative; + else + Res->TheKind = WholeProgramDevirtResolution::SingleImpl; + Res->SingleImplName = std::string(TheFn->getName()); return true; @@ -1393,7 +1388,8 @@ bool DevirtIndex::trySingleImplDevirt(MutableArrayRef<ValueInfo> TargetsForSlot, VTableSlotSummary &SlotSummary, VTableSlotInfo &SlotInfo, WholeProgramDevirtResolution *Res, - std::set<ValueInfo> &DevirtTargets) { + std::set<ValueInfo> &DevirtTargets, + bool HasPublicTarget) { // See if the program contains a single implementation of this virtual // function. auto TheFn = TargetsForSlot[0]; @@ -1427,7 +1423,11 @@ bool DevirtIndex::trySingleImplDevirt(MutableArrayRef<ValueInfo> TargetsForSlot, // Record in summary for use in devirtualization during the ThinLTO import // step. - Res->TheKind = WholeProgramDevirtResolution::SingleImpl; + if (HasPublicTarget) + Res->TheKind = WholeProgramDevirtResolution::Speculative; + else + Res->TheKind = WholeProgramDevirtResolution::SingleImpl; + if (GlobalValue::isLocalLinkage(S->linkage())) { if (IsExported) { // If target is a local function and we are exporting it by @@ -2318,7 +2318,8 @@ void DevirtModule::importResolution(VTableSlot Slot, VTableSlotInfo &SlotInfo) { return; const WholeProgramDevirtResolution &Res = ResI->second; - if (Res.TheKind == WholeProgramDevirtResolution::SingleImpl) { + if (Res.TheKind == WholeProgramDevirtResolution::SingleImpl || + Res.TheKind == WholeProgramDevirtResolution::Speculative) { assert(!Res.SingleImplName.empty()); // The type of the function in the declaration is irrelevant because every // call site will cast it to the correct type. @@ -2333,7 +2334,9 @@ void DevirtModule::importResolution(VTableSlot Slot, VTableSlotInfo &SlotInfo) { // This is the import phase so we should not be exporting anything. bool IsExported = false; - applySingleImplDevirt(SlotInfo, SingleImpl, IsExported); + bool IsSpeculative = + Res.TheKind == WholeProgramDevirtResolution::Speculative; + applySingleImplDevirt(SlotInfo, SingleImpl, IsSpeculative, IsExported); assert(!IsExported); } @@ -2445,7 +2448,7 @@ bool DevirtModule::run() { Function *PublicTypeTestFunc = nullptr; // If we are in speculative devirtualization mode, we can work on the public // type test intrinsics. - if (DevirtSpeculatively) + if (ClDevirtualizeSpeculatively) PublicTypeTestFunc = Intrinsic::getDeclarationIfExists(&M, Intrinsic::public_type_test); Function *TypeTestFunc = @@ -2574,14 +2577,14 @@ bool DevirtModule::run() { ->getOrInsertTypeIdSummary( cast<MDString>(S.first.TypeID)->getString()) .WPDRes[S.first.ByteOffset]; + + bool HasPublicTarget = true; if (tryFindVirtualCallTargets(TargetsForSlot, TypeMemberInfos, - S.first.ByteOffset, ExportSummary)) { - bool SingleImplDevirt = - trySingleImplDevirt(ExportSummary, TargetsForSlot, S.second, Res); - // Out of speculative devirtualization mode, Try to apply virtual constant - // propagation or branch funneling. - // TODO: This should eventually be enabled for non-public type tests. - if (!SingleImplDevirt && !DevirtSpeculatively) { + S.first.ByteOffset, ExportSummary, + HasPublicTarget)) { + bool SingleImplDevirt = trySingleImplDevirt( + ExportSummary, TargetsForSlot, S.second, Res, HasPublicTarget); + if (!SingleImplDevirt && !HasPublicTarget) { DidVirtualConstProp |= tryVirtualConstProp(TargetsForSlot, S.second, Res, S.first); @@ -2725,11 +2728,13 @@ void DevirtIndex::run() { WholeProgramDevirtResolution *Res = &ExportSummary.getTypeIdSummary(S.first.TypeID) ->WPDRes[S.first.ByteOffset]; + + bool HasPublicTarget = true; if (tryFindVirtualCallTargets(TargetsForSlot, *TidSummary, - S.first.ByteOffset)) { + S.first.ByteOffset, HasPublicTarget)) { if (!trySingleImplDevirt(TargetsForSlot, S.first, S.second, Res, - DevirtTargets)) + DevirtTargets, HasPublicTarget)) continue; } } diff --git a/llvm/test/Assembler/thinlto-summary.ll b/llvm/test/Assembler/thinlto-summary.ll index 5b907f9a06039a..31c2aa25024e11 100644 --- a/llvm/test/Assembler/thinlto-summary.ll +++ b/llvm/test/Assembler/thinlto-summary.ll @@ -70,7 +70,7 @@ ^28 = typeid: (name: "_ZTS1B", summary: (typeTestRes: (kind: inline, sizeM1BitWidth: 0, alignLog2: 1, sizeM1: 2, bitMask: 3, inlineBits: 4))) ; Test the AllOnes resolution, and all kinds of WholeProgramDevirtResolution ; types, including all optional resolution by argument kinds. -^29 = typeid: (name: "_ZTS1A", summary: (typeTestRes: (kind: allOnes, sizeM1BitWidth: 7), wpdResolutions: ((offset: 0, wpdRes: (kind: branchFunnel)), (offset: 8, wpdRes: (kind: singleImpl, singleImplName: "_ZN1A1nEi")), (offset: 16, wpdRes: (kind: indir, resByArg: (args: (1, 2), byArg: (kind: indir, byte: 2, bit: 3), args: (3), byArg: (kind: uniformRetVal, info: 1), args: (4), byArg: (kind: uniqueRetVal, info: 1), args: (5), byArg: (kind: virtualConstProp))))))) +^29 = typeid: (name: "_ZTS1A", summary: (typeTestRes: (kind: allOnes, sizeM1BitWidth: 7), wpdResolutions: ((offset: 0, wpdRes: (kind: branchFunnel)), (offset: 8, wpdRes: (kind: singleImpl, singleImplName: "_ZN1A1nEi")), (offset: 16, wpdRes: (kind: speculative, singleImplName: "_ZN1A1mEi")), (offset: 24, wpdRes: (kind: indir, resByArg: (args: (1, 2), byArg: (kind: indir, byte: 2, bit: 3), args: (3), byArg: (kind: uniformRetVal, info: 1), args: (4), byArg: (kind: uniqueRetVal, info: 1), args: (5), byArg: (kind: virtualConstProp))))))) ; Test the other kinds of type test resoultions ^30 = typeid: (name: "_ZTS1D", summary: (typeTestRes: (kind: byteArray, sizeM1BitWidth: 0))) ^31 = typeid: (name: "_ZTS1E", summary: (typeTestRes: (kind: unsat, sizeM1BitWidth: 0))) @@ -109,7 +109,7 @@ ; CHECK: ^26 = gv: (guid: 9123456789101112131, summaries: (function: (module: ^0, flags: (linkage: internal, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0, importType: definition, noRenameOnPromotion: 0), insts: 1))) ; CHECK: ^27 = typeid: (name: "_ZTS1C", summary: (typeTestRes: (kind: single, sizeM1BitWidth: 0))) ; guid = 1884921850105019584 ; CHECK: ^28 = typeid: (name: "_ZTS1B", summary: (typeTestRes: (kind: inline, sizeM1BitWidth: 0, alignLog2: 1, sizeM1: 2, bitMask: 3, inlineBits: 4))) ; guid = 6203814149063363976 -; CHECK: ^29 = typeid: (name: "_ZTS1A", summary: (typeTestRes: (kind: allOnes, sizeM1BitWidth: 7), wpdResolutions: ((offset: 0, wpdRes: (kind: branchFunnel)), (offset: 8, wpdRes: (kind: singleImpl, singleImplName: "_ZN1A1nEi")), (offset: 16, wpdRes: (kind: indir, resByArg: (args: (1, 2), byArg: (kind: indir, byte: 2, bit: 3), args: (3), byArg: (kind: uniformRetVal, info: 1), args: (4), byArg: (kind: uniqueRetVal, info: 1), args: (5), byArg: (kind: virtualConstProp))))))) ; guid = 7004155349499253778 +; CHECK: ^29 = typeid: (name: "_ZTS1A", summary: (typeTestRes: (kind: allOnes, sizeM1BitWidth: 7), wpdResolutions: ((offset: 0, wpdRes: (kind: branchFunnel)), (offset: 8, wpdRes: (kind: singleImpl, singleImplName: "_ZN1A1nEi")), (offset: 16, wpdRes: (kind: speculative, singleImplName: "_ZN1A1mEi")), (offset: 24, wpdRes: (kind: indir, resByArg: (args: (1, 2), byArg: (kind: indir, byte: 2, bit: 3), args: (3), byArg: (kind: uniformRetVal, info: 1), args: (4), byArg: (kind: uniqueRetVal, info: 1), args: (5), byArg: (kind: virtualConstProp))))))) ; guid = 7004155349499253778 ; CHECK: ^30 = typeid: (name: "_ZTS1D", summary: (typeTestRes: (kind: byteArray, sizeM1BitWidth: 0))) ; guid = 9614786172484273522 ; CHECK: ^31 = typeid: (name: "_ZTS1E", summary: (typeTestRes: (kind: unsat, sizeM1BitWidth: 0))) ; guid = 17437243864166745132 ; CHECK: ^32 = flags: 8 diff --git a/llvm/test/ThinLTO/X86/devirt_speculative.ll b/llvm/test/ThinLTO/X86/devirt_speculative.ll new file mode 100644 index 00000000000000..2edd0deb924a92 --- /dev/null +++ b/llvm/test/ThinLTO/X86/devirt_speculative.ll @@ -0,0 +1,71 @@ +; REQUIRES: x86-registered-target + +; Test that speculative devirtualization works with LTO + +; Index based WPD +; RUN: opt -passes=assign-guid -thinlto-bc -o %t2.o %s +; RUN: llvm-lto2 run %t2.o -devirtualize-speculatively -save-temps -pass-remarks=. \ +; RUN: -o %t3 \ +; RUN: -r=%t2.o,test,px \ +; RUN: -r=%t2.o,_ZN1A1fEv,p \ +; RUN: -r=%t2.o,_ZTV1A,px 2>&1 | FileCheck %s --check-prefix=REMARK +; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s + +; Hybrid WPD +; RUN: opt -passes=assign-guid -thinlto-bc -thinlto-split-lto-unit -o %t2.o %s +; RUN: llvm-lto2 run %t2.o -devirtualize-speculatively -save-temps -pass-remarks=. \ +; RUN: -o %t3 \ +; RUN: -r=%t2.o,test,px \ +; RUN: -r=%t2.o,_ZN1A1fEv \ +; RUN: -r=%t2.o,_ZN1A1fEv,p \ +; RUN: -r=%t2.o,_ZTV1A \ +; RUN: -r=%t2.o,_ZTV1A,px 2>&1 | FileCheck %s --check-prefix=REMARK +; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s + +; Regular LTO WPD +; RUN: opt -passes=assign-guid -o %t4.o %s +; RUN: llvm-lto2 run %t4.o -devirtualize-speculatively -save-temps -pass-remarks=. \ +; RUN: -o %t5 \ +; RUN: -r=%t4.o,test,px \ +; RUN: -r=%t4.o,_ZN1A1fEv,p \ +; RUN: -r=%t4.o,_ZTV1A,px 2>&1 | FileCheck %s --check-prefix=REMARK +; RUN: llvm-dis %t5.0.4.opt.bc -o - | FileCheck %s + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-grtev4-linux-gnu" + +; REMARK: <unknown>:0:0: single-impl: devirtualized a call to _ZN1A1fEv + +%struct.A = type { ptr } + +@_ZTV1A = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN1A1fEv] }, !type !0, !vcall_visibility !1 + +define i32 @test(ptr %p) { +entry: + ; CHECK-LABEL: define i32 @test( + ; CHECK-NEXT: [[ENTRY:.*]]: + ; CHECK: [[ICMP:%.*]] = icmp eq ptr [[FPTR:%.*]], @_ZN1A1fEv + ; CHECK-NEXT: br i1 [[ICMP]], label %[[IF_END:.*]], label %[[IF_FALSE_ORIG_INDIRECT:.*]], + ; CHECK: [[IF_FALSE_ORIG_INDIRECT]]: + ; CHECK-NEXT: [[TMP0:%.*]] = tail call i32 [[FPTR]] + ; CHECK-NEXT: br label %[[IF_END]] + ; CHECK: [[IF_END]]: + ; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ [[TMP0]], %[[IF_FALSE_ORIG_INDIRECT]] ], [ 12, %[[ENTRY]] ] + ; CHECK-NEXT: ret i32 [[TMP1]] + %0 = load ptr, ptr %p + %1 = call i1 @llvm.public.type.test(ptr %0, metadata !"_ZTS1A") + call void @llvm.assume(i1 %1) + %2 = load ptr, ptr %0 + %3 = call i32 %2(ptr %p) + ret i32 %3 +} + +define i32 @_ZN1A1fEv(ptr %this) { +entry: + ret i32 12 +} + +declare i1 @llvm.public.type.test(ptr, metadata) + +!0 = !{i64 16, !"_ZTS1A"} +!1 = !{i64 0} diff --git a/llvm/test/ThinLTO/X86/devirt_speculative_mixed_vis.ll b/llvm/test/ThinLTO/X86/devirt_speculative_mixed_vis.ll new file mode 100644 index 00000000000000..bbb237043180b9 --- /dev/null +++ b/llvm/test/ThinLTO/X86/devirt_speculative_mixed_vis.ll @@ -0,0 +1,105 @@ +; REQUIRES: x86-registered-target + +; Test that speculative and non-speculative single-impl devirtualization work +; correctly together based on vcall_visibilities. + +; Index based WPD +; RUN: opt -passes=assign-guid -thinlto-bc -o %t2.o %s +; RUN: llvm-lto2 run %t2.o -devirtualize-speculatively -save-temps -pass-remarks=. \ +; RUN: -o %t3 \ +; RUN: -r=%t2.o,test_speculative,px \ +; RUN: -r=%t2.o,test_nonspeculative,px \ +; RUN: -r=%t2.o,_ZN1A1fEv,p \ +; RUN: -r=%t2.o,_ZN1A1gEv,p \ +; RUN: -r=%t2.o,_ZTV1A,px \ +; RUN: -r=%t2.o,_ZTV1B,px 2>&1 | FileCheck %s --check-prefix=REMARK +; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s + +; Hybrid WPD +; RUN: opt -passes=assign-guid -thinlto-bc -thinlto-split-lto-unit -o %t2.o %s +; RUN: llvm-lto2 run %t2.o -devirtualize-speculatively -save-temps -pass-remarks=. \ +; RUN: -o %t3 \ +; RUN: -r=%t2.o,test_speculative,px \ +; RUN: -r=%t2.o,test_nonspeculative,px \ +; RUN: -r=%t2.o,_ZN1A1fEv \ +; RUN: -r=%t2.o,_ZN1A1gEv \ +; RUN: -r=%t2.o,_ZN1A1fEv,p \ +; RUN: -r=%t2.o,_ZN1A1gEv,p \ +; RUN: -r=%t2.o,_ZTV1A \ +; RUN: -r=%t2.o,_ZTV1B \ +; RUN: -r=%t2.o,_ZTV1A,px \ +; RUN: -r=%t2.o,_ZTV1B,px 2>&1 | FileCheck %s --check-prefix=REMARK +; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s + +; Regular LTO WPD +; RUN: opt -passes=assign-guid -o %t4.o %s +; RUN: llvm-lto2 run %t4.o -devirtualize-speculatively -save-temps -pass-remarks=. \ +; RUN: -o %t5 \ +; RUN: -r=%t4.o,test_speculative,px \ +; RUN: -r=%t4.o,test_nonspeculative,px \ +; RUN: -r=%t4.o,_ZN1A1fEv,p \ +; RUN: -r=%t4.o,_ZN1A1gEv,p \ +; RUN: -r=%t4.o,_ZTV1A,px \ +; RUN: -r=%t4.o,_ZTV1B,px 2>&1 | FileCheck %s --check-prefix=REMARK +; RUN: llvm-dis %t5.0.4.opt.bc -o - | FileCheck %s + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-grtev4-linux-gnu" + +; REMARK-DAG: <unknown>:0:0: single-impl: devirtualized a call to _ZN1A1fEv +; REMARK-DAG: <unknown>:0:0: single-impl: devirtualized a call to _ZN1A1gEv + +%struct.A = type { ptr } +%struct.B = type { ptr } + +@_ZTV1A = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN1A1fEv] }, !type !0, !vcall_visibility !2 +@_ZTV1B = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN1A1gEv] }, !type !1, !vcall_visibility !3 + +define i32 @test_speculative(ptr %p) { +entry: + ; CHECK-LABEL: define i32 @test_speculative( + ; CHECK-NEXT: [[ENTRY1:.*]]: + ; CHECK: [[ICMP:%.*]] = icmp eq ptr [[FPTR:%.*]], @_ZN1A1fEv + ; CHECK-NEXT: br i1 [[ICMP]], label %[[IF_END:.*]], label %[[IF_FALSE_ORIG_INDIRECT:.*]], + ; CHECK: [[IF_FALSE_ORIG_INDIRECT]]: + ; CHECK-NEXT: [[TMP0:%.*]] = tail call i32 [[FPTR]] + ; CHECK-NEXT: br label %[[IF_END]] + ; CHECK: [[IF_END]]: + ; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ [[TMP0]], %[[IF_FALSE_ORIG_INDIRECT]] ], [ 12, %[[ENTRY1]] ] + ; CHECK-NEXT: ret i32 [[TMP1]] + %0 = load ptr, ptr %p + %1 = call i1 @llvm.public.type.test(ptr %0, metadata !"_ZTS1A") + call void @llvm.assume(i1 %1) + %2 = load ptr, ptr %0 + %3 = call i32 %2(ptr %p) + ret i32 %3 +} + +define i32 @test_nonspeculative(ptr %p) { +entry: + ; CHECK-LABEL: define {{.*}} i32 @test_nonspeculative( + ; CHECK: ret i32 23 + %0 = load ptr, ptr %p + %1 = call i1 @llvm.public.type.test(ptr %0, metadata !"_ZTS1B") + call void @llvm.assume(i1 %1) + %2 = load ptr, ptr %0 + %3 = call i32 %2(ptr %p) + ret i32 %3 +} + +define i32 @_ZN1A1fEv(ptr %this) { +entry: + ret i32 12 +} + +define i32 @_ZN1A1gEv(ptr %this) { +entry: + ret i32 23 +} + +declare i1 @llvm.public.type.test(ptr, metadata) + +!0 = !{i64 16, !"_ZTS1A"} +!1 = !{i64 16, !"_ZTS1B"} +!2 = !{i64 0} +!3 = !{i64 1} diff --git a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check-cfi.ll b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check-cfi.ll new file mode 100644 index 00000000000000..80081d145d1aa0 --- /dev/null +++ b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check-cfi.ll @@ -0,0 +1,68 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt -S -passes=wholeprogramdevirt -wholeprogramdevirt-check=fallback -whole-program-visibility -pass-remarks=wholeprogramdevirt %s 2>&1 | FileCheck %s + +; Check whether the type.test intrinsic is removed after devirtualization when +; the fallback mode is used for WPD. + +target datalayout = "e-p:64:64" +target triple = "x86_64-unknown-linux-gnu" + +; CHECK: remark: <unknown>:0:0: single-impl: devirtualized a call to _ZN1A1fEv + +%struct.A = type { ptr } + +@_ZTV1A = linkonce_odr hidden constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN1A1fEv] }, !type !0, !vcall_visibility !1 + +define i32 @test(ptr %p) { +; CHECK-LABEL: define i32 @test( +; CHECK-SAME: ptr [[P:%.*]]) !guid [[META2:![0-9]+]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[P]], align 8 +; CHECK-NEXT: [[VTABLE:%.*]] = load ptr, ptr [[TMP0]], align 8 +; CHECK-NEXT: br i1 true, label %[[CONT:.*]], label %[[TRAP:.*]] +; CHECK: [[TRAP]]: +; CHECK-NEXT: call void @llvm.ubsantrap(i8 2) +; CHECK-NEXT: unreachable +; CHECK: [[CONT]]: +; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[VTABLE]], i32 0 +; CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP1]], align 8 +; CHECK-NEXT: [[TMP4:%.*]] = icmp eq ptr [[TMP2]], @_ZN1A1fEv +; CHECK-NEXT: br i1 [[TMP4]], label %[[IF_TRUE_DIRECT_TARG:.*]], label %[[IF_FALSE_ORIG_INDIRECT:.*]], !prof [[PROF3:![0-9]+]] +; CHECK: [[IF_TRUE_DIRECT_TARG]]: +; CHECK-NEXT: [[CALL:%.*]] = call i32 @_ZN1A1fEv(ptr [[TMP0]]) +; CHECK-NEXT: br label %[[IF_END_ICP:.*]] +; CHECK: [[IF_FALSE_ORIG_INDIRECT]]: +; CHECK-NEXT: [[CALL1:%.*]] = call i32 [[TMP2]](ptr [[TMP0]]) +; CHECK-NEXT: br label %[[IF_END_ICP]] +; CHECK: [[IF_END_ICP]]: +; CHECK-NEXT: [[TMP6:%.*]] = phi i32 [ [[CALL1]], %[[IF_FALSE_ORIG_INDIRECT]] ], [ [[CALL]], %[[IF_TRUE_DIRECT_TARG]] ] +; CHECK-NEXT: ret i32 [[TMP6]] +; +entry: + %0 = load ptr, ptr %p + %vtable = load ptr, ptr %0 + %1 = call { ptr, i1 } @llvm.type.checked.load(ptr %vtable, i32 0, metadata !"_ZTS1A") + %2 = extractvalue { ptr, i1 } %1, 1 + br i1 %2, label %cont, label %trap + +trap: + call void @llvm.ubsantrap(i8 2) + unreachable + +cont: + %3 = extractvalue { ptr, i1 } %1, 0 + %call = call i32 %3(ptr %0) + ret i32 %call +} + +declare i32 @_ZN1A1fEv(ptr %this) + +declare { ptr, i1 } @llvm.type.checked.load(ptr, i32, metadata) +declare void @llvm.ubsantrap(i8 immarg) + +!0 = !{i64 16, !"_ZTS1A"} +!1 = !{i64 1} +;. +; CHECK: [[META2]] = !{i64 8346051122425466633} +; CHECK: [[PROF3]] = !{!"branch_weights", i32 1048575, i32 1} +;. diff --git a/llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-cfi.ll b/llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-cfi.ll new file mode 100644 index 00000000000000..bc13a884f5a033 --- /dev/null +++ b/llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-cfi.ll @@ -0,0 +1,68 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt -S -passes=wholeprogramdevirt -devirtualize-speculatively -pass-remarks=wholeprogramdevirt %s 2>&1 | FileCheck %s + +; Check that the type.test intrinsic is not removed after speculative +; devirtualization, since the indirect call will not be eliminated in this case. + +target datalayout = "e-p:64:64" +target triple = "x86_64-unknown-linux-gnu" + +; CHECK: remark: <unknown>:0:0: single-impl: devirtualized a call to _ZN1A1fEv + +%struct.A = type { ptr } + +@_ZTV1A = linkonce_odr hidden constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN1A1fEv] }, !type !0, !vcall_visibility !1 + +define i32 @test(ptr %p) { +; CHECK-LABEL: define i32 @test( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[P]], align 8 +; CHECK-NEXT: [[VTABLE:%.*]] = load ptr, ptr [[TMP0]], align 8 +; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.type.test(ptr [[VTABLE]], metadata !"_ZTS1A") +; CHECK-NEXT: br i1 [[TMP3]], label %[[CONT:.*]], label %[[TRAP:.*]] +; CHECK: [[TRAP]]: +; CHECK-NEXT: call void @llvm.ubsantrap(i8 2) +; CHECK-NEXT: unreachable +; CHECK: [[CONT]]: +; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[VTABLE]], i32 0 +; CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP1]], align 8 +; CHECK-NEXT: [[TMP4:%.*]] = icmp eq ptr [[TMP2]], @_ZN1A1fEv +; CHECK-NEXT: br i1 [[TMP4]], label %[[IF_TRUE_DIRECT_TARG:.*]], label %[[IF_FALSE_ORIG_INDIRECT:.*]], !prof [[PROF1:![0-9]+]] +; CHECK: [[IF_TRUE_DIRECT_TARG]]: +; CHECK-NEXT: [[CALL:%.*]] = call i32 @_ZN1A1fEv(ptr [[TMP0]]) +; CHECK-NEXT: br label %[[IF_END_ICP:.*]] +; CHECK: [[IF_FALSE_ORIG_INDIRECT]]: +; CHECK-NEXT: [[CALL1:%.*]] = call i32 [[TMP2]](ptr [[TMP0]]) +; CHECK-NEXT: br label %[[IF_END_ICP]] +; CHECK: [[IF_END_ICP]]: +; CHECK-NEXT: [[TMP6:%.*]] = phi i32 [ [[CALL1]], %[[IF_FALSE_ORIG_INDIRECT]] ], [ [[CALL]], %[[IF_TRUE_DIRECT_TARG]] ] +; CHECK-NEXT: ret i32 [[TMP6]] +; +entry: + %vptr = load ptr, ptr %p, align 8 + %vtable = load ptr, ptr %vptr, align 8 + %1 = call { ptr, i1 } @llvm.type.checked.load(ptr %vtable, i32 0, metadata !"_ZTS1A") + %2 = extractvalue { ptr, i1 } %1, 1 + br i1 %2, label %cont, label %trap + +trap: + call void @llvm.ubsantrap(i8 2) + unreachable + +cont: + %3 = extractvalue { ptr, i1 } %1, 0 + %call = call i32 %3(ptr %vptr) + ret i32 %call +} + +declare i32 @_ZN1A1fEv(ptr %this) + +declare { ptr, i1 } @llvm.type.checked.load(ptr, i32, metadata) +declare void @llvm.ubsantrap(i8 immarg) + +!0 = !{i64 16, !"_ZTS1A"} +!1 = !{i64 0} +;. +; CHECK: [[PROF1]] = !{!"branch_weights", i32 1048575, i32 1} +;. diff --git a/llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-mixed-vis.ll b/llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-mixed-vis.ll new file mode 100644 index 00000000000000..c84922800d7810 --- /dev/null +++ b/llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-mixed-vis.ll @@ -0,0 +1,77 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt -S -passes=wholeprogramdevirt -devirtualize-speculatively -pass-remarks=wholeprogramdevirt %s 2>&1 | FileCheck %s + +; Check that speculative and non-speculative single-impl devirtualization +; work correctly together based on vcall_visibilities. + +target datalayout = "e-p:64:64" +target triple = "x86_64-unknown-linux-gnu" + +; CHECK: remark: <unknown>:0:0: single-impl: devirtualized a call to _ZN1A1fEv +; CHECK: remark: <unknown>:0:0: single-impl: devirtualized a call to _ZN1B1fEv + +%struct.A = type { ptr } +%struct.B = type { ptr } + +@_ZTV1A = linkonce_odr hidden constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN1A1fEv] }, !type !0, !vcall_visibility !2 +@_ZTV1B = linkonce_odr hidden constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr null, ptr @_ZN1B1fEv] }, !type !1, !vcall_visibility !3 + +define i32 @test_speculative(ptr %p) { +; CHECK-LABEL: define i32 @test_speculative( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[P]], align 8 +; CHECK-NEXT: [[TMP1:%.*]] = call i1 @llvm.public.type.test(ptr [[TMP0]], metadata !"_ZTS1A") +; CHECK-NEXT: call void @llvm.assume(i1 [[TMP1]]) +; CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP0]], align 8 +; CHECK-NEXT: [[TMP3:%.*]] = icmp eq ptr [[TMP2]], @_ZN1A1fEv +; CHECK-NEXT: br i1 [[TMP3]], label %[[IF_TRUE_DIRECT_TARG:.*]], label %[[IF_FALSE_ORIG_INDIRECT:.*]], !prof [[PROF2:![0-9]+]] +; CHECK: [[IF_TRUE_DIRECT_TARG]]: +; CHECK-NEXT: [[TMP4:%.*]] = call i32 @_ZN1A1fEv(ptr [[P]]) +; CHECK-NEXT: br label %[[IF_END_ICP:.*]] +; CHECK: [[IF_FALSE_ORIG_INDIRECT]]: +; CHECK-NEXT: [[TMP5:%.*]] = call i32 [[TMP2]](ptr [[P]]) +; CHECK-NEXT: br label %[[IF_END_ICP]] +; CHECK: [[IF_END_ICP]]: +; CHECK-NEXT: [[TMP6:%.*]] = phi i32 [ [[TMP5]], %[[IF_FALSE_ORIG_INDIRECT]] ], [ [[TMP4]], %[[IF_TRUE_DIRECT_TARG]] ] +; CHECK-NEXT: ret i32 [[TMP6]] +; +entry: + %2 = load ptr, ptr %p + %3 = call i1 @llvm.public.type.test(ptr %2, metadata !"_ZTS1A") ; public vcall_visibility + call void @llvm.assume(i1 %3) + %4 = load ptr, ptr %2 + %5 = call i32 %4(ptr %p) + ret i32 %5 +} + +define i32 @test_non_speculative(ptr %p) { +; CHECK-LABEL: define i32 @test_non_speculative( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[P]], align 8 +; CHECK-NEXT: [[TMP1:%.*]] = call i1 @llvm.public.type.test(ptr [[TMP0]], metadata !"_ZTS1B") +; CHECK-NEXT: call void @llvm.assume(i1 [[TMP1]]) +; CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP0]], align 8 +; CHECK-NEXT: [[TMP3:%.*]] = call i32 @_ZN1B1fEv(ptr [[P]]) +; CHECK-NEXT: ret i32 [[TMP3]] +; +entry: + %2 = load ptr, ptr %p + %3 = call i1 @llvm.public.type.test(ptr %2, metadata !"_ZTS1B") ; hidden vcall_visibility + call void @llvm.assume(i1 %3) + %4 = load ptr, ptr %2 + %5 = call i32 %4(ptr %p) + ret i32 %5 +} + +declare i32 @_ZN1A1fEv(ptr %this) +declare i32 @_ZN1B1fEv(ptr %this) + +!0 = !{i64 16, !"_ZTS1A"} +!1 = !{i64 16, !"_ZTS1B"} +!2 = !{i64 0} +!3 = !{i64 1} +;. +; CHECK: [[PROF2]] = !{!"branch_weights", i32 1048575, i32 1} +;. diff --git a/llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-single-impl.ll b/llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-single-impl.ll index 0d30d09ccbb742..d9af1753294cbe 100644 --- a/llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-single-impl.ll +++ b/llvm/test/Transforms/WholeProgramDevirt/speculative-devirt-single-impl.ll @@ -11,10 +11,11 @@ target datalayout = "e-p:64:64" target triple = "x86_64-unknown-linux-gnu" ; CHECK: remark: devirt-single.cc:30:32: single-impl: devirtualized a call to vf +; CHECK: remark: devirt-single.cc:35:32: single-impl: devirtualized a call to vf_empty ; CHECK: remark: devirt-single.cc:41:32: single-impl: devirtualized a call to vf ; CHECK: remark: devirt-single.cc:51:32: single-impl: devirtualized a call to vf ; CHECK: remark: devirt-single.cc:13:0: devirtualized vf -; CHECK-NOT: devirtualized +; CHECK: remark: devirt-single.cc:23:0: devirtualized vf_empty @vt1 = constant [1 x ptr] [ptr @vf], !type !8 @vt2 = constant [1 x ptr] [ptr @vf_empty], !type !12 @@ -23,8 +24,6 @@ define i1 @vf(ptr %this) #0 !dbg !7 { ret i1 true } -; This should NOT be devirtualized because during non-lto empty functions -; are skipped. define void @vf_empty(ptr %this) !dbg !11 { ret void } @@ -50,7 +49,10 @@ define void @call1(ptr %obj) #1 !dbg !9 { %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid1") call void @llvm.assume(i1 %p) %fptr = load ptr, ptr %vtable, align 8 - ; CHECK: call i1 %fptr + ; CHECK: if.true.direct_targ: + ; CHECK: call i1 @vf_empty( + ; CHECK: if.false.orig_indirect: + ; CHECK: call i1 %fptr( %1 = call i1 %fptr(ptr %obj), !dbg !10 ret void } @@ -130,5 +132,5 @@ declare void @llvm.assume(i1) -; CHECK: 1 wholeprogramdevirt - Number of whole program devirtualization targets -; CHECK: 3 wholeprogramdevirt - Number of single implementation devirtualizations +; CHECK: 2 wholeprogramdevirt - Number of whole program devirtualization targets +; CHECK: 4 wholeprogramdevirt - Number of single implementation devirtualizations _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
