https://github.com/llvmbot created https://github.com/llvm/llvm-project/pull/176758
Backport 31f5be4ce54107ed8adc79d773a7dbf5c7640938 Requested by: @bd1976bris >From fb3e1d9f49fa3b13fae51b52ab96259fa7677f8e Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin <[email protected]> Date: Mon, 19 Jan 2026 14:06:58 +0000 Subject: [PATCH] [DTLTO] Add DTLTO-specific LTO input handling time-trace scopes (#175799) Add time-trace scopes to the DTLTO-specific input-handling code to improve observability and debugging. These scopes are tested via LLD, as the primary purpose of this code is to support member files of non-thin archives as DTLTO inputs. `llvm-lto2` does not currently support archives. Adding archive support to `llvm-lto2` solely for testing these scopes does not appear to be worthwhile. As part of this change, the deletion of temporary DTLTO input files has been moved. Cleanup now occurs after LTO has completed, rather than during destruction of the LTO object. This is required since by the time the LTO object is destroyed, time-traces have already been finalized, so no additional trace data can be recorded. Recording time-trace data for temporary file deletion is important, as this has been a source of performance issues in the past and an area where we expect to make further performance improvements if supported by the data. SIE internal tracker: TOOLCHAIN-21021 (cherry picked from commit 31f5be4ce54107ed8adc79d773a7dbf5c7640938) --- lld/test/ELF/dtlto/timetrace.test | 65 +++++++++++++++++++++++++++++++ llvm/include/llvm/DTLTO/DTLTO.h | 31 +++++++-------- llvm/include/llvm/LTO/LTO.h | 9 ++++- llvm/lib/DTLTO/DTLTO.cpp | 23 ++++++----- llvm/lib/LTO/LTO.cpp | 2 + 5 files changed, 103 insertions(+), 27 deletions(-) create mode 100644 lld/test/ELF/dtlto/timetrace.test diff --git a/lld/test/ELF/dtlto/timetrace.test b/lld/test/ELF/dtlto/timetrace.test new file mode 100644 index 0000000000000..639ad36f8019f --- /dev/null +++ b/lld/test/ELF/dtlto/timetrace.test @@ -0,0 +1,65 @@ +REQUIRES: x86 + +## Test that DTLTO-specific LTO input file handling time-trace output is +## produced as expected. + +RUN: rm -rf %t && split-file %s %t && cd %t + +RUN: sed 's/@t1/@t2/g' t1.ll > t2.ll + +## Generate ThinLTO bitcode files. +RUN: opt -thinlto-bc t1.ll -o t1.bc +RUN: opt -thinlto-bc t2.ll -o t2.bc + +## Create archives. +RUN: llvm-ar rcs t1.a t1.bc +RUN: llvm-ar rcsT t2.thin.a t2.bc + +## Generate object files for mock.py to return. +RUN: llc t1.ll --filetype=obj -o t1.o +RUN: llc t2.ll --filetype=obj -o t2.o + +## Link and generate a time-trace. +## Note: mock.py doesn't compile; it copies the specified object files to the +## outputs in job order. +RUN: ld.lld --whole-archive t1.a t2.thin.a -o my.elf \ +RUN: --thinlto-distributor=%python \ +RUN: --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \ +RUN: --thinlto-distributor-arg=t1.o --thinlto-distributor-arg=t2.o \ +RUN: --time-trace-granularity=0 --time-trace=%t.json +RUN: %python filter_order_and_pprint.py %t.json | FileCheck %s + +## Check that DTLTO add input file events are recorded. +CHECK: "name": "Add input for DTLTO" +CHECK: "name": "Add input for DTLTO" +CHECK: "name": "Remove temporary inputs for DTLTO" +CHECK: "name": "Save input archive member for DTLTO" +CHECK-SAME: "detail": "t1.a(t1.bc at [[#ARCHIVE_OFFSET:]]).1.[[PID:[A-F0-9]+]].o" +CHECK: "name": "Total Add input for DTLTO" +CHECK-SAME: "count": 2, +CHECK: "name": "Total Remove temporary inputs for DTLTO" +CHECK-SAME: "count": 1, +CHECK: "name": "Total Save input archive member for DTLTO" +CHECK-SAME: "count": 1, + +#--- t1.ll +target triple = "x86_64-unknown-linux-gnu" +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" + +define void @t1() { + ret void +} + +#--- filter_order_and_pprint.py +import json, sys + +data = json.load(open(sys.argv[1], "r", encoding="utf-8")) + +# Get DTLTO events. +events = [e for e in data["traceEvents"] if "DTLTO" in e["name"]] +events.sort(key=lambda e: (e["name"], str(e.get("args", {}).get("detail", "")))) + +# Print an event per line. Ensure 'name' is the first key. +for ev in events: + name = ev.pop("name") + print(json.dumps({"name": name, **ev})) diff --git a/llvm/include/llvm/DTLTO/DTLTO.h b/llvm/include/llvm/DTLTO/DTLTO.h index f9fad743e721c..14f1f5fd00e30 100644 --- a/llvm/include/llvm/DTLTO/DTLTO.h +++ b/llvm/include/llvm/DTLTO/DTLTO.h @@ -16,19 +16,27 @@ namespace llvm { namespace lto { class DTLTO : public LTO { + using Base = LTO; + public: - // Inherit contructors from LTO base class. - using LTO::LTO; - ~DTLTO() { removeTempFiles(); } + // Inherit constructors. + using Base::Base; + ~DTLTO() override = default; + + // Add an input file and prepare it for distribution. + LLVM_ABI Expected<std::shared_ptr<InputFile>> + addInput(std::unique_ptr<InputFile> InputPtr) override; + +protected: + LLVM_ABI llvm::Error handleArchiveInputs() override; + + LLVM_ABI void cleanup() override; private: // Bump allocator for a purpose of saving updated module IDs. BumpPtrAllocator PtrAlloc; StringSaver Saver{PtrAlloc}; - // Removes temporary files. - LLVM_ABI void removeTempFiles(); - // Determines if a file at the given path is a thin archive file. Expected<bool> isThinArchive(const StringRef ArchivePath); @@ -44,17 +52,8 @@ class DTLTO : public LTO { // A cache to avoid repeatedly reading the same archive file. StringMap<bool> ArchiveFiles; - -public: - // Adds the input file to the LTO object's list of input files. - // For archive members, generates a new module ID which is a path to a real - // file on a filesystem. - LLVM_ABI virtual Expected<std::shared_ptr<lto::InputFile>> - addInput(std::unique_ptr<lto::InputFile> InputPtr) override; - - // Entry point for DTLTO archives support. - LLVM_ABI virtual llvm::Error handleArchiveInputs() override; }; + } // namespace lto } // namespace llvm diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 819be1909ec12..b0d776d973c90 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -443,6 +443,13 @@ class LTO { LLVM_ABI static SmallVector<const char *> getRuntimeLibcallSymbols(const Triple &TT); +protected: + // Called at the start of run(). + virtual Error handleArchiveInputs() { return Error::success(); } + + // Called before returning from run(). + virtual void cleanup() {} + private: Config Conf; @@ -620,8 +627,6 @@ class LTO { addInput(std::unique_ptr<lto::InputFile> InputPtr) { return std::shared_ptr<lto::InputFile>(InputPtr.release()); } - - virtual llvm::Error handleArchiveInputs() { return llvm::Error::success(); } }; /// The resolution for a symbol. The linker must provide a SymbolResolution for diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp index 7ba4bfd80b6ab..c85ce77720dee 100644 --- a/llvm/lib/DTLTO/DTLTO.cpp +++ b/llvm/lib/DTLTO/DTLTO.cpp @@ -25,6 +25,7 @@ #include "llvm/Support/MemoryBufferRef.h" #include "llvm/Support/Path.h" #include "llvm/Support/Process.h" +#include "llvm/Support/TimeProfiler.h" #include "llvm/Support/raw_ostream.h" #include <iostream> @@ -116,15 +117,6 @@ Expected<bool> lto::DTLTO::isThinArchive(const StringRef ArchivePath) { return IsThin; } -// Removes any temporary regular archive member files that were created during -// processing. -void lto::DTLTO::removeTempFiles() { - for (auto &Input : InputFiles) { - if (Input->isMemberOfArchive()) - sys::fs::remove(Input->getName(), /*IgnoreNonExisting=*/true); - } -} - // This function performs the following tasks: // 1. Adds the input file to the LTO object's list of input files. // 2. For thin archive members, generates a new module ID which is a path to a @@ -133,6 +125,7 @@ void lto::DTLTO::removeTempFiles() { // 4. Updates the bitcode module's identifier. Expected<std::shared_ptr<lto::InputFile>> lto::DTLTO::addInput(std::unique_ptr<lto::InputFile> InputPtr) { + TimeTraceScope TimeScope("Add input for DTLTO"); // Add the input file to the LTO object. InputFiles.emplace_back(InputPtr.release()); @@ -180,6 +173,7 @@ lto::DTLTO::addInput(std::unique_ptr<lto::InputFile> InputPtr) { Error lto::DTLTO::saveInputArchiveMember(lto::InputFile *Input) { StringRef ModuleId = Input->getName(); if (Input->isMemberOfArchive()) { + TimeTraceScope TimeScope("Save input archive member for DTLTO", ModuleId); MemoryBufferRef MemoryBufferRef = Input->getFileBuffer(); if (Error EC = saveBuffer(MemoryBufferRef.getBuffer(), ModuleId)) return EC; @@ -210,3 +204,14 @@ llvm::Error lto::DTLTO::handleArchiveInputs() { return EC; return Error::success(); } + +// Remove temporary archive member files created to enable distribution. +void lto::DTLTO::cleanup() { + { + TimeTraceScope TimeScope("Remove temporary inputs for DTLTO"); + for (auto &Input : InputFiles) + if (Input->isMemberOfArchive()) + sys::fs::remove(Input->getName(), /*IgnoreNonExisting=*/true); + } + Base::cleanup(); +} diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index ff6762ebb59be..5f0ff9a1c39ac 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -1215,6 +1215,8 @@ Error LTO::checkPartiallySplit() { } Error LTO::run(AddStreamFn AddStream, FileCache Cache) { + llvm::scope_exit CleanUp([this]() { cleanup(); }); + if (Error EC = handleArchiveInputs()) return EC; _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
