llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-platform-windows @llvm/pr-subscribers-lld Author: llvmbot <details> <summary>Changes</summary> Backport 914d5a1 4f5675a Requested by: @<!-- -->bd1976bris --- Full diff: https://github.com/llvm/llvm-project/pull/211206.diff 14 Files Affected: - (modified) lld/COFF/Driver.cpp (+3) - (modified) lld/COFF/LTO.cpp (+5) - (modified) lld/COFF/LTO.h (+1) - (modified) lld/COFF/SymbolTable.cpp (+5) - (modified) lld/COFF/SymbolTable.h (+2) - (modified) lld/ELF/Config.h (+1) - (modified) lld/ELF/Driver.cpp (+9) - (modified) lld/ELF/LTO.cpp (+5) - (modified) lld/ELF/LTO.h (+1) - (modified) llvm/include/llvm/DTLTO/DTLTO.h (+23-1) - (modified) llvm/include/llvm/LTO/LTO.h (+8) - (modified) llvm/lib/DTLTO/CMakeLists.txt (+3) - (modified) llvm/lib/DTLTO/DTLTO.cpp (+48-16) - (modified) llvm/tools/llvm-lto2/llvm-lto2.cpp (+1) ``````````diff diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp index dc4903a5fe126..2d70193187ef1 100644 --- a/lld/COFF/Driver.cpp +++ b/lld/COFF/Driver.cpp @@ -2960,6 +2960,9 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) { // Write the result. writeResult(ctx); + // LTO cleanup may create time trace events. Wait for it to complete before + // writing the time trace data. + ctx.forEachSymtab([](SymbolTable &symtab) { symtab.waitForLTOCleanup(); }); // Stop early so we can print the results. rootTimer.stop(); diff --git a/lld/COFF/LTO.cpp b/lld/COFF/LTO.cpp index 0329f6c2e9cea..71413b0315c08 100644 --- a/lld/COFF/LTO.cpp +++ b/lld/COFF/LTO.cpp @@ -156,6 +156,11 @@ BitcodeCompiler::BitcodeCompiler(COFFLinkerContext &c) : ctx(c) { BitcodeCompiler::~BitcodeCompiler() = default; +void BitcodeCompiler::waitForLTOCleanup() { + if (ltoObj) + ltoObj->waitForCleanup(); +} + static void undefine(Symbol *s) { replaceSymbol<Undefined>(s, s->getName()); } void BitcodeCompiler::add(BitcodeFile &f) { diff --git a/lld/COFF/LTO.h b/lld/COFF/LTO.h index 73e855e567b09..f097d53b9a15f 100644 --- a/lld/COFF/LTO.h +++ b/lld/COFF/LTO.h @@ -46,6 +46,7 @@ class BitcodeCompiler { void add(BitcodeFile &f); std::vector<InputFile *> compile(); void setBitcodeLibFuncs(ArrayRef<StringRef> bitcodeLibFuncs); + void waitForLTOCleanup(); private: std::unique_ptr<llvm::lto::LTO> ltoObj; diff --git a/lld/COFF/SymbolTable.cpp b/lld/COFF/SymbolTable.cpp index df540be800e41..ea5bdbc0be489 100644 --- a/lld/COFF/SymbolTable.cpp +++ b/lld/COFF/SymbolTable.cpp @@ -1511,4 +1511,9 @@ void SymbolTable::compileBitcodeFiles() { } } +void SymbolTable::waitForLTOCleanup() { + if (lto) + lto->waitForLTOCleanup(); +} + } // namespace lld::coff diff --git a/lld/COFF/SymbolTable.h b/lld/COFF/SymbolTable.h index aadd366c7d39f..12d2e8cb23f8e 100644 --- a/lld/COFF/SymbolTable.h +++ b/lld/COFF/SymbolTable.h @@ -109,6 +109,8 @@ class SymbolTable { // added and before the writer writes results to a file. void compileBitcodeFiles(); + void waitForLTOCleanup(); + // Creates an Undefined symbol and marks it as live. Symbol *addGCRoot(StringRef sym, bool aliasEC = false); diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h index 863f3223094bb..1426aab12758f 100644 --- a/lld/ELF/Config.h +++ b/lld/ELF/Config.h @@ -215,6 +215,7 @@ class LinkerDriver { void createFiles(llvm::opt::InputArgList &args); void loadFiles(); void inferMachineType(); + void waitForLTOCleanup(); template <class ELFT> void link(llvm::opt::InputArgList &args); template <class ELFT> void compileBitcodeFiles(bool skipLinkedOutput); // True if we are in --whole-archive and --no-whole-archive. diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp index 77821e8c813ef..51d9e32419f9f 100644 --- a/lld/ELF/Driver.cpp +++ b/lld/ELF/Driver.cpp @@ -625,6 +625,11 @@ constexpr const char *saveTempsValues[] = { LinkerDriver::LinkerDriver(Ctx &ctx) : ctx(ctx) {} +void LinkerDriver::waitForLTOCleanup() { + if (lto) + lto->waitForLTOCleanup(); +} + void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) { ELFOptTable parser; opt::InputArgList args = parser.parse(ctx, argsArr.slice(1)); @@ -707,6 +712,10 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) { invokeELFT(link, args); } + // LTO cleanup may create time trace events. Wait for it to complete before + // writing the time trace data. + waitForLTOCleanup(); + if (ctx.arg.timeTraceEnabled) { checkError(ctx.e, timeTraceProfilerWrite( args.getLastArgValue(OPT_time_trace_eq).str(), diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp index e40575bffec62..a11681133373d 100644 --- a/lld/ELF/LTO.cpp +++ b/lld/ELF/LTO.cpp @@ -231,6 +231,11 @@ BitcodeCompiler::BitcodeCompiler(Ctx &ctx) : ctx(ctx) { BitcodeCompiler::~BitcodeCompiler() = default; +void BitcodeCompiler::waitForLTOCleanup() { + if (ltoObj) + ltoObj->waitForCleanup(); +} + void BitcodeCompiler::add(BitcodeFile &f) { lto::InputFile &obj = *f.obj; bool isExec = !ctx.arg.shared && !ctx.arg.relocatable; diff --git a/lld/ELF/LTO.h b/lld/ELF/LTO.h index c8cb2156d90ca..fdbeb32030eac 100644 --- a/lld/ELF/LTO.h +++ b/lld/ELF/LTO.h @@ -44,6 +44,7 @@ class BitcodeCompiler { void add(BitcodeFile &f); SmallVector<std::unique_ptr<InputFile>, 0> compile(); void setBitcodeLibFuncs(ArrayRef<StringRef> bitcodeLibFuncs); + void waitForLTOCleanup(); private: Ctx &ctx; diff --git a/llvm/include/llvm/DTLTO/DTLTO.h b/llvm/include/llvm/DTLTO/DTLTO.h index 2d532bc384685..13b83ec6e0770 100644 --- a/llvm/include/llvm/DTLTO/DTLTO.h +++ b/llvm/include/llvm/DTLTO/DTLTO.h @@ -19,10 +19,12 @@ #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SmallString.h" #include "llvm/LTO/LTO.h" -#include "llvm/Support/ManagedStatic.h" #include "llvm/Support/Signals.h" +#include "llvm/Support/ThreadPool.h" #include <functional> +#include <string> +#include <utility> #include <vector> namespace llvm { @@ -94,6 +96,12 @@ class LLVM_ABI DTLTO : public LTO { /// Cache) for each task identifier. virtual Error run(AddStreamFn AddStream, FileCache Cache = {}) override; + /// Wait for LTO cleanup. Clients may call this after run() once subsequent + /// linking work that can overlap with cleanup is complete. Cleanup may emit + /// time trace events, so this must be called before time trace data is + /// finalized. + void waitForCleanup() override; + private: /// DTLTO archive support. /// @@ -356,6 +364,20 @@ class LLVM_ABI DTLTO : public LTO { // Cleanup files list. std::vector<std::string> CleanupList; + // There can be many temporary files to remove. Performing deletion in the + // background can save a few seconds on Windows hosts. + struct BackgroundDeletion : DefaultThreadPool { + BackgroundDeletion(); + ~BackgroundDeletion(); + + void removeFiles(std::vector<std::string> &&Files, const Config &Conf); + void waitForTasks(); + + std::vector<std::string> Warnings; + }; + + BackgroundDeletion BackgroundDeleter; + // Record a file for cleanup and register signal-time removal if requested. void addToCleanup(StringRef Filename) { CleanupList.push_back(Filename.str()); diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 32aa671183890..cea66f99b99bb 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -439,6 +439,14 @@ class LLVM_ABI LTO { /// Cache) for each task identifier. virtual Error run(AddStreamFn AddStream, FileCache Cache = {}); + /// Wait for cleanup work started by run() to finish. + /// + /// A client may delay this call to overlap asynchronous cleanup with later + /// linking work, but must call it before finalizing time trace data because + /// cleanup may emit time trace events. Most LTO implementations have no + /// asynchronous cleanup. + virtual void waitForCleanup() {} + /// Static method that returns a list of libcall symbols that can be generated /// by LTO but might not be visible from bitcode symbol table. static SmallVector<const char *> getRuntimeLibcallSymbols(const Triple &TT); diff --git a/llvm/lib/DTLTO/CMakeLists.txt b/llvm/lib/DTLTO/CMakeLists.txt index 09c8c20b6a475..f40f570932108 100644 --- a/llvm/lib/DTLTO/CMakeLists.txt +++ b/llvm/lib/DTLTO/CMakeLists.txt @@ -3,6 +3,9 @@ add_llvm_component_library(LLVMDTLTO DTLTO.cpp DTLTODistributionDriver.cpp + LINK_LIBS + ${LLVM_PTHREAD_LIB} + LINK_COMPONENTS BinaryFormat Core diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp index bca05fa53e889..1d42d09d0820a 100644 --- a/llvm/lib/DTLTO/DTLTO.cpp +++ b/llvm/lib/DTLTO/DTLTO.cpp @@ -27,27 +27,59 @@ #include "llvm/Support/raw_ostream.h" #include <string> +#include <system_error> +#include <utility> +#include <vector> using namespace llvm; +// Experimentation showed that serial deletion is most efficient, hence +// a single thread. +lto::DTLTO::BackgroundDeletion::BackgroundDeletion() + : DefaultThreadPool(hardware_concurrency(1)) {} + +lto::DTLTO::BackgroundDeletion::~BackgroundDeletion() { waitForTasks(); } + +void lto::DTLTO::BackgroundDeletion::waitForTasks() { + wait(); + for (const std::string &Warning : Warnings) + errs() << "warning: could not remove the file " << Warning << "\n"; + Warnings.clear(); +} + +void lto::DTLTO::BackgroundDeletion::removeFiles( + std::vector<std::string> &&Files, const Config &Conf) { + if (Files.empty()) + return; + + async([this, Files = std::move(Files), TTE = Conf.TimeTraceEnabled, + TTG = Conf.TimeTraceGranularity] { + if (LLVM_ENABLE_THREADS && TTE) + timeTraceProfilerInitialize(TTG, "Remove DTLTO temporary files"); + { + TimeTraceScope TimeScope("Remove DTLTO temporary files"); + for (const std::string &Path : Files) { + std::error_code EC = sys::fs::remove(Path, true); + if (!EC || + EC == std::make_error_code(std::errc::no_such_file_or_directory)) + continue; + + Warnings.emplace_back("'" + Path + "': " + EC.message()); + } + } + if (LLVM_ENABLE_THREADS && TTE) + timeTraceProfilerFinishThread(); + }); +} + +void lto::DTLTO::waitForCleanup() { BackgroundDeleter.waitForTasks(); } + // Remove temporary files created to enable distribution. void lto::DTLTO::cleanup() { - if (!SaveTemps) { - // Remove one file, report error if any. - auto removeFile = [](StringRef FileName) -> void { - std::error_code EC = sys::fs::remove(FileName, true); - if (EC && - EC != std::make_error_code(std::errc::no_such_file_or_directory)) - errs() << "warning: could not remove the file '" << FileName - << "': " << EC.message() << "\n"; - }; - - TimeTraceScope JobScope("Remove DTLTO temporary files"); - for (const auto &Name : CleanupList) - removeFile(Name); - // Clean the CleanupList for safety. - CleanupList.clear(); - } + if (SaveTemps) + return; + + BackgroundDeleter.removeFiles(std::move(CleanupList), Conf); } // Runs the DTLTO thin link phase, producing per-module summary indices, diff --git a/llvm/tools/llvm-lto2/llvm-lto2.cpp b/llvm/tools/llvm-lto2/llvm-lto2.cpp index 4bcf04df89ac9..7dd7d9ea67150 100644 --- a/llvm/tools/llvm-lto2/llvm-lto2.cpp +++ b/llvm/tools/llvm-lto2/llvm-lto2.cpp @@ -541,6 +541,7 @@ static int run(int argc, char **argv) { "failed to create cache"); check(Lto->run(AddStream, Cache), "LTO::run failed"); + Lto->waitForCleanup(); return static_cast<int>(HasErrors); } `````````` </details> https://github.com/llvm/llvm-project/pull/211206 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
