================ @@ -1,83 +1,407 @@ -//===- DTLTO.h - Distributed ThinLTO functions and classes ----*- C++ -*-===// +//===- DTLTO.h - Integrated Distributed ThinLTO implementation ------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -//===---------------------------------------------------------------------===// +//===----------------------------------------------------------------------===// +// +// \file +// Declarations for Integrated Distributed ThinLTO, including the DTLTO class +// and the distribution driver. The implementation focuses on preparing input +// files for distribution. +// +//===----------------------------------------------------------------------===// #ifndef LLVM_DTLTO_DTLTO_H #define LLVM_DTLTO_DTLTO_H +#include "llvm/ADT/SmallString.h" #include "llvm/LTO/LTO.h" -#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/Signals.h" + +#include <functional> +#include <vector> namespace llvm { namespace lto { -// The purpose of this class is to prepare inputs so that distributed ThinLTO -// backend compilations can succeed. -// -// For distributed compilation, each input must exist as an individual bitcode -// file on disk and be loadable via its ModuleID. This requirement is not met -// for archive members, as an archive is a collection of files rather than a -// standalone file. Similarly, for FatLTO objects, the bitcode is stored in a -// section of the containing ELF object file. To address this, the class ensures -// that an individual bitcode file exists for each input (by writing it out if -// necessary) and that the ModuleID is updated to point to it. Module IDs are -// also normalized on Windows to remove short 8.3 form paths that cannot be -// loaded on remote machines. -// -// The class ensures that lto::InputFile objects are preserved until enough of -// the LTO pipeline has executed to determine the required per-module -// information, such as whether a module will participate in ThinLTO. +/// Prepares inputs for Distributed ThinLTO so that backend compilations can use +/// individual bitcode paths and consistent module IDs. +/// +/// Each input must exist as an individual bitcode file on disk and be loadable +/// via its ModuleID. Archive members and FatLTO objects do not satisfy that by +/// default; this class writes bitcode out when needed and updates ModuleID. +/// On Windows, module IDs are normalized to remove short 8.3 path components +/// that are machine-local and break distribution; other normalization is left +/// to DTLTO distributors. +/// +/// Input files are kept until the pipeline has determined per-module ThinLTO +/// participation. addInput() performs: (1) register the input; (2) on Windows, +/// normalize module ID for standalone bitcode; (3) for thin archive members, +/// set module ID to the on-disk member path; (4) for other archives and FatLTO, +/// set module ID to a unique path and serialize content in +/// serializeBitcodeArchiveMembers(). class DTLTO : public LTO { using Base = LTO; public: - LLVM_ABI DTLTO(Config Conf, ThinBackend Backend, - unsigned ParallelCodeGenParallelismLevel, LTOKind LTOMode, - StringRef LinkerOutputFile, bool SaveTemps) - : Base(std::move(Conf), Backend, ParallelCodeGenParallelismLevel, - LTOMode), - LinkerOutputFile(LinkerOutputFile), SaveTemps(SaveTemps) { + LLVM_ABI DTLTO(Config Conf, unsigned ParallelCodeGenParallelismLevel, + LTOKind LTOMode, IndexWriteCallback OnWrite, + bool EmitIndexFiles, bool EmitImportsFiles, + StringRef LinkerOutputFile, StringRef Distributor, + ArrayRef<StringRef> DistributorArgs, StringRef RemoteCompiler, + ArrayRef<StringRef> RemoteCompilerPrependArgs, + ArrayRef<StringRef> RemoteCompilerArgs, + AddBufferFn AddBufferArg, bool SaveTempsArg) + : Base(std::move(Conf), writeIndexesBackendInstance(), + ParallelCodeGenParallelismLevel, LTOMode), + AddBuffer(AddBufferArg), SaveTemps(SaveTempsArg), + ShouldEmitIndexFiles(EmitIndexFiles), + ShouldEmitImportFiles(EmitImportsFiles), OnIndexWriteCb(OnWrite), + DistributorParams{Distributor, DistributorArgs, + RemoteCompiler, RemoteCompilerPrependArgs, + RemoteCompilerArgs, LinkerOutputFile} { assert(!LinkerOutputFile.empty() && "expected a valid linker output file"); } - // Add an input file and prepare it for distribution. + // Create an instance of WriteIndexesBackend class. + static lto::ThinBackend writeIndexesBackendInstance() { + return lto::createWriteIndexesThinBackend(hardware_concurrency(), "", "", + "", true, nullptr, nullptr); + } + + /// Add an input file and prepare it for distribution. + /// + /// This function performs the following tasks: + /// 1. Add the input file to the LTO object's list of input files. + /// 2. For individual bitcode file inputs on Windows only, overwrite the + /// module ID with a normalized path to remove short 8.3 form components. + /// 3. For thin archive members, overwrite the module ID with the path + /// (normalized on Windows) to the member file on disk. + /// 4. For archive members and FatLTO objects, overwrite the module ID with a + /// unique path (normalized on Windows) naming a file that will contain the + /// member content. The file is created and populated later (see + /// serializeInputs()). LLVM_ABI Expected<std::shared_ptr<InputFile>> addInput(std::unique_ptr<InputFile> InputPtr) override; -protected: - // Save the contents of ThinLTO-enabled input files that must be serialized - // for distribution, such as archive members and FatLTO objects, to individual - // bitcode files named after the module ID. - LLVM_ABI llvm::Error serializeInputsForDistribution() override; + /// Runs the DTLTO pipeline. This function calls the supplied AddStream + /// function to add native object files to the link. + /// + /// The Cache parameter is optional. If supplied, it will be used to cache + /// native object files and add them to the link. + /// + /// The client will receive at most one callback (via either AddStream or + /// Cache) for each task identifier. + LLVM_ABI virtual Error run(AddStreamFn AddStream, + FileCache Cache = {}) override; + +private: + /// DTLTO archives support. + /// + /// Save the contents of ThinLTO-enabled input files that must be serialized + /// for distribution, such as archive members and FatLTO objects, to + /// individual bitcode files named after the module ID. + /// + /// Must be called after all input files are added but before optimization + /// begins. If a file with that name already exists, it is likely a leftover + /// from a previously terminated linker process and can be safely overwritten. + LLVM_ABI Error serializeBitcodeArchiveMembers(); + // Remove temporary files created to enable distribution. LLVM_ABI void cleanup() override; +public: + // Mutable and const accessors to the LTO configuration object. + Config &getConfig() { return Conf; } + const Config &getConfig() const { return Conf; } + + // Set the LTO kind. + void setLTOMode(LTOKind Knd) { LTOMode = Knd; } + // Replace the ThinLTO backend (e.g. WriteIndexesThinBackend for the thin + // link). + void setThinBackend(ThinBackend Backend) { ThinLTO.Backend = Backend; } ---------------- kbelochapka wrote:
Removed `setThinBackend()` Removed `setLTOMode()`, `LTOMode' variable initialization moved to `DTLTO` class constructor. Replaced `ImportsFilesLists[Task]` with ` J.ImportsFiles` Renamed `handleArchiveInputs()` into `serializeLTOInputs()` https://github.com/llvm/llvm-project/pull/192629 _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
