================
@@ -513,6 +541,208 @@ static void codegen(const Config &Conf, TargetMachine *TM,
report_fatal_error(std::move(Err));
}
+static unsigned calFunctionSize(const llvm::Function &F) {
+ unsigned size = 0;
+ for (const auto &BB : F)
+ size += std::distance(BB.begin(), BB.end());
+ return size;
+}
+
+static unsigned calModuleSize(const llvm::Module &M) {
+ unsigned size = 0;
+ for (const auto &F : M)
+ size += calFunctionSize(F);
+ return size;
+}
+
+static bool canDoSplitModule(const llvm::Module &M) {
+ if (calModuleSize(M) < ThinLTOSplitModuleSizeThreshold)
+ return false;
+ return true;
+}
+
+static bool HasLargeCG(Module &Mod, const ModuleSummaryIndex &CombinedIndex) {
+ // TODO: Check whether there has large callgraphs. When multiple callgraphs
+ // are split, thinlto parallel compilation can bring benefits.
+ return true;
+}
+
+struct TaskIdAllocator {
+ using TaskId = unsigned;
+
+ // Use the most significant bit (MSB) as a namespace tag.
+ // - Original ThinLTO backend tasks are expected to have MSB == 0.
+ // - Split partitions allocated by this allocator always have MSB == 1.
+ // This guarantees the two ID spaces never overlap.
+ static constexpr TaskId tag() {
+ return TaskId{1} << (std::numeric_limits<TaskId>::digits - 1);
+ }
+
+ // Monotonic sequence counter for split partitions (MSB must remain 0 here).
+ std::atomic<TaskId> seq{0};
+
+ // Allocate a globally unique TaskId for a split partition.
+ // The returned ID is `tag() | seq`, so it lives in the MSB==1 namespace.
+ TaskId alloc() {
+ TaskId v = seq.fetch_add(1, std::memory_order_relaxed);
+
+ // If the counter ever reaches the MSB, we'd overlap namespaces.
+ // This indicates an overflow / too many partitions.
+ if (v & tag())
+ report_fatal_error("Partition TaskId overflow: seq reached the tag
bit.");
+
+ return tag() | v;
+ }
+
+ // Helper for sanity checks / debugging.
+ static bool isPartition(TaskId id) { return (id & tag()) != 0; }
+};
+
+// Global allocator shared by all split partitions.
+static TaskIdAllocator gSplitTaskIds;
+
+static bool splitOptAndCodeGenThin(unsigned task, const Config &C,
+ TargetMachine *TM, AddStreamFn AddStream,
+ unsigned ParallelCodeGenParallelismLevel,
+ Module &Mod,
+ const ModuleSummaryIndex &CombinedIndex,
+ const std::vector<uint8_t> &CmdArgs,
+ bool DoOpt, AddStreamFn IRAddStream,
+ ArrayRef<StringRef> &BitcodeLibFuncs) {
+ unsigned ThreadCount = 0;
+ const Target *T = &TM->getTarget();
+
+ static std::mutex PrintMutex;
+
+ SplitModuleCG SplitModuleCG(Mod, CombinedIndex,
ParallelCodeGenParallelismLevel);
+ ParallelCodeGenParallelismLevel = SplitModuleCG.getPartitionNum();
+
+ std::vector<std::string> TempObjectFiles(ParallelCodeGenParallelismLevel);
+ std::vector<llvm::FileRemover>
TempFileRemovers(ParallelCodeGenParallelismLevel);
+
+ const auto HandleModulePartition = [&](std::unique_ptr<Module> MPart,
+ unsigned PartitionId) {
+ unsigned CurrentThreadId, UniqueTaskId;
+ {
+ std::lock_guard<std::mutex> Lock(PrintMutex);
+ CurrentThreadId = ThreadCount++;
+
+ // In distributed ThinLTO, `task` may be a sentinel (e.g. -1 cast to
+ // unsigned), which becomes UINT_MAX and naturally has MSB==1. Treat it
+ // as "no base task id" and don't enforce the namespace check on it.
+ //
+ // We do not rely on the incoming `task` for partition uniqueness: split
+ // partitions get a dedicated UniqueTaskId allocated below.
+ if (task != std::numeric_limits<unsigned>::max()) {
+ assert(!TaskIdAllocator::isPartition(task) &&
+ "Original ThinLTO TaskId unexpectedly overlaps the partition "
+ "namespace");
+ }
+ UniqueTaskId = gSplitTaskIds.alloc();
+ }
+
+ std::unique_ptr<TargetMachine> ThreadTM = createTargetMachine(C, T,
*MPart);
+
+ if (DoOpt) {
+ if (!opt(C, ThreadTM.get(), UniqueTaskId, *MPart, /*IsThinLTO=*/true,
+ /*ExportSummary=*/nullptr, /*ImportSummary=*/&CombinedIndex,
+ CmdArgs, BitcodeLibFuncs)) {
+ report_fatal_error("Failed to gen opt for split mod in thread.");
+ }
+
+ // Save the current module before the first codegen round.
+ // Note that the second codegen round runs only `codegen()` without
+ // running `opt()`. We're not reaching here as it's bailed out earlier
+ // with `CodeGenOnly` which has been set in `SecondRoundThinBackend`.
+ if (IRAddStream)
+ cgdata::saveModuleForTwoRounds(*MPart, task + CurrentThreadId,
+ IRAddStream);
+ }
+
+ auto splitStream = [&](unsigned task, const Twine &moduleName)
+ -> Expected<std::unique_ptr<CachedFileStream>> {
+ int FD;
+ SmallString<128> TempFilename;
+ if (std::error_code EC = sys::fs::createTemporaryFile(
+ "thinlto-split", "o", FD, TempFilename))
+ return errorCodeToError(EC);
+
+ TempObjectFiles[PartitionId] = std::string(TempFilename.str());
+ TempFileRemovers[PartitionId].setFile(TempObjectFiles[PartitionId]);
+
+ auto OS = std::make_unique<raw_fd_ostream>(
+ FD, true, /*CloseOnDestruct*/true);
+
+ auto Stream = std::make_unique<CachedFileStream>(
+ std::move(OS), std::string(TempFilename.str()));
+
+ return std::move(Stream);
+ };
+
+ codegen(C, ThreadTM.get(), splitStream, UniqueTaskId, *MPart,
+ CombinedIndex);
+ };
+
+ SplitModuleCG.SplitModule(HandleModulePartition, C);
+
+ // Use ld.lld to combine the partitions into a object.
+ if (TempObjectFiles.empty()) {
+ llvm::errs() << "TempObjectFiles.empty()\n";
+ return true;
+ }
+
+ auto FinalStream = AddStream(task, Mod.getModuleIdentifier());
+ if (!FinalStream)
+ report_fatal_error("Failed to open final output stream");
+
+ int MergedFD;
+ SmallString<128> MergedFilename;
+ if (sys::fs::createTemporaryFile("thinlto-merged", "o", MergedFD,
+ MergedFilename))
+ report_fatal_error("Failed to create merged temp file.");
+ llvm::FileRemover MergedFileRemover(MergedFilename);
+ sys::fs::closeFile(MergedFD);
+
+ std::vector<StringRef> Args;
+ std::string LinkerPath = "";
+ if (auto Path = sys::findProgramByName("ld.lld"))
----------------
mmjjpp wrote:
We've moved lld -r logic into the driver. We use ToolChain.GetProgramPath() to
find the linker executable, and its lookup order is straightforward:
1. First look through every prefix directory passed with -B;
2. Then check the toolchain's program paths from getProgramPaths() — this
normally includes the folder where clang lives;
3. If those checks fail, it finally looks up the binary from the PATH
environment variable.
https://github.com/llvm/llvm-project/pull/198702
_______________________________________________
cfe-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
