================
@@ -194,6 +209,1267 @@ static void applyArgsForStdModuleManifestInputs(
}
}
+/// Computes the -fmodule-cache-path for this compilation.
+static std::optional<std::string>
+getModuleCachePath(llvm::opt::DerivedArgList &Args) {
+ if (const Arg *A = Args.getLastArg(options::OPT_fmodules_cache_path))
+ return A->getValue();
+
+ if (SmallString<128> Path; Driver::getDefaultModuleCachePath(Path))
+ return std::string(Path);
+
+ return std::nullopt;
+}
+
+/// Returns true if a dependency scan can be performed using \p Job.
+static bool isDependencyScannableJob(const Command &Job) {
+ if (!isCC1Job(Job))
+ return false;
+ const auto &InputInfos = Job.getInputInfos();
+ return !InputInfos.empty() && types::isSrcFile(InputInfos.front().getType());
+}
+
+namespace {
+/// Pool of reusable dependency scanning workers and their contexts with
+/// RAII-based acquire/release.
+class ScanningWorkerPool {
+public:
+ ScanningWorkerPool(size_t NumWorkers,
+ deps::DependencyScanningService &ScanningService) {
+ for (size_t I = 0; I < NumWorkers; ++I)
+ Slots.emplace_back(ScanningService);
+
+ AvailableSlots.resize(NumWorkers);
+ std::iota(AvailableSlots.begin(), AvailableSlots.end(), 0);
+ }
+
+ /// Acquires a unique pointer to a dependency scanning worker and its
+ /// context.
+ ///
+ /// The worker bundle automatically released back to the pool when the
+ /// pointer is destroyed. The pool has to outlive the leased worker bundle.
+ [[nodiscard]] auto scopedAcquire() {
+ std::unique_lock<std::mutex> UL(Lock);
+ CV.wait(UL, [&] { return !AvailableSlots.empty(); });
+ const size_t Index = AvailableSlots.pop_back_val();
+ auto ReleaseHandle = [this, Index](WorkerBundle *) { release(Index); };
+ return std::unique_ptr<WorkerBundle, decltype(ReleaseHandle)>(
+ &Slots[Index], ReleaseHandle);
+ }
+
+private:
+ /// Releases the worker bundle at \c Index back into the pool.
+ void release(size_t Index) {
+ {
+ std::scoped_lock<std::mutex> SL(Lock);
+ AvailableSlots.push_back(Index);
+ }
+ CV.notify_one();
+ }
+
+ /// A scanning worker with its associated context.
+ struct WorkerBundle {
+ WorkerBundle(deps::DependencyScanningService &ScanningService)
+ : Worker(std::make_unique<deps::DependencyScanningWorker>(
+ ScanningService)) {}
+
+ std::unique_ptr<deps::DependencyScanningWorker> Worker;
+ llvm::DenseSet<deps::ModuleID> SeenModules;
+ };
+
+ std::mutex Lock;
+ std::condition_variable CV;
+ SmallVector<size_t> AvailableSlots;
+ SmallVector<WorkerBundle, 0> Slots;
+};
+} // anonymous namespace
+
+// Creates a ThreadPool and a corresponding ScanningWorkerPool optimized for
+// the configuration of dependency scan inputs.
+static std::pair<std::unique_ptr<llvm::ThreadPoolInterface>,
+ std::unique_ptr<ScanningWorkerPool>>
+createOptimalThreadAndWorkerPool(
+ size_t NumScanInputs, bool HasStdlibModuleInputs,
+ deps::DependencyScanningService &ScanningService) {
+ // TODO: Benchmark: Determine the optimal number of worker threads for a
+ // given number of inputs. How many inputs are required for multi-threading
+ // to be beneficial? How many inputs should each thread scan at least?
+#if LLVM_ENABLE_THREADS
+ std::unique_ptr<llvm::ThreadPoolInterface> ThreadPool;
+ size_t WorkerCount;
+
+ if (NumScanInputs == 1 || (HasStdlibModuleInputs && NumScanInputs <= 2)) {
+ auto S = llvm::optimal_concurrency(1);
+ ThreadPool = std::make_unique<llvm::SingleThreadExecutor>(std::move(S));
+ WorkerCount = 1;
+ } else {
+ auto ThreadPoolStrategy = llvm::optimal_concurrency(
+ NumScanInputs - static_cast<size_t>(HasStdlibModuleInputs));
+ ThreadPool = std::make_unique<llvm::DefaultThreadPool>(
+ std::move(ThreadPoolStrategy));
+ const size_t MaxConcurrency = ThreadPool->getMaxConcurrency();
+ const size_t MaxConcurrentlyScannedInputs =
+ NumScanInputs -
+ (HasStdlibModuleInputs && NumScanInputs < MaxConcurrency ? 1 : 0);
+ WorkerCount = std::min(MaxConcurrency, MaxConcurrentlyScannedInputs);
+ }
+#else
+ auto ThreadPool = std::make_unique<llvm::SingleThreadExecutor>();
+ size_t WorkerCount = 1;
+#endif
+
+ return {std::move(ThreadPool),
+ std::make_unique<ScanningWorkerPool>(WorkerCount, ScanningService)};
+}
+
+static StringRef getTriple(const Command &Job) {
+ return Job.getCreator().getToolChain().getTriple().getTriple();
+}
+
+using ModuleNameAndTriple = std::pair<StringRef, StringRef>;
+
+namespace {
+/// Helper to schedule on-demand dependency scans for modules originating from
+/// the Standard library module manifest.
+struct StdlibModuleScanScheduler {
+ StdlibModuleScanScheduler(const llvm::DenseMap<ModuleNameAndTriple, size_t>
+ &StdlibModuleScanIndexByID)
+ : StdlibModuleScanIndexByID(StdlibModuleScanIndexByID) {
+ ScheduledScanInputs.reserve(StdlibModuleScanIndexByID.size());
+ }
+
+ /// Returns the indices of scan inputs corresponding to newly imported
+ /// Standard library modules.
+ ///
+ /// Thread-safe.
+ SmallVector<size_t, 2> getNewScanInputs(ArrayRef<std::string>
NamedModuleDeps,
+ StringRef Triple) {
+ SmallVector<size_t, 2> NewScanInputs;
+ std::scoped_lock<std::mutex> Guard(Lock);
+ for (const auto &ModuleName : NamedModuleDeps) {
+ const auto It = StdlibModuleScanIndexByID.find({ModuleName, Triple});
+ if (It == StdlibModuleScanIndexByID.end())
+ continue;
+ const size_t ScanIndex = It->second;
+ const bool AlreadyScheduled =
+ !ScheduledScanInputs.insert(ScanIndex).second;
+ if (AlreadyScheduled)
+ continue;
+ NewScanInputs.push_back(ScanIndex);
+ }
+ return NewScanInputs;
+ }
+
+private:
+ const llvm::DenseMap<ModuleNameAndTriple, size_t> &StdlibModuleScanIndexByID;
+ llvm::SmallDenseSet<size_t> ScheduledScanInputs;
+ std::mutex Lock;
+};
+
+/// Collects diagnostics in a form that can be retained until after their
+/// associated SourceManager is destroyed.
+class StandaloneDiagCollector : public DiagnosticConsumer {
+public:
+ void BeginSourceFile(const LangOptions &LangOpts,
+ const Preprocessor *PP = nullptr) override {
+ this->LangOpts = &LangOpts;
+ }
+
+ void HandleDiagnostic(DiagnosticsEngine::Level Level,
+ const Diagnostic &Info) override {
+ StoredDiagnostic StoredDiag(Level, Info);
+ StandaloneDiags.emplace_back(*LangOpts, StoredDiag);
+ DiagnosticConsumer::HandleDiagnostic(Level, Info);
+ }
+
+ SmallVector<StandaloneDiagnostic, 0> takeDiagnostics() {
+ return std::move(StandaloneDiags);
+ }
+
+private:
+ const LangOptions *LangOpts = nullptr;
+ SmallVector<StandaloneDiagnostic, 0> StandaloneDiags;
+};
+
+/// RAII utility to report collected StandaloneDiagnostic through a
+/// DiagnosticsEngine.
+///
+/// The driver's DiagnosticsEngine usually does not have a SourceManager at
+/// this point of building the compilation, in which case the
+/// StandaloneDiagReporter supplies its own.
+class StandaloneDiagReporter {
+public:
+ explicit StandaloneDiagReporter(DiagnosticsEngine &Diags) : Diags(Diags) {
+ if (!Diags.hasSourceManager()) {
+ FileSystemOptions Opts;
+ Opts.WorkingDir = ".";
+ OwnedFileMgr = llvm::makeIntrusiveRefCnt<FileManager>(std::move(Opts));
+ OwnedSrcMgr =
+ llvm::makeIntrusiveRefCnt<SourceManager>(Diags, *OwnedFileMgr);
+ }
+ }
+
+ /// Emits all diagnostics in \c StandaloneDiags using the associated
+ /// DiagnosticsEngine.
+ void Report(ArrayRef<StandaloneDiagnostic> StandaloneDiags) const {
+ llvm::StringMap<SourceLocation> SrcLocCache;
+ Diags.getClient()->BeginSourceFile(LangOptions(), nullptr);
+ for (const auto &StandaloneDiag : StandaloneDiags) {
+ const auto StoredDiag = translateStandaloneDiag(
+ getFileManager(), getSourceManager(), StandaloneDiag, SrcLocCache);
+ Diags.Report(StoredDiag);
+ }
+ Diags.getClient()->EndSourceFile();
+ }
+
+private:
+ DiagnosticsEngine &Diags;
+ IntrusiveRefCntPtr<FileManager> OwnedFileMgr;
+ IntrusiveRefCntPtr<SourceManager> OwnedSrcMgr;
+
+ FileManager &getFileManager() const {
+ if (OwnedFileMgr)
+ return *OwnedFileMgr;
+ return Diags.getSourceManager().getFileManager();
+ }
+
+ SourceManager &getSourceManager() const {
+ if (OwnedSrcMgr)
+ return *OwnedSrcMgr;
+ return Diags.getSourceManager();
+ }
+};
+} // anonymous namespace
+
+/// Report the diagnostics collected during each dependency scan.
+static void reportAllScanDiagnostics(
+ SmallVectorImpl<SmallVector<StandaloneDiagnostic, 0>> &&AllScanDiags,
+ DiagnosticsEngine &Diags) {
+ StandaloneDiagReporter Reporter(Diags);
+ for (auto &SingleScanDiags : AllScanDiags)
+ Reporter.Report(SingleScanDiags);
+}
+
+/// Construct a path for the explicitly built PCM.
+static std::string constructPCMPath(const deps::ModuleID &ID,
+ StringRef OutputDir) {
+ assert(!ID.ModuleName.empty() && !ID.ContextHash.empty() &&
+ "Invalid ModuleID!");
+ SmallString<256> ExplicitPCMPath(OutputDir);
+ llvm::sys::path::append(ExplicitPCMPath, ID.ContextHash,
+ ID.ModuleName + "-" + ID.ContextHash + ".pcm");
+ return std::string(ExplicitPCMPath);
+}
+
+namespace {
+/// A simple dependency action controller that only provides module lookup for
+/// Clang modules.
+class ModuleLookupController : public deps::DependencyActionController {
+public:
+ ModuleLookupController(StringRef OutputDir) : OutputDir(OutputDir) {}
+
+ std::string lookupModuleOutput(const deps::ModuleDeps &MD,
+ deps::ModuleOutputKind Kind) override {
+ if (Kind == deps::ModuleOutputKind::ModuleFile)
+ return constructPCMPath(MD.ID, OutputDir);
+
+ // Driver command lines that trigger lookups for unsupported
+ // ModuleOutputKinds are not supported by the modules driver. Those
+ // command lines should probably be adjusted or rejected in
+ // Driver::handleArguments or Driver::HandleImmediateArgs.
+ llvm::reportFatalInternalError(
+ "call to lookupModuleOutput with unexpected ModuleOutputKind");
+ }
+
+ std::unique_ptr<DependencyActionController> clone() const override {
+ return std::make_unique<ModuleLookupController>(OutputDir);
+ }
+
+private:
+ StringRef OutputDir;
+};
+
+/// The full dependencies for a specific command-line input.
+struct InputDependencies {
+ /// The name of the C++20 module provided by this translation unit.
+ std::string ModuleName;
+
+ /// A list of modules this translation unit directly depends on, not
including
+ /// transitive dependencies.
+ ///
+ /// This may include modules with a different context hash when it can be
+ /// determined that the differences are benign for this compilation.
+ std::vector<deps::ModuleID> ClangModuleDeps;
+
+ /// A list of the C++20 named modules this translation unit depends on.
+ ///
+ /// All C++20 named module dependencies are expected to target the same
triple
+ /// as this translation unit.
----------------
Bigcheese wrote:
This should be something like "compatible invocation", more than the triple
matters.
https://github.com/llvm/llvm-project/pull/152770
_______________________________________________
cfe-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
