Author: Qiongsi Wu Date: 2026-07-31T08:54:13-07:00 New Revision: 5f4d725bca798dc9ab56af114865212849eb6bda
URL: https://github.com/llvm/llvm-project/commit/5f4d725bca798dc9ab56af114865212849eb6bda DIFF: https://github.com/llvm/llvm-project/commit/5f4d725bca798dc9ab56af114865212849eb6bda.diff LOG: [clang][DependencyScanning] Implement the Streaming Style By-name Scanning API (#211406) This PR implements a streaming stype by-name scanning API. The new API takes a `DependencyConsumer`, a `DiagnosticsConsumer` and a `getNextName` lambda. Any diagnositcs flow to the `DependencyConsumer`, and the results flows to to `DependencyConsumer`. The `getNextName` lambda is allows the client to stream names into the API, and the API keeps scanning dependencies for input names until `getNextName` does not return more names. rdar://178088113 --- <sub>Stack created with <a href="https://github.com/github/gh-stack";>GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback";>Give Feedback 💬</a></sub> Added: Modified: clang/include/clang/DependencyScanning/DependencyConsumer.h clang/include/clang/DependencyScanning/DependencyScannerImpl.h clang/include/clang/Tooling/DependencyScanningTool.h clang/lib/Tooling/DependencyScanningTool.cpp clang/test/ClangScanDeps/modules-full-by-mult-mod-names-diagnostics.c clang/test/ClangScanDeps/modules-invalid-target.c clang/tools/clang-scan-deps/ClangScanDeps.cpp Removed: ################################################################################ diff --git a/clang/include/clang/DependencyScanning/DependencyConsumer.h b/clang/include/clang/DependencyScanning/DependencyConsumer.h index 85dca17ee2f6f..54152635a3253 100644 --- a/clang/include/clang/DependencyScanning/DependencyConsumer.h +++ b/clang/include/clang/DependencyScanning/DependencyConsumer.h @@ -41,6 +41,8 @@ class DependencyConsumer { virtual void handleVisibleModule(std::string ModuleName) = 0; virtual void handleContextHash(std::string Hash) = 0; + + virtual void finishQuery(StringRef ModuleName, bool Success) {} }; } // namespace clang::dependencies diff --git a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h index 893017e68233b..f973429a783c3 100644 --- a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h +++ b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h @@ -69,20 +69,6 @@ struct DiagnosticsEngineWithDiagOpts { DiagnosticConsumer &DC); }; -struct TextDiagnosticsPrinterWithOutput { - // We need to bound the lifetime of the data that supports the DiagPrinter - // with it together so they have the same lifetime. - std::string DiagnosticOutput; - llvm::raw_string_ostream DiagnosticsOS; - std::unique_ptr<DiagnosticOptions> DiagOpts; - TextDiagnosticPrinter DiagPrinter; - - TextDiagnosticsPrinterWithOutput(ArrayRef<std::string> CommandLine) - : DiagnosticsOS(DiagnosticOutput), - DiagOpts(createDiagOptions(CommandLine)), - DiagPrinter(DiagnosticsOS, *DiagOpts) {} -}; - std::unique_ptr<CompilerInvocation> createCompilerInvocation(ArrayRef<std::string> CommandLine, DiagnosticsEngine &Diags); diff --git a/clang/include/clang/Tooling/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanningTool.h index 908a70efee89b..b3ed10f7a50d9 100644 --- a/clang/include/clang/Tooling/DependencyScanningTool.h +++ b/clang/include/clang/Tooling/DependencyScanningTool.h @@ -101,16 +101,21 @@ class DependencyScanningTool { dependencies::LookupModuleOutputCallback LookupModuleOutput, std::optional<llvm::MemoryBufferRef> TUBuffer = std::nullopt); - /// Given a compilation context specified via the Clang driver command-line, - /// gather modular dependencies of module with the given name, and return the - /// information needed for explicit build. - /// TODO: this method should be removed as soon as Swift and our C-APIs adopt - /// CompilerInstanceWithContext. We are keeping it here so that it is easier - /// to coordinate with Swift and C-API changes. - llvm::Expected<dependencies::TranslationUnitDeps> getModuleDependencies( - StringRef ModuleName, ArrayRef<std::string> CommandLine, StringRef CWD, - const llvm::DenseSet<dependencies::ModuleID> &AlreadySeen, - dependencies::DependencyActionController &Controller); + /// By-name scanning given a Clang command-line. The scanning context is + /// initialized once and reused for each name pulled from + /// \p getNextName until it returns std::nullopt. Results flow to + /// \p DepConsumer (with per-name status via finishQuery); diagnostics flow to + /// \p DiagConsumer. + /// + /// \returns true if the scanning context initialized successfully and all + /// scans succeeded. Query the DepConsumer for the state of each individual + /// scan. + bool getByNameDependencies( + StringRef CWD, ArrayRef<std::string> CommandLine, + DiagnosticConsumer &DiagConsumer, + dependencies::DependencyActionController &Controller, + llvm::function_ref<std::optional<std::string>()> getNextName, + dependencies::DependencyConsumer &DepConsumer); /// Returns the worker tracing VFS, if it was requested via the service. llvm::vfs::TracingFileSystem *getWorkerTracingVFS() const { @@ -152,8 +157,6 @@ class CompilerInstanceWithContext { DiagnosticConsumer *DiagConsumer = nullptr; std::unique_ptr<dependencies::DiagnosticsEngineWithDiagOpts> DiagEngineWithCmdAndOpts; - std::unique_ptr<dependencies::TextDiagnosticsPrinterWithOutput> - DiagPrinterWithOS; // Context - compiler invocation std::unique_ptr<CompilerInvocation> OriginalInvocation; @@ -181,22 +184,6 @@ class CompilerInstanceWithContext { IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS); public: - /// @brief Initialize the tool's compiler instance from the commandline. - /// The compiler instance only takes a `-cc1` job, so this method - /// builds the `-cc1` job from the CommandLine input. - /// @param Tool The dependency scanning tool whose compiler instance - /// with context is initialized. - /// @param CWD The current working directory. - /// @param CommandLine This command line may be a driver command or a cc1 - /// command. - /// @param DC A diagnostics consumer to report error if the initialization - /// fails. - static std::optional<CompilerInstanceWithContext> initializeFromCommandline( - DependencyScanningTool &Tool, StringRef CWD, - ArrayRef<std::string> CommandLine, - dependencies::DependencyActionController &Controller, - DiagnosticConsumer &DC); - /// @brief Initialize the compiler instance from an already-lowered cc1 /// commandline (driver-free). /// @param Worker The dependency scanning worker to initialize the compiler @@ -216,42 +203,11 @@ class CompilerInstanceWithContext { IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS, dependencies::DependencyActionController &Controller); - /// @brief Initializing the context and the compiler instance. - /// This method must be called before calling - /// computeDependenciesByNameWithContext. - /// @param CWD The current working directory used during the scan. - /// @param CommandLine The commandline used for the scan. - /// @return Error if the initializaiton fails. - static llvm::Expected<CompilerInstanceWithContext> - initializeOrError(DependencyScanningTool &Tool, StringRef CWD, - ArrayRef<std::string> CommandLine, - dependencies::DependencyActionController &Controller); - bool computeDependencies(StringRef ModuleName, dependencies::DependencyConsumer &Consumer, dependencies::DependencyActionController &Controller); - /// @brief Computes the dependeny for the module named ModuleName. - /// @param ModuleName The name of the module for which this method computes - ///. dependencies. - /// @param AlreadySeen This stores modules which have previously been - /// reported. Use the same instance for all calls to this - /// function for a single \c DependencyScanningTool in a - /// single build. Note that this parameter is not part of - /// the context because it can be shared across diff erent - /// worker threads and each worker thread may update it. - /// @param LookupModuleOutput This function is called to fill in - /// "-fmodule-file=", "-o" and other output - /// arguments for dependencies. - /// @return An instance of \c TranslationUnitDeps if the scan is successful. - /// Otherwise it returns an error. - llvm::Expected<dependencies::TranslationUnitDeps> - computeDependenciesByNameOrError( - StringRef ModuleName, - const llvm::DenseSet<dependencies::ModuleID> &AlreadySeen, - dependencies::DependencyActionController &Controller); - // MaxNumOfQueries is the upper limit of the number of names the by-name // scanning API (computeDependencies) can support after a // CompilerInstanceWithContext is initialized. At the time of this commit, the diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp index 6d2c57204b249..5ffd8c0b02115 100644 --- a/clang/lib/Tooling/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanningTool.cpp @@ -171,12 +171,6 @@ static bool computeDependenciesForDriverCommandLine( std::move(OverlayFS)); } -static llvm::Error makeErrorFromDiagnosticsOS( - TextDiagnosticsPrinterWithOutput &DiagPrinterWithOS) { - return llvm::make_error<llvm::StringError>( - DiagPrinterWithOS.DiagnosticsOS.str(), llvm::inconvertibleErrorCode()); -} - bool tooling::computeDependencies( DependencyScanningWorker &Worker, StringRef WorkingDirectory, ArrayRef<std::string> CommandLine, DependencyConsumer &Consumer, @@ -329,20 +323,6 @@ DependencyScanningTool::getTranslationUnitDependencies( return Consumer.takeTranslationUnitDeps(); } -llvm::Expected<TranslationUnitDeps> -DependencyScanningTool::getModuleDependencies( - StringRef ModuleName, ArrayRef<std::string> CommandLine, StringRef CWD, - const llvm::DenseSet<ModuleID> &AlreadySeen, - DependencyActionController &Controller) { - auto MaybeCIWithContext = CompilerInstanceWithContext::initializeOrError( - *this, CWD, CommandLine, Controller); - if (auto Error = MaybeCIWithContext.takeError()) - return Error; - - return MaybeCIWithContext->computeDependenciesByNameOrError( - ModuleName, AlreadySeen, Controller); -} - static std::optional<SmallVector<std::string, 0>> getFirstCC1CommandLine(ArrayRef<std::string> CommandLine, DiagnosticsEngine &Diags, @@ -366,39 +346,46 @@ getFirstCC1CommandLine(ArrayRef<std::string> CommandLine, return std::nullopt; } -std::optional<CompilerInstanceWithContext> -CompilerInstanceWithContext::initializeFromCommandline( - DependencyScanningTool &Tool, StringRef CWD, - ArrayRef<std::string> CommandLine, DependencyActionController &Controller, - DiagnosticConsumer &DC) { +bool DependencyScanningTool::getByNameDependencies( + StringRef CWD, ArrayRef<std::string> CommandLine, + DiagnosticConsumer &DiagConsumer, DependencyActionController &Controller, + llvm::function_ref<std::optional<std::string>()> getNextName, + DependencyConsumer &DepConsumer) { auto [OverlayFS, ModifiedCommandLine] = initVFSForByNameScanning(CommandLine); - auto FS = Tool.Worker.makeEffectiveVFS(CWD, OverlayFS); - - auto DiagEngineWithCmdAndOpts = - std::make_unique<DiagnosticsEngineWithDiagOpts>(ModifiedCommandLine, FS, - DC); - + auto FS = Worker.makeEffectiveVFS(CWD, OverlayFS); + std::vector<std::string> CC1CommandLine; if (ModifiedCommandLine.size() >= 2 && ModifiedCommandLine[1] == "-cc1") { - // The input command line is already a -cc1 invocation; initialize the - // compiler instance directly from it. - return initializeFromCC1Commandline(Tool.Worker, CWD, ModifiedCommandLine, - std::move(DiagEngineWithCmdAndOpts), - std::move(OverlayFS), Controller); + CC1CommandLine = std::move(ModifiedCommandLine); + } else { + // Driver-style (or ill-formed): lower to a cc1 command line, or diagnose. + DiagnosticsEngineWithDiagOpts DiagEngineWithOpts(ModifiedCommandLine, FS, + DiagConsumer); + auto MaybeFirstCC1 = getFirstCC1CommandLine( + ModifiedCommandLine, *DiagEngineWithOpts.DiagEngine, FS); + if (!MaybeFirstCC1) + return false; + CC1CommandLine.assign(MaybeFirstCC1->begin(), MaybeFirstCC1->end()); } - // The input command line is either a driver-style command line, or - // ill-formed. In this case, we will first call the Driver to build a -cc1 - // command line for this compilation or diagnose any ill-formed input. - const auto MaybeFirstCC1 = getFirstCC1CommandLine( - ModifiedCommandLine, *DiagEngineWithCmdAndOpts->DiagEngine, FS); - if (!MaybeFirstCC1) - return std::nullopt; + // Build one scanning session (a CIWC reused across all input names) and pull + // names until getNextName is exhausted. + auto DiagEngineWithOpts = std::make_unique<DiagnosticsEngineWithDiagOpts>( + CC1CommandLine, FS, DiagConsumer); + std::optional<CompilerInstanceWithContext> CIWC = + CompilerInstanceWithContext::initializeFromCC1Commandline( + Worker, CWD, CC1CommandLine, std::move(DiagEngineWithOpts), + std::move(OverlayFS), Controller); + if (!CIWC) + return false; - std::vector<std::string> CC1CommandLine(MaybeFirstCC1->begin(), - MaybeFirstCC1->end()); - return initializeFromCC1Commandline(Tool.Worker, CWD, CC1CommandLine, - std::move(DiagEngineWithCmdAndOpts), - std::move(OverlayFS), Controller); + bool AllScansSucceeded = true; + while (std::optional<std::string> NextName = getNextName()) { + bool Success = + CIWC->computeDependencies(*NextName, DepConsumer, Controller); + DepConsumer.finishQuery(*NextName, Success); + AllScansSucceeded = AllScansSucceeded && Success; + } + return AllScansSucceeded; } std::optional<CompilerInstanceWithContext> @@ -416,51 +403,14 @@ CompilerInstanceWithContext::initializeFromCC1Commandline( return std::move(CIWC); } -llvm::Expected<CompilerInstanceWithContext> -CompilerInstanceWithContext::initializeOrError( - DependencyScanningTool &Tool, StringRef CWD, - ArrayRef<std::string> CommandLine, DependencyActionController &Controller) { - { - auto LogLine = Tool.Worker.Service.getLogger().log(); - LogLine << "init_compiler_instance_with_context:"; - for (const auto &C : CommandLine) { - LogLine << " " << C; - } - } - auto DiagPrinterWithOS = - std::make_unique<TextDiagnosticsPrinterWithOutput>(CommandLine); - - auto Result = initializeFromCommandline(Tool, CWD, CommandLine, Controller, - DiagPrinterWithOS->DiagPrinter); - if (Result) { - Result->DiagPrinterWithOS = std::move(DiagPrinterWithOS); - return std::move(*Result); - } - return makeErrorFromDiagnosticsOS(*DiagPrinterWithOS); -} - -llvm::Expected<TranslationUnitDeps> -CompilerInstanceWithContext::computeDependenciesByNameOrError( - StringRef ModuleName, const llvm::DenseSet<ModuleID> &AlreadySeen, - DependencyActionController &Controller) { - Worker.Service.getLogger().log() << "start scan_by_name: " << ModuleName; - llvm::scope_exit ExitLogging([&] { - Worker.Service.getLogger().log() << "finish scan_by_name: " << ModuleName; - }); - - FullDependencyConsumer Consumer(AlreadySeen); - // We need to clear the DiagnosticOutput so that each by-name lookup - // has a clean diagnostics buffer. - DiagPrinterWithOS->DiagnosticOutput.clear(); - if (computeDependencies(ModuleName, Consumer, Controller)) - return Consumer.takeTranslationUnitDeps(); - return makeErrorFromDiagnosticsOS(*DiagPrinterWithOS); -} - bool CompilerInstanceWithContext::initialize( DependencyActionController &Controller, std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts, IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) { + { + auto LogLine = Worker.Service.getLogger().log(); + LogLine.logArray("init_compiler_instance_with_context:", " ", CommandLine); + } assert(DiagEngineWithDiagOpts && "Valid diagnostics engine required!"); DiagEngineWithCmdAndOpts = std::move(DiagEngineWithDiagOpts); DiagConsumer = DiagEngineWithCmdAndOpts->DiagEngine->getClient(); @@ -514,6 +464,10 @@ bool CompilerInstanceWithContext::initialize( bool CompilerInstanceWithContext::computeDependencies( StringRef ModuleName, DependencyConsumer &Consumer, DependencyActionController &Controller) { + Worker.Service.getLogger().log() << "start scan_by_name: " << ModuleName; + llvm::scope_exit ExitLogging([&] { + Worker.Service.getLogger().log() << "finish scan_by_name: " << ModuleName; + }); if (SrcLocOffset >= MaxNumOfQueries) llvm::report_fatal_error("exceeded maximum by-name scans for worker"); diff --git a/clang/test/ClangScanDeps/modules-full-by-mult-mod-names-diagnostics.c b/clang/test/ClangScanDeps/modules-full-by-mult-mod-names-diagnostics.c index 0eafb119c6bfe..ae8b116a2a4a6 100644 --- a/clang/test/ClangScanDeps/modules-full-by-mult-mod-names-diagnostics.c +++ b/clang/test/ClangScanDeps/modules-full-by-mult-mod-names-diagnostics.c @@ -40,11 +40,9 @@ module root2 { header "root2.h" } // RUN: cat %t/error.cc1.txt | FileCheck %s --check-prefixes=ERROR // RUN: cat %t/result.cc1.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s -// ERROR: Error while scanning dependencies for modA: +// ERROR: Diagnostics while scanning dependencies for 'modA,root,modB,modC,root2': // ERROR-NEXT: module-include.input:1:1: fatal error: module 'modA' not found -// ERROR-NEXT: Error while scanning dependencies for modB: // ERROR-NEXT: module-include.input:1:3: fatal error: module 'modB' not found -// ERROR-NEXT: Error while scanning dependencies for modC: // ERROR-NEXT: module-include.input:1:4: fatal error: module 'modC' not found // CHECK: { // CHECK-NEXT: "modules": [ diff --git a/clang/test/ClangScanDeps/modules-invalid-target.c b/clang/test/ClangScanDeps/modules-invalid-target.c index 333bfdfe48c9e..8432bbbcd0b34 100644 --- a/clang/test/ClangScanDeps/modules-invalid-target.c +++ b/clang/test/ClangScanDeps/modules-invalid-target.c @@ -12,7 +12,7 @@ // Check that CompilerInstanceWithContext::initializeOrError properly errors // during target creation, instead of an assert or a segfault later down the // line: -// CHECK: Error while scanning dependencies for M: +// CHECK: Diagnostics while scanning dependencies for 'M': // CHECK-NEXT: error: unknown target triple 'unknown-unknown-unknown' //--- module.modulemap diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index 2b35d54349f3c..83b4860c130c3 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -644,35 +644,6 @@ class FullDeps { std::vector<InputDeps> Inputs; }; -static bool handleModuleResult(StringRef ModuleName, - llvm::Expected<TranslationUnitDeps> &MaybeTUDeps, - FullDeps &FD, size_t InputIndex, - SharedStream &OS, SharedStream &Errs) { - if (!MaybeTUDeps) { - llvm::handleAllErrors(MaybeTUDeps.takeError(), - [&ModuleName, &Errs](llvm::StringError &Err) { - Errs.applyLocked([&](raw_ostream &OS) { - OS << "Error while scanning dependencies for " - << ModuleName << ":\n"; - OS << Err.getMessage(); - }); - }); - return true; - } - FD.mergeDeps(std::move(MaybeTUDeps->ModuleGraph), InputIndex); - return false; -} - -static void handleErrorWithInfoString(StringRef Info, llvm::Error E, - SharedStream &OS, SharedStream &Errs) { - llvm::handleAllErrors(std::move(E), [&Info, &Errs](llvm::StringError &Err) { - Errs.applyLocked([&](raw_ostream &OS) { - OS << "Error: " << Info << ":\n"; - OS << Err.getMessage(); - }); - }); -} - class P1689Deps { public: void printDependencies(raw_ostream &OS) { @@ -865,6 +836,32 @@ getCompilationDatabase(int argc, char **argv, std::string &ErrorMessage) { FEOpts.Inputs[0].getFile(), OutputFile, CommandLine); } +namespace { +struct ByNameConsumer : DependencyConsumer { + FullDeps &FD; + size_t InputIndex; + ModuleDepsGraph ModuleGraph; + + ByNameConsumer(FullDeps &FD, size_t InputIndex) + : FD(FD), InputIndex(InputIndex) {} + + void handleDependencyOutputOpts(const DependencyOutputOptions &) override {} + void handleFileDependency(StringRef) override {} + void handlePrebuiltModuleDependency(PrebuiltModuleDep) override {} + void handleDirectModuleDependency(ModuleID) override {} + void handleVisibleModule(std::string) override {} + void handleContextHash(std::string) override {} + void handleModuleDependency(ModuleDeps MD) override { + ModuleGraph.push_back(std::move(MD)); + } + void finishQuery(StringRef, bool Success) override { + if (Success) + FD.mergeDeps(std::move(ModuleGraph), InputIndex); + ModuleGraph.clear(); + } +}; +} // namespace + int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { llvm::InitializeAllTargetInfos(); std::string ErrorMessage; @@ -1110,35 +1107,21 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { ModuleNameRef.split(Names, ','); CallbackActionController Controller(LookupOutput); - - if (Names.size() == 1) { - auto MaybeModuleDepsGraph = WorkerTool.getModuleDependencies( - Names[0], Input->CommandLine, CWD, AlreadySeenModules, - Controller); - if (handleModuleResult(Names[0], MaybeModuleDepsGraph, *FD, - LocalIndex, DependencyOS, Errs)) - HadErrors = true; - } else { - auto CIWithCtx = CompilerInstanceWithContext::initializeOrError( - WorkerTool, CWD, Input->CommandLine, Controller); - if (llvm::Error Err = CIWithCtx.takeError()) { - handleErrorWithInfoString( - "Compiler instance with context setup error", std::move(Err), - DependencyOS, Errs); - HadErrors = true; - continue; - } - - for (auto N : Names) { - auto MaybeModuleDepsGraph = - CIWithCtx->computeDependenciesByNameOrError( - N, AlreadySeenModules, Controller); - if (handleModuleResult(N, MaybeModuleDepsGraph, *FD, LocalIndex, - DependencyOS, Errs)) { - HadErrors = true; - } - } - } + ByNameConsumer DepConsumer(*FD, LocalIndex); + + unsigned NameIdx = 0; + auto GetNextName = [&]() -> std::optional<std::string> { + if (NameIdx >= Names.size()) + return std::nullopt; + return Names[NameIdx++].str(); + }; + + bool Success = WorkerTool.getByNameDependencies( + CWD, Input->CommandLine, DiagConsumer, Controller, GetNextName, + DepConsumer); + handleDiagnostics(ModuleNameRef, S, Errs); + if (!Success) + HadErrors = true; } else { std::unique_ptr<llvm::MemoryBuffer> TU; std::optional<llvm::MemoryBufferRef> TUBuffer; _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
