llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-clang Author: Naveen Seth Hanig (naveen-seth) <details> <summary>Changes</summary> This is the second patch in a series that removes the dependency of `clangDependencyScanning` on `clangDriver`, splitting the work from #<!-- -->169964 into smaller changes (see comment linked below). This patch updates the by-name scanning interface in `DependencyScanningWorker` to accept only `-cc1` command lines directly and moves the logic for handling driver-style command lines into `DependencyScanningTool` in `clangTooling`. Support for `-cc1` command lines in by-name scanning is introduced in this patch. The next patch will update the remaining parts of `DependencyScanningWorker` to operate only on `-cc1` command lines, allowing its dependency on `clangDriver` to be removed. https://github.com/llvm/llvm-project/pull/169964#pullrequestreview-3545879529 --- Patch is 28.13 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/171238.diff 9 Files Affected: - (modified) clang/include/clang/DependencyScanning/DependencyScannerImpl.h (+10-22) - (modified) clang/include/clang/DependencyScanning/DependencyScanningWorker.h (+25-21) - (modified) clang/include/clang/Tooling/DependencyScanningTool.h (+6-6) - (modified) clang/lib/DependencyScanning/DependencyScannerImpl.cpp (+25-39) - (modified) clang/lib/DependencyScanning/DependencyScanningWorker.cpp (+23-24) - (modified) clang/lib/Tooling/DependencyScanningTool.cpp (+94-22) - (modified) clang/test/ClangScanDeps/modules-full-by-mod-name.c (+11) - (modified) clang/test/ClangScanDeps/modules-full-by-mult-mod-names.c (+11) - (modified) clang/tools/clang-scan-deps/ClangScanDeps.cpp (+3-3) ``````````diff diff --git a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h index 352a0ad44fb7f..9e23c0f87f273 100644 --- a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h +++ b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h @@ -17,6 +17,7 @@ #include "clang/Frontend/CompilerInvocation.h" #include "clang/Frontend/TextDiagnosticPrinter.h" #include "clang/Serialization/ObjectFilePCHContainerReader.h" +#include "llvm/Support/VirtualFileSystem.h" namespace clang { class DiagnosticConsumer; @@ -63,15 +64,15 @@ class DependencyScanningAction { std::unique_ptr<DiagnosticOptions> createDiagOptions(ArrayRef<std::string> CommandLine); -struct DignosticsEngineWithDiagOpts { +struct DiagnosticsEngineWithDiagOpts { // We need to bound the lifetime of the DiagOpts used to create the // DiganosticsEngine with the DiagnosticsEngine itself. std::unique_ptr<DiagnosticOptions> DiagOpts; IntrusiveRefCntPtr<DiagnosticsEngine> DiagEngine; - DignosticsEngineWithDiagOpts(ArrayRef<std::string> CommandLine, - IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, - DiagnosticConsumer &DC); + DiagnosticsEngineWithDiagOpts(ArrayRef<std::string> CommandLine, + IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, + DiagnosticConsumer &DC); }; struct TextDiagnosticsPrinterWithOutput { @@ -143,22 +144,11 @@ class CompilerInstanceWithContext { llvm::StringRef CWD; std::vector<std::string> CommandLine; - // Context - file systems - llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS; - // Context - Diagnostics engine. - std::unique_ptr<TextDiagnosticsPrinterWithOutput> DiagPrinterWithOS; - // DiagConsumer may points to DiagPrinterWithOS->DiagPrinter, or a custom - // DiagnosticConsumer passed in from initialize. DiagnosticConsumer *DiagConsumer = nullptr; - std::unique_ptr<DignosticsEngineWithDiagOpts> DiagEngineWithCmdAndOpts; + std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithCmdAndOpts; // Context - compiler invocation - // Compilation's command's arguments may be owned by Alloc when expanded from - // response files, so we need to keep Alloc alive in the context. - llvm::BumpPtrAllocator Alloc; - std::unique_ptr<clang::driver::Driver> Driver; - std::unique_ptr<clang::driver::Compilation> Compilation; std::unique_ptr<CompilerInvocation> OriginalInvocation; // Context - output options @@ -180,15 +170,13 @@ class CompilerInstanceWithContext { : Worker(Worker), CWD(CWD), CommandLine(CMD) {}; // The three methods below returns false when they fail, with the detail - // accumulated in DiagConsumer. - bool initialize(DiagnosticConsumer *DC); + // accumulated in \c DiagEngineWithDiagOpts's diagnostic consumer. + bool initialize( + std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts, + IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS); bool computeDependencies(StringRef ModuleName, DependencyConsumer &Consumer, DependencyActionController &Controller); bool finalize(); - - // The method below turns the return status from the above methods - // into an llvm::Error using a default DiagnosticConsumer. - llvm::Error handleReturnStatus(bool Success); }; } // namespace dependencies } // namespace clang diff --git a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h index ebd7d42786753..489fba4ed3f6b 100644 --- a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h @@ -12,12 +12,14 @@ #include "clang/Basic/DiagnosticOptions.h" #include "clang/Basic/FileManager.h" #include "clang/Basic/LLVM.h" +#include "clang/DependencyScanning/DependencyScannerImpl.h" #include "clang/DependencyScanning/DependencyScanningService.h" #include "clang/DependencyScanning/ModuleDepCollector.h" #include "clang/Frontend/PCHContainerOperations.h" #include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBufferRef.h" +#include "llvm/Support/VirtualFileSystem.h" #include <optional> #include <string> @@ -119,13 +121,28 @@ class DependencyScanningWorker { /// dependency scanning. They together enable the dependency scanning worker /// to more effectively perform scanning for a sequence of modules /// by name when the CWD and CommandLine do not change across the queries. + /// The initialization function asks the client for a DiagnosticsConsumer + /// that it direct the diagnostics to. /// @brief Initializing the context and the compiler instance. /// @param CWD The current working directory used during the scan. /// @param CommandLine The commandline used for the scan. - /// @return Error if the initializaiton fails. - llvm::Error initializeCompilerInstanceWithContextOrError( - StringRef CWD, ArrayRef<std::string> CommandLine); + /// @return False if the initializaiton fails. + bool initializeCompilerInstanceWithContext(StringRef CWD, + ArrayRef<std::string> CommandLine, + DiagnosticConsumer &DC); + + /// @brief Initializing the context and the compiler instance. + /// @param CWD The current working directory used during the scan. + /// @param CommandLine The commandline used for the scan. + /// @param DiagEngineWithCmdAndOpts Preconfigured diagnostics engine and + /// options associated with the cc1 command line. + /// @param FS The overlay file system to use for this compiler instance. + /// @return False if the initializaiton fails. + bool initializeCompilerInstanceWithContext( + StringRef CWD, ArrayRef<std::string> CommandLine, + std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithCmdAndOpts, + IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS); /// @brief Performaces dependency scanning for the module whose name is /// specified. @@ -133,28 +150,15 @@ class DependencyScanningWorker { /// scanned. /// @param Consumer The dependency consumer that stores the results. /// @param Controller The controller for the dependency scanning action. - /// @return Error if the scanner incurs errors. - llvm::Error computeDependenciesByNameWithContextOrError( - StringRef ModuleName, DependencyConsumer &Consumer, - DependencyActionController &Controller); - - /// @brief Finalizes the diagnostics engine and deletes the compiler instance. - /// @return Error if errors occur during finalization. - llvm::Error finalizeCompilerInstanceWithContextOrError(); - - /// The three methods below provides the same functionality as the - /// three methods above. Instead of returning `llvm::Error`s, these - /// three methods return a flag to indicate if the call is successful. - /// The initialization function asks the client for a DiagnosticsConsumer - /// that it direct the diagnostics to. - bool initializeCompilerInstanceWithContext(StringRef CWD, - ArrayRef<std::string> CommandLine, - DiagnosticConsumer *DC = nullptr); + /// @return False if the scanner incurs errors. bool computeDependenciesByNameWithContext(StringRef ModuleName, DependencyConsumer &Consumer, DependencyActionController &Controller); - bool finalizeCompilerInstance(); + + /// @brief Finalizes the diagnostics engine and deletes the compiler instance. + /// @return False if errors occur during finalization. + bool finalizeCompilerInstanceWithContext(); llvm::vfs::FileSystem &getVFS() const { return *DepFS; } diff --git a/clang/include/clang/Tooling/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanningTool.h index 0af07ea8ca97a..44d7a338a87f7 100644 --- a/clang/include/clang/Tooling/DependencyScanningTool.h +++ b/clang/include/clang/Tooling/DependencyScanningTool.h @@ -9,6 +9,7 @@ #ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNINGTOOL_H #define LLVM_CLANG_TOOLING_DEPENDENCYSCANNINGTOOL_H +#include "clang/DependencyScanning/DependencyScannerImpl.h" #include "clang/DependencyScanning/DependencyScanningService.h" #include "clang/DependencyScanning/DependencyScanningUtils.h" #include "clang/DependencyScanning/DependencyScanningWorker.h" @@ -119,9 +120,8 @@ class DependencyScanningTool { /// @param CWD The current working directory used during the scan. /// @param CommandLine The commandline used for the scan. /// @return Error if the initializaiton fails. - llvm::Error - initializeCompilerInstanceWithContext(StringRef CWD, - ArrayRef<std::string> CommandLine); + llvm::Error initializeCompilerInstanceWithContextOrError( + StringRef CWD, ArrayRef<std::string> CommandLine); /// @brief Computes the dependeny for the module named ModuleName. /// @param ModuleName The name of the module for which this method computes @@ -137,8 +137,8 @@ class DependencyScanningTool { /// arguments for dependencies. /// @return An instance of \c TranslationUnitDeps if the scan is successful. /// Otherwise it returns an error. - llvm::Expected<dependencies::TranslationUnitDeps> - computeDependenciesByNameWithContext( + llvm::Expected<clang::dependencies::TranslationUnitDeps> + computeDependenciesByNameWithContextOrError( StringRef ModuleName, const llvm::DenseSet<dependencies::ModuleID> &AlreadySeen, dependencies::LookupModuleOutputCallback LookupModuleOutput); @@ -147,7 +147,7 @@ class DependencyScanningTool { /// diagnostics and deletes the compiler instance. Call this method /// once all names for a same commandline are scanned. /// @return Error if an error occured during finalization. - llvm::Error finalizeCompilerInstanceWithContext(); + llvm::Error finalizeCompilerInstanceWithContextOrError(); llvm::vfs::FileSystem &getWorkerVFS() const { return Worker.getVFS(); } diff --git a/clang/lib/DependencyScanning/DependencyScannerImpl.cpp b/clang/lib/DependencyScanning/DependencyScannerImpl.cpp index acd05cc50daa8..227c2b74eb1c8 100644 --- a/clang/lib/DependencyScanning/DependencyScannerImpl.cpp +++ b/clang/lib/DependencyScanning/DependencyScannerImpl.cpp @@ -13,6 +13,7 @@ #include "clang/Driver/Driver.h" #include "clang/Frontend/FrontendActions.h" #include "llvm/ADT/ScopeExit.h" +#include "llvm/Support/VirtualFileSystem.h" #include "llvm/TargetParser/Host.h" using namespace clang; @@ -367,7 +368,7 @@ dependencies::createDiagOptions(ArrayRef<std::string> CommandLine) { return DiagOpts; } -DignosticsEngineWithDiagOpts::DignosticsEngineWithDiagOpts( +DiagnosticsEngineWithDiagOpts::DiagnosticsEngineWithDiagOpts( ArrayRef<std::string> CommandLine, IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, DiagnosticConsumer &DC) { std::vector<const char *> CCommandLine(CommandLine.size(), nullptr); @@ -713,38 +714,31 @@ bool DependencyScanningAction::runInvocation( return Result; } -bool CompilerInstanceWithContext::initialize(DiagnosticConsumer *DC) { - if (DC) { - DiagConsumer = DC; - } else { - DiagPrinterWithOS = - std::make_unique<TextDiagnosticsPrinterWithOutput>(CommandLine); - DiagConsumer = &DiagPrinterWithOS->DiagPrinter; +bool CompilerInstanceWithContext::initialize( + std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts, + IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS) { + assert(DiagEngineWithDiagOpts && "Valid diagnostics engine required!"); + DiagEngineWithCmdAndOpts = std::move(DiagEngineWithDiagOpts); + DiagConsumer = DiagEngineWithDiagOpts->DiagEngine->getClient(); + + IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS = Worker.DepFS; + if (OverlayFS) { +#ifndef NDEBUG + bool SawDepFS = false; + OverlayFS->visit([&](llvm::vfs::FileSystem &VFS) { + SawDepFS |= &VFS == Worker.DepFS.get(); + }); + assert(SawDepFS && "OverlayFS not based on DepFS"); +#endif + FS = std::move(OverlayFS); } - std::tie(OverlayFS, CommandLine) = initVFSForByNameScanning( - Worker.DepFS, CommandLine, CWD, "ScanningByName"); - - DiagEngineWithCmdAndOpts = std::make_unique<DignosticsEngineWithDiagOpts>( - CommandLine, OverlayFS, *DiagConsumer); - - std::tie(Driver, Compilation) = buildCompilation( - CommandLine, *DiagEngineWithCmdAndOpts->DiagEngine, OverlayFS, Alloc); - - if (!Compilation) - return false; + // Reset what might have been modified in the previous worker invocation. + FS->setCurrentWorkingDirectory(CWD); - assert(Compilation->getJobs().size() && - "Must have a job list of non-zero size"); - const driver::Command &Command = *(Compilation->getJobs().begin()); - const auto &CommandArgs = Command.getArguments(); - assert(!CommandArgs.empty() && "Cannot have a command with 0 args"); - assert(StringRef(CommandArgs[0]) == "-cc1" && "Requires a cc1 job."); - OriginalInvocation = std::make_unique<CompilerInvocation>(); - - if (!CompilerInvocation::CreateFromArgs(*OriginalInvocation, CommandArgs, - *DiagEngineWithCmdAndOpts->DiagEngine, - Command.getExecutable())) { + OriginalInvocation = createCompilerInvocation( + CommandLine, *DiagEngineWithCmdAndOpts->DiagEngine); + if (!OriginalInvocation) { DiagEngineWithCmdAndOpts->DiagEngine->Report( diag::err_fe_expected_compiler_job) << llvm::join(CommandLine, " "); @@ -763,7 +757,7 @@ bool CompilerInstanceWithContext::initialize(DiagnosticConsumer *DC) { auto &CI = *CIPtr; if (!initializeScanCompilerInstance( - CI, OverlayFS, DiagEngineWithCmdAndOpts->DiagEngine->getClient(), + CI, FS, DiagEngineWithCmdAndOpts->DiagEngine->getClient(), Worker.Service, Worker.DepFS)) return false; @@ -876,11 +870,3 @@ bool CompilerInstanceWithContext::finalize() { DiagConsumer->finish(); return true; } - -llvm::Error CompilerInstanceWithContext::handleReturnStatus(bool Success) { - assert(DiagPrinterWithOS && "Must use the default DiagnosticConsumer."); - return Success ? llvm::Error::success() - : llvm::make_error<llvm::StringError>( - DiagPrinterWithOS->DiagnosticsOS.str(), - llvm::inconvertibleErrorCode()); -} diff --git a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp index 7b03abd8e3138..ef16b14e7cc6e 100644 --- a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp @@ -7,10 +7,13 @@ //===----------------------------------------------------------------------===// #include "clang/DependencyScanning/DependencyScanningWorker.h" +#include "clang/Basic/Diagnostic.h" #include "clang/Basic/DiagnosticFrontend.h" #include "clang/DependencyScanning/DependencyScannerImpl.h" #include "clang/Driver/Driver.h" #include "clang/Driver/Tool.h" +#include "clang/Serialization/ObjectFilePCHContainerReader.h" +#include "llvm/Support/VirtualFileSystem.h" using namespace clang; using namespace dependencies; @@ -100,7 +103,7 @@ bool DependencyScanningWorker::scanDependencies( FS = std::move(OverlayFS); } - DignosticsEngineWithDiagOpts DiagEngineWithCmdAndOpts(CommandLine, FS, DC); + DiagnosticsEngineWithDiagOpts DiagEngineWithCmdAndOpts(CommandLine, FS, DC); DependencyScanningAction Action(Service, WorkingDirectory, Consumer, Controller, DepFS); @@ -165,33 +168,29 @@ bool DependencyScanningWorker::computeDependencies( DC); } -llvm::Error -DependencyScanningWorker::initializeCompilerInstanceWithContextOrError( - StringRef CWD, ArrayRef<std::string> CommandLine) { - bool Success = initializeCompilerInstanceWithContext(CWD, CommandLine); - return CIWithContext->handleReturnStatus(Success); -} - -llvm::Error -DependencyScanningWorker::computeDependenciesByNameWithContextOrError( - StringRef ModuleName, DependencyConsumer &Consumer, - DependencyActionController &Controller) { - bool Success = - computeDependenciesByNameWithContext(ModuleName, Consumer, Controller); - return CIWithContext->handleReturnStatus(Success); -} - -llvm::Error -DependencyScanningWorker::finalizeCompilerInstanceWithContextOrError() { - bool Success = finalizeCompilerInstance(); - return CIWithContext->handleReturnStatus(Success); +bool DependencyScanningWorker::initializeCompilerInstanceWithContext( + StringRef CWD, ArrayRef<std::string> CommandLine, DiagnosticConsumer &DC) { + auto OverlayFSAndArgs = + initVFSForByNameScanning(DepFS, CommandLine, CWD, "ScanningByName"); + auto &OverlayFS = OverlayFSAndArgs.first; + const auto &ModifiedCommandLine = OverlayFSAndArgs.second; + + auto DiagEngineWithCmdAndOpts = + std::make_unique<DiagnosticsEngineWithDiagOpts>(ModifiedCommandLine, + OverlayFS, DC); + + return initializeCompilerInstanceWithContext( + CWD, ModifiedCommandLine, std::move(DiagEngineWithCmdAndOpts), OverlayFS); } bool DependencyScanningWorker::initializeCompilerInstanceWithContext( - StringRef CWD, ArrayRef<std::string> CommandLine, DiagnosticConsumer *DC) { + StringRef CWD, ArrayRef<std::string> CommandLine, + std::unique_ptr<DiagnosticsEngineWithDiagOpts> DiagEngineWithDiagOpts, + IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS) { CIWithContext = std::make_unique<CompilerInstanceWithContext>(*this, CWD, CommandLine); - return CIWithContext->initialize(DC); + return CIWithContext->initialize(std::move(DiagEngineWithDiagOpts), + OverlayFS); } bool DependencyScanningWorker::computeDependenciesByNameWithContext( @@ -201,6 +200,6 @@ bool DependencyScanningWorker::computeDependenciesByNameWithContext( return CIWithContext->computeDependencies(ModuleName, Consumer, Controller); } -bool DependencyScanningWorker::finalizeCompilerInstance() { +bool DependencyScanningWorker::finalizeCompilerInstanceWithContext() { return CIWithContext->finalize(); } diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp index 9c0b095705d49..6be90597d6a2d 100644 --- a/clang/lib/Tooling/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanningTool.cpp @@ -7,7 +7,13 @@ //===----------------------------------------------------------------------===// #include "clang/Tooling/DependencyScanningTool.h" +#include "clang/Basic/DiagnosticFrontend.h" +#include "clang/DependencyScanning/DependencyScannerImpl.h" +#include "clang/Driver/Driver.h" +#include "clang/Driver/Tool.h" #include "clang/Frontend/Utils.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/TargetParser/Host.h" #include <optional> using namespace clang; @@ -161,43 +167,109 @@ DependencyScanningTool::getModuleDependencies( StringRef ModuleName, ArrayRef<std::string> CommandLine, StringRef CWD, const llvm::DenseSet<ModuleID> &AlreadySeen, LookupModuleOutputCallback LookupModuleOutput) { - FullDependencyConsumer Consumer(AlreadySeen); - CallbackActionController Controller(LookupModuleOutput); if (auto Error = - Worker.initializeCompilerInstanceWithContextOrError(CWD, CommandLine)) - return std::move(Error); + initializeCompilerInstanceWithContextOrError(CWD, CommandLine)) + return Error; - auto Result = Worker.computeDependenciesByNameWithContextOrError( - ModuleName, Consumer, Controller); + auto Result = computeDependenciesByNameWithContextOrError( + ModuleName, AlreadySeen, LookupModuleOutput); - if (auto Error = Worker.finalizeCompilerInstanceWithContextOrError()) - return std::move(Error); + if (auto Error = finalizeCompilerInstanceWithContextOrError()) + return Error; - if (Result) - return std::move(Result); + return Result; +} - return ... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/171238 _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
