llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-clang Author: Qiongsi Wu (qiongsiwu) <details> <summary>Changes</summary> This PR follows https://github.com/llvm/llvm-project/pull/160795, and it is the second of a series of planned PRs to land https://github.com/llvm/llvm-project/pull/160207 in smaller pieces. The initialization steps within `DependencyScanningAction::runInvocation` are broken up in to several helper functions. The intention is to reuse the helper functions in a followup PR to initialize the `CompilerInstanceWithContext`. This PR is an NFC in spirit but it changes some execution orders in `DependencyScanningAction::runInvocation`. The code to initialize the scanner instance's option flags are collected together in `initializeScanCompilerInstance`, in contrast to the current way where the flags are set in piece-meal fashion before and after we compute the `StableDirs` and the `rebuiltModulesASTMap`. Part of work for rdar://136303612. --- Patch is 25.08 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/161300.diff 3 Files Affected: - (modified) clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp (+183-43) - (modified) clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h (+52-1) - (modified) clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp (+19-84) ``````````diff diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp index d370bfd0dd10f..5a16edf859193 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp @@ -9,8 +9,10 @@ #include "DependencyScannerImpl.h" #include "clang/Basic/DiagnosticFrontend.h" #include "clang/Basic/DiagnosticSerialization.h" +#include "clang/Driver/Driver.h" #include "clang/Frontend/FrontendActions.h" #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" +#include "llvm/TargetParser/Host.h" using namespace clang; using namespace tooling; @@ -334,6 +336,17 @@ class ScanningDependencyDirectivesGetter : public DependencyDirectivesGetter { }; } // namespace +std::unique_ptr<DiagnosticOptions> +clang::tooling::dependencies::createDiagOptions( + const std::vector<std::string> &CommandLine) { + std::vector<const char *> CLI; + for (const std::string &Arg : CommandLine) + CLI.push_back(Arg.c_str()); + auto DiagOpts = CreateAndPopulateDiagOpts(CLI); + sanitizeDiagOpts(*DiagOpts); + return DiagOpts; +} + /// Sanitize diagnostic options for dependency scan. void clang::tooling::dependencies::sanitizeDiagOpts( DiagnosticOptions &DiagOpts) { @@ -356,43 +369,98 @@ void clang::tooling::dependencies::sanitizeDiagOpts( }); } -bool DependencyScanningAction::runInvocation( - std::shared_ptr<CompilerInvocation> Invocation, - IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, - std::shared_ptr<PCHContainerOperations> PCHContainerOps, - DiagnosticConsumer *DiagConsumer) { - // Making sure that we canonicalize the defines before we create the deep - // copy to avoid unnecessary variants in the scanner and in the resulting - // explicit command lines. - if (any(Service.getOptimizeArgs() & ScanningOptimizations::Macros)) - canonicalizeDefines(Invocation->getPreprocessorOpts()); +std::pair<std::unique_ptr<driver::Driver>, std::unique_ptr<driver::Compilation>> +clang::tooling::dependencies::buildCompilation( + ArrayRef<std::string> ArgStrs, DiagnosticsEngine &Diags, + IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS) { + SmallVector<const char *, 256> Argv; + Argv.reserve(ArgStrs.size()); + for (const std::string &Arg : ArgStrs) + Argv.push_back(Arg.c_str()); + + std::unique_ptr<driver::Driver> Driver = std::make_unique<driver::Driver>( + Argv[0], llvm::sys::getDefaultTargetTriple(), Diags, + "clang LLVM compiler", FS); + Driver->setTitle("clang_based_tool"); + + llvm::BumpPtrAllocator Alloc; + bool CLMode = driver::IsClangCL( + driver::getDriverMode(Argv[0], ArrayRef(Argv).slice(1))); + + if (llvm::Error E = + driver::expandResponseFiles(Argv, CLMode, Alloc, FS.get())) { + Diags.Report(diag::err_drv_expand_response_file) + << llvm::toString(std::move(E)); + return std::make_pair(nullptr, nullptr); + } - // Make a deep copy of the original Clang invocation. - CompilerInvocation OriginalInvocation(*Invocation); + std::unique_ptr<driver::Compilation> Compilation( + Driver->BuildCompilation(llvm::ArrayRef(Argv))); + if (!Compilation) + return std::make_pair(nullptr, nullptr); - if (Scanned) { - // Scanning runs once for the first -cc1 invocation in a chain of driver - // jobs. For any dependent jobs, reuse the scanning result and just - // update the LastCC1Arguments to correspond to the new invocation. - // FIXME: to support multi-arch builds, each arch requires a separate scan - setLastCC1Arguments(std::move(OriginalInvocation)); - return true; - } + if (Compilation->containsError()) + return std::make_pair(nullptr, nullptr); + ; - Scanned = true; + return std::make_pair(std::move(Driver), std::move(Compilation)); +} - // Create a compiler instance to handle the actual work. - auto ModCache = makeInProcessModuleCache(Service.getModuleCacheEntries()); - ScanInstanceStorage.emplace(std::move(Invocation), std::move(PCHContainerOps), - ModCache.get()); - CompilerInstance &ScanInstance = *ScanInstanceStorage; +llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> +clang::tooling::dependencies::initVFSForByNameScanning( + llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS, + std::vector<std::string> &CommandLine, StringRef WorkingDirectory, + StringRef FakeFileNamePrefix) { + // Reset what might have been modified in the previous worker invocation. + BaseFS->setCurrentWorkingDirectory(WorkingDirectory); + + // If we're scanning based on a module name alone, we don't expect the client + // to provide us with an input file. However, the driver really wants to have + // one. Let's just make it up to make the driver happy. + auto OverlayFS = + llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS); + auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>(); + InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory); + SmallString<128> FakeInputPath; + // TODO: We should retry the creation if the path already exists. + llvm::sys::fs::createUniquePath(FakeFileNamePrefix + "-%%%%%%%%.input", + FakeInputPath, + /*MakeAbsolute=*/false); + InMemoryFS->addFile(FakeInputPath, 0, llvm::MemoryBuffer::getMemBuffer("")); + llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay = InMemoryFS; + OverlayFS->pushOverlay(InMemoryOverlay); + + CommandLine.emplace_back(FakeInputPath); + + return OverlayFS; +} + +std::unique_ptr<CompilerInvocation> +clang::tooling::dependencies::createCompilerInvocation( + const std::vector<std::string> CommandLine, DiagnosticsEngine &Diags) { + llvm::opt::ArgStringList Argv; + for (const std::string &Str : ArrayRef(CommandLine).drop_front()) + Argv.push_back(Str.c_str()); + + auto Invocation = std::make_unique<CompilerInvocation>(); + if (!CompilerInvocation::CreateFromArgs(*Invocation, Argv, Diags)) { + // FIXME: Should we just go on like cc1_main does? + return nullptr; + } + return Invocation; +} + +bool clang::tooling::dependencies::initializeScanCompilerInstance( + CompilerInstance &ScanInstance, + IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, + DiagnosticConsumer *DiagConsumer, DependencyScanningService &Service, + llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS) { ScanInstance.setBuildingModule(false); ScanInstance.createVirtualFileSystem(FS, DiagConsumer); // Create the compiler's actual diagnostics engine. sanitizeDiagOpts(ScanInstance.getDiagnosticOpts()); - assert(!DiagConsumerFinished && "attempt to reuse finished consumer"); ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false); if (!ScanInstance.hasDiagnostics()) return false; @@ -434,6 +502,26 @@ bool DependencyScanningAction::runInvocation( ScanInstance.createSourceManager(*FileMgr); + // Consider different header search and diagnostic options to create + // different modules. This avoids the unsound aliasing of module PCMs. + // + // TODO: Implement diagnostic bucketing to reduce the impact of strict + // context hashing. + ScanInstance.getHeaderSearchOpts().ModulesStrictContextHash = true; + ScanInstance.getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true; + ScanInstance.getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true; + ScanInstance.getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true; + ScanInstance.getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings = true; + ScanInstance.getHeaderSearchOpts().ModulesForceValidateUserHeaders = false; + + // Avoid some checks and module map parsing when loading PCM files. + ScanInstance.getPreprocessorOpts().ModulesCheckRelocated = false; + + return true; +} + +llvm::SmallVector<StringRef> clang::tooling::dependencies::computeStableDirs( + CompilerInstance &ScanInstance) { // Create a collection of stable directories derived from the ScanInstance // for determining whether module dependencies would fully resolve from // those directories. @@ -441,7 +529,12 @@ bool DependencyScanningAction::runInvocation( const StringRef Sysroot = ScanInstance.getHeaderSearchOpts().Sysroot; if (!Sysroot.empty() && (llvm::sys::path::root_directory(Sysroot) != Sysroot)) StableDirs = {Sysroot, ScanInstance.getHeaderSearchOpts().ResourceDir}; + return StableDirs; +} +std::optional<PrebuiltModulesAttrsMap> +clang::tooling::dependencies::computePrebuiltModulesASTMap( + CompilerInstance &ScanInstance, llvm::SmallVector<StringRef> &StableDirs) { // Store a mapping of prebuilt module files and their properties like header // search options. This will prevent the implicit build to create duplicate // modules and will force reuse of the existing prebuilt module files @@ -453,8 +546,18 @@ bool DependencyScanningAction::runInvocation( ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance, ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles, PrebuiltModulesASTMap, ScanInstance.getDiagnostics(), StableDirs)) - return false; + return {}; + + return PrebuiltModulesASTMap; +} +void clang::tooling::dependencies::initializeModuleDepCollector( + CompilerInstance &ScanInstance, std::shared_ptr<ModuleDepCollector> &MDC, + StringRef WorkingDirectory, DependencyConsumer &Consumer, + DependencyScanningService &Service, CompilerInvocation &Inv, + DependencyActionController &Controller, + PrebuiltModulesAttrsMap PrebuiltModulesASTMap, + llvm::SmallVector<StringRef> &StableDirs) { // Create the dependency collector that will collect the produced // dependencies. // @@ -463,6 +566,11 @@ bool DependencyScanningAction::runInvocation( // which ensures that the compiler won't create new dependency collectors, // and thus won't write out the extra '.d' files to disk. auto Opts = std::make_unique<DependencyOutputOptions>(); + + // We rely on the behaviour that that ScanInstance's Invocation instance's + // dependency output opts are cleared here. + // TODO: we will need to preserve and recover the original dependency output + // opts if we want to reuse ScanInstance. std::swap(*Opts, ScanInstance.getInvocation().getDependencyOutputOpts()); // We need at least one -MT equivalent for the generator of make dependency // files to work. @@ -480,26 +588,58 @@ bool DependencyScanningAction::runInvocation( case ScanningOutputFormat::P1689: case ScanningOutputFormat::Full: MDC = std::make_shared<ModuleDepCollector>( - Service, std::move(Opts), ScanInstance, Consumer, Controller, - OriginalInvocation, std::move(PrebuiltModulesASTMap), StableDirs); + Service, std::move(Opts), ScanInstance, Consumer, Controller, Inv, + std::move(PrebuiltModulesASTMap), StableDirs); ScanInstance.addDependencyCollector(MDC); break; } +} - // Consider different header search and diagnostic options to create - // different modules. This avoids the unsound aliasing of module PCMs. - // - // TODO: Implement diagnostic bucketing to reduce the impact of strict - // context hashing. - ScanInstance.getHeaderSearchOpts().ModulesStrictContextHash = true; - ScanInstance.getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true; - ScanInstance.getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true; - ScanInstance.getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true; - ScanInstance.getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings = true; - ScanInstance.getHeaderSearchOpts().ModulesForceValidateUserHeaders = false; +bool DependencyScanningAction::runInvocation( + std::unique_ptr<CompilerInvocation> Invocation, + IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, + std::shared_ptr<PCHContainerOperations> PCHContainerOps, + DiagnosticConsumer *DiagConsumer) { + // Making sure that we canonicalize the defines before we create the deep + // copy to avoid unnecessary variants in the scanner and in the resulting + // explicit command lines. + if (any(Service.getOptimizeArgs() & ScanningOptimizations::Macros)) + canonicalizeDefines(Invocation->getPreprocessorOpts()); - // Avoid some checks and module map parsing when loading PCM files. - ScanInstance.getPreprocessorOpts().ModulesCheckRelocated = false; + // Make a deep copy of the original Clang invocation. + CompilerInvocation OriginalInvocation(*Invocation); + + if (Scanned) { + // Scanning runs once for the first -cc1 invocation in a chain of driver + // jobs. For any dependent jobs, reuse the scanning result and just + // update the LastCC1Arguments to correspond to the new invocation. + // FIXME: to support multi-arch builds, each arch requires a separate scan + setLastCC1Arguments(std::move(OriginalInvocation)); + return true; + } + + Scanned = true; + + // Create a compiler instance to handle the actual work. + auto ModCache = makeInProcessModuleCache(Service.getModuleCacheEntries()); + ScanInstanceStorage.emplace(std::move(Invocation), std::move(PCHContainerOps), + ModCache.get()); + CompilerInstance &ScanInstance = *ScanInstanceStorage; + + assert(!DiagConsumerFinished && "attempt to reuse finished consumer"); + if (!initializeScanCompilerInstance(ScanInstance, FS, DiagConsumer, Service, + DepFS)) + return false; + + llvm::SmallVector<StringRef> StableDirs = computeStableDirs(ScanInstance); + auto MaybePrebuiltModulesASTMap = + computePrebuiltModulesASTMap(ScanInstance, StableDirs); + if (!MaybePrebuiltModulesASTMap) + return false; + + initializeModuleDepCollector(ScanInstance, MDC, WorkingDirectory, Consumer, + Service, OriginalInvocation, Controller, + *MaybePrebuiltModulesASTMap, StableDirs); std::unique_ptr<FrontendAction> Action; diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h b/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h index 32fbcfffde53c..4e01419b3af8e 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h +++ b/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h @@ -9,8 +9,10 @@ #ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNER_H #define LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNER_H +#include "clang/Driver/Compilation.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/CompilerInvocation.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" #include "clang/Serialization/ObjectFilePCHContainerReader.h" #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h" #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h" @@ -35,7 +37,7 @@ class DependencyScanningAction { : Service(Service), WorkingDirectory(WorkingDirectory), Consumer(Consumer), Controller(Controller), DepFS(std::move(DepFS)), ModuleName(ModuleName) {} - bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation, + bool runInvocation(std::unique_ptr<CompilerInvocation> Invocation, IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, std::shared_ptr<PCHContainerOperations> PCHContainerOps, DiagnosticConsumer *DiagConsumer); @@ -73,8 +75,57 @@ class DependencyScanningAction { }; // Helper functions +std::unique_ptr<DiagnosticOptions> +createDiagOptions(const std::vector<std::string> &CommandLine); void sanitizeDiagOpts(DiagnosticOptions &DiagOpts); +struct TextDiagnosticsPrinterWithOutput { + std::string DiagnosticOutput; + llvm::raw_string_ostream DiagnosticsOS; + std::unique_ptr<clang::DiagnosticOptions> DiagOpts; + TextDiagnosticPrinter DiagPrinter; + + TextDiagnosticsPrinterWithOutput(const std::vector<std::string> &CommandLine) + : DiagnosticsOS(DiagnosticOutput), + DiagOpts(createDiagOptions(CommandLine)), + DiagPrinter(DiagnosticsOS, *DiagOpts) {} +}; + +std::pair<std::unique_ptr<driver::Driver>, std::unique_ptr<driver::Compilation>> +buildCompilation(ArrayRef<std::string> ArgStrs, DiagnosticsEngine &Diags, + IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS); + +std::unique_ptr<CompilerInvocation> +createCompilerInvocation(const std::vector<std::string> CommandLine, + DiagnosticsEngine &Diags); + +llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> +initVFSForByNameScanning(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS, + std::vector<std::string> &CommandLine, + StringRef WorkingDirectory, + StringRef FakeFileNamePrefix); + +bool initializeScanCompilerInstance( + CompilerInstance &ScanInstance, + IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, + DiagnosticConsumer *DiagConsumer, DependencyScanningService &Service, + llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS); + +llvm::SmallVector<StringRef> computeStableDirs(CompilerInstance &ScanInstance); + +std::optional<PrebuiltModulesAttrsMap> +computePrebuiltModulesASTMap(CompilerInstance &ScanInstance, + llvm::SmallVector<StringRef> &StableDirs); + +void initializeModuleDepCollector(CompilerInstance &ScanInstance, + std::shared_ptr<ModuleDepCollector> &MDC, + StringRef WorkingDirectory, + DependencyConsumer &Consumer, + DependencyScanningService &Service, + CompilerInvocation &Inv, + DependencyActionController &Controller, + PrebuiltModulesAttrsMap PrebuiltModulesASTMap, + llvm::SmallVector<StringRef> &StableDirs); } // namespace dependencies } // namespace tooling } // namespace clang diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index 796e587ba9147..ddc5e9f26f4c3 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -63,32 +63,19 @@ DependencyScanningWorker::DependencyScanningWorker( } } -static std::unique_ptr<DiagnosticOptions> -createDiagOptions(const std::vector<std::string> &CommandLine) { - std::vector<const char *> CLI; - for (const std::string &Arg : CommandLine) - CLI.push_back(Arg.c_str()); - auto DiagOpts = CreateAndPopulateDiagOpts(CLI); - sanitizeDiagOpts(*DiagOpts); - return DiagOpts; -} - llvm::Error DependencyScanningWorker::computeDependencies( StringRef WorkingDirectory, const std::vector<std::string> &CommandLine, DependencyConsumer &Consumer, DependencyActionController &Controller, std::optional<llvm::MemoryBufferRef> TUBuffer) { // Capture the emitted diagnostics and report them to the client // in the case of a failure. - std::string DiagnosticOutput; - llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput); - auto DiagOpts = createDiagOptions(CommandLine); - TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, *DiagOpts); + TextDiagnosticsPrinterWithOutput DiagPrinterWithOS(CommandLine); if (computeDependencies(WorkingDirectory, CommandLine, Consumer, Controller, - DiagPrinter, TUBuffer)) + DiagPrinterWithOS.DiagPrinter, TUBuffer)) return llvm::Error::success(); - return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(), - llvm::inconvertibleErrorCode()); + return llvm::make_error<llvm::StringError>( + DiagPrinterWithOS.DiagnosticsOS.str(), llvm::inconvertibleErrorCode()); } llvm::Error DependencyScanningWorker::computeDependencies( @@ -97,51 +84,24 @@ llvm::Error DependencyScanningWorker::computeDependencies( StringRef ModuleName) { // Capture the emitted diagnostics and report them to the client // in the case of a failure. - std::string DiagnosticOutput; - llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput); - auto DiagOpts = createDiagOptions(CommandLine); - TextDiagnosticPrinter DiagPrinter(D... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/161300 _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
