llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-clang Author: Thurston Dang (thurstond) <details> <summary>Changes</summary> Reverts llvm/llvm-project#<!-- -->183396 due to buildbot breakage (reported earlier in https://github.com/llvm/llvm-project/pull/183396#issuecomment-4015167408) --- Full diff: https://github.com/llvm/llvm-project/pull/185150.diff 6 Files Affected: - (modified) clang/include/clang/DependencyScanning/DependencyScannerImpl.h (+1) - (modified) clang/include/clang/DependencyScanning/DependencyScanningWorker.h (+14) - (modified) clang/include/clang/Tooling/DependencyScanningTool.h (-10) - (modified) clang/lib/DependencyScanning/DependencyScanningWorker.cpp (+55) - (modified) clang/lib/Tooling/DependencyScanningTool.cpp (+2-64) - (modified) clang/test/ClangScanDeps/modules-full-by-mult-mod-names-diagnostics.c (+3-3) ``````````diff diff --git a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h index a54a6269dbdc4..24cf1afaaa450 100644 --- a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h +++ b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h @@ -16,6 +16,7 @@ #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/CompilerInvocation.h" #include "clang/Frontend/TextDiagnosticPrinter.h" +#include "clang/Serialization/ObjectFilePCHContainerReader.h" #include "llvm/Support/VirtualFileSystem.h" namespace clang { diff --git a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h index 92da219d85d56..0a5c4936f7fe7 100644 --- a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h @@ -140,6 +140,20 @@ class DependencyScanningWorker { friend tooling::CompilerInstanceWithContext; }; + +std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>, + std::vector<std::string>> +initVFSForTUBufferScanning(IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS, + ArrayRef<std::string> CommandLine, + StringRef WorkingDirectory, + llvm::MemoryBufferRef TUBuffer); + +std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>, + std::vector<std::string>> +initVFSForByNameScanning(IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS, + ArrayRef<std::string> CommandLine, + StringRef WorkingDirectory, StringRef ModuleName); + } // end namespace dependencies } // end namespace clang diff --git a/clang/include/clang/Tooling/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanningTool.h index 30846ae0ebf3e..40e791fd5bff5 100644 --- a/clang/include/clang/Tooling/DependencyScanningTool.h +++ b/clang/include/clang/Tooling/DependencyScanningTool.h @@ -226,16 +226,6 @@ class CompilerInstanceWithContext { StringRef ModuleName, const llvm::DenseSet<dependencies::ModuleID> &AlreadySeen, dependencies::LookupModuleOutputCallback LookupModuleOutput); - - // MaxNumOfQueries is the upper limit of the number of names the by-name - // scanning API (computeDependencies) can support after a - // CompilerInstanceWithContext is initialized. At the time of this commit, the - // estimated number of total unique importable names is around 3000 from - // Apple's SDKs. We usually import them in parallel, so it is unlikely that - // all names are all scanned by the same dependency scanning worker. Therefore - // the 64k (20x bigger than our estimate) size is sufficient to hold the - // unique source locations to report diagnostics per worker. - static const int32_t MaxNumOfQueries = 1 << 16; }; } // end namespace tooling diff --git a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp index 75eb821ac651a..7a7f34a43b338 100644 --- a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp @@ -108,3 +108,58 @@ bool DependencyScanningWorker::computeDependencies( return Success && Action.hasScanned(); } + +std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>, + std::vector<std::string>> +dependencies::initVFSForTUBufferScanning( + IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS, + ArrayRef<std::string> CommandLine, StringRef WorkingDirectory, + llvm::MemoryBufferRef TUBuffer) { + // Reset what might have been modified in the previous worker invocation. + BaseFS->setCurrentWorkingDirectory(WorkingDirectory); + + auto OverlayFS = + llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS); + auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>(); + InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory); + auto InputPath = TUBuffer.getBufferIdentifier(); + InMemoryFS->addFile( + InputPath, 0, llvm::MemoryBuffer::getMemBufferCopy(TUBuffer.getBuffer())); + IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay = InMemoryFS; + + OverlayFS->pushOverlay(InMemoryOverlay); + std::vector<std::string> ModifiedCommandLine(CommandLine); + ModifiedCommandLine.emplace_back(InputPath); + + return std::make_pair(OverlayFS, ModifiedCommandLine); +} + +std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>, + std::vector<std::string>> +dependencies::initVFSForByNameScanning( + IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS, + ArrayRef<std::string> CommandLine, StringRef WorkingDirectory, + StringRef ModuleName) { + // Reset what might have been modified in the previous worker invocation. + BaseFS->setCurrentWorkingDirectory(WorkingDirectory); + + // If we're scanning based on a module name alone, we don't expect the client + // to provide us with an input file. However, the driver really wants to have + // one. Let's just make it up to make the driver happy. + auto OverlayFS = + llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS); + auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>(); + InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory); + SmallString<128> FakeInputPath; + // TODO: We should retry the creation if the path already exists. + llvm::sys::fs::createUniquePath(ModuleName + "-%%%%%%%%.input", FakeInputPath, + /*MakeAbsolute=*/false); + InMemoryFS->addFile(FakeInputPath, 0, llvm::MemoryBuffer::getMemBuffer("")); + IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay = InMemoryFS; + OverlayFS->pushOverlay(InMemoryOverlay); + + std::vector<std::string> ModifiedCommandLine(CommandLine); + ModifiedCommandLine.emplace_back(FakeInputPath); + + return std::make_pair(OverlayFS, ModifiedCommandLine); +} diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp index 147588eff59c7..b570bc8e5ae30 100644 --- a/clang/lib/Tooling/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanningTool.cpp @@ -253,65 +253,6 @@ std::optional<P1689Rule> DependencyScanningTool::getP1689ModuleDependencyFile( return Rule; } -static std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>, - std::vector<std::string>> -initVFSForTUBufferScanning(IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS, - ArrayRef<std::string> CommandLine, - StringRef WorkingDirectory, - llvm::MemoryBufferRef TUBuffer) { - // Reset what might have been modified in the previous worker invocation. - BaseFS->setCurrentWorkingDirectory(WorkingDirectory); - - auto OverlayFS = - llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS); - auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>(); - InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory); - auto InputPath = TUBuffer.getBufferIdentifier(); - InMemoryFS->addFile( - InputPath, 0, llvm::MemoryBuffer::getMemBufferCopy(TUBuffer.getBuffer())); - IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay = InMemoryFS; - - OverlayFS->pushOverlay(InMemoryOverlay); - std::vector<std::string> ModifiedCommandLine(CommandLine); - ModifiedCommandLine.emplace_back(InputPath); - - return std::make_pair(OverlayFS, ModifiedCommandLine); -} - -// The fake input buffer is read-only, and it is used to produce -// unique source locations for the diagnostics. Therefore sharing -// this global buffer across threads is ok. -static const std::string - FakeInput(" ", - clang::tooling::CompilerInstanceWithContext::MaxNumOfQueries); - -static std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>, - std::vector<std::string>> -initVFSForByNameScanning(IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS, - ArrayRef<std::string> CommandLine, - StringRef WorkingDirectory) { - // Reset what might have been modified in the previous worker invocation. - BaseFS->setCurrentWorkingDirectory(WorkingDirectory); - - // If we're scanning based on a module name alone, we don't expect the client - // to provide us with an input file. However, the driver really wants to have - // one. Let's just make it up to make the driver happy. - auto OverlayFS = - llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS); - auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>(); - InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory); - StringRef FakeInputPath("module-include.input"); - InMemoryFS->addFile(FakeInputPath, 0, - llvm::MemoryBuffer::getMemBuffer(FakeInput)); - IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay = InMemoryFS; - OverlayFS->pushOverlay(InMemoryOverlay); - - std::vector<std::string> ModifiedCommandLine(CommandLine); - ModifiedCommandLine.emplace_back(FakeInputPath); - - return std::make_pair(OverlayFS, ModifiedCommandLine); -} - std::optional<TranslationUnitDeps> DependencyScanningTool::getTranslationUnitDependencies( ArrayRef<std::string> CommandLine, StringRef CWD, @@ -379,8 +320,8 @@ std::optional<CompilerInstanceWithContext> CompilerInstanceWithContext::initializeFromCommandline( DependencyScanningTool &Tool, StringRef CWD, ArrayRef<std::string> CommandLine, DiagnosticConsumer &DC) { - auto [OverlayFS, ModifiedCommandLine] = - initVFSForByNameScanning(&Tool.Worker.getVFS(), CommandLine, CWD); + auto [OverlayFS, ModifiedCommandLine] = initVFSForByNameScanning( + &Tool.Worker.getVFS(), CommandLine, CWD, "ScanningByName"); auto DiagEngineWithCmdAndOpts = std::make_unique<DiagnosticsEngineWithDiagOpts>(ModifiedCommandLine, OverlayFS, DC); @@ -505,9 +446,6 @@ bool CompilerInstanceWithContext::initialize( bool CompilerInstanceWithContext::computeDependencies( StringRef ModuleName, DependencyConsumer &Consumer, DependencyActionController &Controller) { - if (SrcLocOffset >= MaxNumOfQueries) - llvm::report_fatal_error("exceeded maximum by-name scans for worker"); - assert(CIPtr && "CIPtr must be initialized before calling this method"); auto &CI = *CIPtr; diff --git a/clang/test/ClangScanDeps/modules-full-by-mult-mod-names-diagnostics.c b/clang/test/ClangScanDeps/modules-full-by-mult-mod-names-diagnostics.c index f99690c3a7715..9889982354e90 100644 --- a/clang/test/ClangScanDeps/modules-full-by-mult-mod-names-diagnostics.c +++ b/clang/test/ClangScanDeps/modules-full-by-mult-mod-names-diagnostics.c @@ -27,11 +27,11 @@ module root2 { header "root2.h" } // RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s // ERROR: Error while scanning dependencies for modA: -// ERROR-NEXT: module-include.input:1:1: fatal error: module 'modA' not found +// ERROR-NEXT: {{.*}}: fatal error: module 'modA' not found // ERROR-NEXT: Error while scanning dependencies for modB: -// ERROR-NEXT: module-include.input:1:3: fatal error: module 'modB' not found +// ERROR-NEXT: {{.*}}: fatal error: module 'modB' not found // ERROR-NEXT: Error while scanning dependencies for modC: -// ERROR-NEXT: module-include.input:1:4: fatal error: module 'modC' not found +// ERROR-NEXT: {{.*}}: fatal error: module 'modC' not found // CHECK: { // CHECK-NEXT: "modules": [ // CHECK-NEXT: { `````````` </details> https://github.com/llvm/llvm-project/pull/185150 _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
