jansvoboda11 created this revision.
jansvoboda11 added a reviewer: benlangmuir.
Herald added a subscriber: ributzka.
Herald added a project: All.
jansvoboda11 requested review of this revision.
Herald added a project: clang.
Herald added a subscriber: cfe-commits.
Although generating command lines and collecting file dependencies recently got
faster, there are still benefits to be had from doing these lazily, on-demand.
This patch makes it so that the `ModuleDepsGraph` keeps the scanning instance
alive. Instances of `ModuleDeps` can then use pointer to the
`ModuleDepCollector` to compute the information lazily.
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D158469
Files:
clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h
clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h
clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp
clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
clang/tools/clang-scan-deps/ClangScanDeps.cpp
Index: clang/tools/clang-scan-deps/ClangScanDeps.cpp
===================================================================
--- clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -351,14 +351,24 @@
}
void mergeDeps(ModuleDepsGraph Graph, size_t InputIndex) {
- std::unique_lock<std::mutex> ul(Lock);
- for (const ModuleDeps &MD : Graph) {
- auto I = Modules.find({MD.ID, 0});
- if (I != Modules.end()) {
- I->first.InputIndex = std::min(I->first.InputIndex, InputIndex);
- continue;
+ std::vector<ModuleDeps *> NewMDs;
+ {
+ std::unique_lock<std::mutex> ul(Lock);
+ for (const ModuleDeps &MD : Graph.MDs) {
+ auto I = Modules.find({MD.ID, 0});
+ if (I != Modules.end()) {
+ I->first.InputIndex = std::min(I->first.InputIndex, InputIndex);
+ continue;
+ }
+ auto NewIt = Modules.insert(I, {{MD.ID, InputIndex}, std::move(MD)});
+ NewMDs.push_back(&NewIt->second);
}
- Modules.insert(I, {{MD.ID, InputIndex}, std::move(MD)});
+ }
+ // Eagerly compute the lazy members before the graph goes out of scope.
+ // This is somewhat costly, so do it outside the critical section.
+ for (ModuleDeps *MD : NewMDs) {
+ (void)MD->getBuildArguments();
+ (void)MD->getFileDeps();
}
}
@@ -382,7 +392,7 @@
/*ShouldOwnClient=*/false);
for (auto &&M : Modules)
- if (roundTripCommand(M.second.BuildArguments, *Diags))
+ if (roundTripCommand(M.second.getBuildArguments(), *Diags))
return true;
for (auto &&I : Inputs)
@@ -408,10 +418,10 @@
Object O{
{"name", MD.ID.ModuleName},
{"context-hash", MD.ID.ContextHash},
- {"file-deps", toJSONSorted(MD.FileDeps)},
+ {"file-deps", toJSONSorted(MD.getFileDeps())},
{"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)},
{"clang-modulemap-file", MD.ClangModuleMapFile},
- {"command-line", MD.BuildArguments},
+ {"command-line", MD.getBuildArguments()},
};
OutModules.push_back(std::move(O));
}
Index: clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
===================================================================
--- clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
+++ clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
@@ -467,20 +467,6 @@
serialization::ModuleFile *MF =
MDC.ScanInstance.getASTReader()->getModuleManager().lookup(
M->getASTFile());
- MDC.ScanInstance.getASTReader()->visitInputFileInfos(
- *MF, /*IncludeSystem=*/true,
- [&](const serialization::InputFileInfo &IFI, bool IsSystem) {
- // __inferred_module.map is the result of the way in which an implicit
- // module build handles inferred modules. It adds an overlay VFS with
- // this file in the proper directory and relies on the rest of Clang to
- // handle it like normal. With explicitly built modules we don't need
- // to play VFS tricks, so replace it with the correct module map.
- if (StringRef(IFI.Filename).endswith("__inferred_module.map")) {
- MDC.addFileDep(MD, ModuleMap->getName());
- return;
- }
- MDC.addFileDep(MD, IFI.Filename);
- });
llvm::DenseSet<const Module *> SeenDeps;
addAllSubmodulePrebuiltDeps(M, MD, SeenDeps);
@@ -510,7 +496,9 @@
// Finish the compiler invocation. Requires dependencies and the context hash.
MDC.addOutputPaths(CI, MD);
- MD.BuildArguments = CI.getCC1CommandLine();
+ // Wire up lazy info computation.
+ MD.MDC = &MDC;
+ MDC.LazyModuleDepsInfoByID.insert({MD.ID, {MF, std::move(CI)}});
return MD.ID;
}
@@ -643,5 +631,50 @@
void ModuleDepCollector::addFileDep(ModuleDeps &MD, StringRef Path) {
llvm::SmallString<256> Storage;
Path = makeAbsoluteAndPreferred(ScanInstance, Path, Storage);
- MD.FileDeps.insert(Path);
+ MD.FileDeps->insert(Path);
+}
+
+void ModuleDepCollector::addFileDeps(ModuleDeps &MD) {
+ auto It = LazyModuleDepsInfoByID.find(MD.ID);
+ assert(It != LazyModuleDepsInfoByID.end());
+
+ MD.FileDeps = llvm::StringSet<>{};
+
+ ScanInstance.getASTReader()->visitInputFileInfos(
+ *It->second.MF, /*IncludeSystem=*/true,
+ [&](const serialization::InputFileInfo &IFI, bool IsSystem) {
+ // __inferred_module.map is the result of the way in which an implicit
+ // module build handles inferred modules. It adds an overlay VFS with
+ // this file in the proper directory and relies on the rest of Clang to
+ // handle it like normal. With explicitly built modules we don't need
+ // to play VFS tricks, so replace it with the correct module map.
+ if (StringRef(IFI.Filename).endswith("__inferred_module.map")) {
+ FileManager &FileMgr = ScanInstance.getFileManager();
+ auto ModuleMap = FileMgr.getOptionalFileRef(MD.ClangModuleMapFile);
+ assert(ModuleMap && "Module map file of a dependency still exists");
+ addFileDep(MD, ModuleMap->getName());
+ return;
+ }
+ addFileDep(MD, IFI.Filename);
+ });
+}
+
+void ModuleDepCollector::addBuildArguments(ModuleDeps &MD) {
+ auto It = LazyModuleDepsInfoByID.find(MD.ID);
+ assert(It != LazyModuleDepsInfoByID.end());
+ MD.BuildArguments = It->second.CI.getCC1CommandLine();
+}
+
+const llvm::StringSet<> &ModuleDeps::getFileDeps() {
+ if (FileDeps)
+ return *FileDeps;
+ MDC->addFileDeps(*this);
+ return *FileDeps;
+}
+
+const std::vector<std::string> &ModuleDeps::getBuildArguments() {
+ if (BuildArguments)
+ return *BuildArguments;
+ MDC->addBuildArguments(*this);
+ return *BuildArguments;
}
Index: clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
===================================================================
--- clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -165,7 +165,8 @@
Scanned = true;
// Create a compiler instance to handle the actual work.
- ScanInstanceStorage.emplace(std::move(PCHContainerOps));
+ ScanInstanceStorage =
+ std::make_shared<CompilerInstance>(std::move(PCHContainerOps));
CompilerInstance &ScanInstance = *ScanInstanceStorage;
ScanInstance.setInvocation(std::move(Invocation));
@@ -268,6 +269,8 @@
if (Result)
setLastCC1Arguments(std::move(OriginalInvocation));
+ Consumer.handleScanInstance(ScanInstanceStorage);
+
return Result;
}
@@ -299,7 +302,7 @@
bool EagerLoadModules;
bool DisableFree;
std::optional<StringRef> ModuleName;
- std::optional<CompilerInstance> ScanInstanceStorage;
+ std::shared_ptr<CompilerInstance> ScanInstanceStorage;
std::shared_ptr<ModuleDepCollector> MDC;
std::vector<std::string> LastCC1Arguments;
bool Scanned = false;
Index: clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp
===================================================================
--- clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp
+++ clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp
@@ -41,6 +41,7 @@
void handleModuleDependency(ModuleDeps MD) override {}
void handleDirectModuleDependency(ModuleID ID) override {}
void handleContextHash(std::string Hash) override {}
+ void handleScanInstance(std::shared_ptr<CompilerInstance> CI) override {}
void printDependencies(std::string &S) {
assert(Opts && "Handled dependency output options.");
@@ -172,22 +173,14 @@
TU.FileDeps = std::move(Dependencies);
TU.PrebuiltModuleDeps = std::move(PrebuiltModuleDeps);
TU.Commands = std::move(Commands);
-
- for (auto &&M : ClangModuleDeps) {
- auto &MD = M.second;
- // TODO: Avoid handleModuleDependency even being called for modules
- // we've already seen.
- if (AlreadySeen.count(M.first))
- continue;
- TU.ModuleGraph.push_back(std::move(MD));
- }
+ TU.ModuleGraph = takeModuleGraphDeps();
TU.ClangModuleDeps = std::move(DirectModuleDeps);
return TU;
}
ModuleDepsGraph FullDependencyConsumer::takeModuleGraphDeps() {
- ModuleDepsGraph ModuleGraph;
+ ModuleDepsGraph ModuleGraph(std::move(ScanInstance));
for (auto &&M : ClangModuleDeps) {
auto &MD = M.second;
@@ -195,7 +188,7 @@
// we've already seen.
if (AlreadySeen.count(M.first))
continue;
- ModuleGraph.push_back(std::move(MD));
+ ModuleGraph.MDs.push_back(std::move(MD));
}
return ModuleGraph;
Index: clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h
===================================================================
--- clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h
+++ clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h
@@ -104,7 +104,21 @@
DiagnosticSerializationFile,
};
-struct ModuleDeps {
+class ModuleDepCollector;
+class ModuleDepCollectorPP;
+
+class ModuleDeps {
+ /// This facilitates the lazy computation of file deps and build arguments.
+ ModuleDepCollector *MDC;
+ /// Storage for the lazily-computed file dependencies.
+ std::optional<llvm::StringSet<>> FileDeps;
+ /// Storage for the lazily-computed build arguments.
+ std::optional<std::vector<std::string>> BuildArguments;
+
+ friend ModuleDepCollector;
+ friend ModuleDepCollectorPP;
+
+public:
/// The identifier of the module.
ModuleID ID;
@@ -117,9 +131,10 @@
/// additionally appear in \c FileDeps as a dependency.
std::string ClangModuleMapFile;
- /// A collection of absolute paths to files that this module directly depends
- /// on, not including transitive dependencies.
- llvm::StringSet<> FileDeps;
+ /// Compute/get the set of absolute paths to files that this module directly
+ /// depends on, not including transitive dependencies. Must be first called
+ /// during the lifetime of the parent dependency graph.
+ const llvm::StringSet<> &getFileDeps();
/// A collection of absolute paths to module map files that this module needs
/// to know about. The ordering is significant.
@@ -136,13 +151,12 @@
/// determined that the differences are benign for this compilation.
std::vector<ModuleID> ClangModuleDeps;
- /// Compiler invocation that can be used to build this module. Does not
- /// include argv[0].
- std::vector<std::string> BuildArguments;
+ /// Compute/get the compiler invocation that can be used to build this module.
+ /// Does not include argv[0]. Must be first called during the lifetime of the
+ /// parent dependency graph.
+ const std::vector<std::string> &getBuildArguments();
};
-class ModuleDepCollector;
-
/// Callback that records textual includes and direct modular includes/imports
/// during preprocessing. At the end of the main file, it also collects
/// transitive modular dependencies and passes everything to the
@@ -216,6 +230,13 @@
private:
friend ModuleDepCollectorPP;
+ friend ModuleDeps;
+
+ /// Information we keep to be able to compute some ModuleDeps info lazily.
+ struct LazyModuleDepsInfo {
+ serialization::ModuleFile *MF;
+ CompilerInvocation CI;
+ };
/// The compiler instance for scanning the current translation unit.
CompilerInstance &ScanInstance;
@@ -235,6 +256,8 @@
/// Secondary mapping for \c ModularDeps allowing lookup by ModuleID without
/// a preprocessor. Storage owned by \c ModularDeps.
llvm::DenseMap<ModuleID, ModuleDeps *> ModuleDepsByID;
+ /// Mapping for lazy computation of some ModuleDeps info.
+ llvm::DenseMap<ModuleID, LazyModuleDepsInfo> LazyModuleDepsInfoByID;
/// Direct modular dependencies that have already been built.
llvm::MapVector<const Module *, PrebuiltModuleDep> DirectPrebuiltModularDeps;
/// Working set of direct modular dependencies.
@@ -280,6 +303,11 @@
void addModuleFiles(CompilerInvocation &CI,
ArrayRef<ModuleID> ClangModuleDeps) const;
+ /// Compute the file deps and store them into \c MD.
+ void addFileDeps(ModuleDeps &MD);
+ /// Compute the build arguments and store them into \c MD.
+ void addBuildArguments(ModuleDeps &MD);
+
/// Add paths that require looking up outputs to the given dependencies.
void addOutputPaths(CompilerInvocation &CI, ModuleDeps &Deps);
Index: clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
===================================================================
--- clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
+++ clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
@@ -58,6 +58,8 @@
virtual void handleDirectModuleDependency(ModuleID MD) = 0;
+ virtual void handleScanInstance(std::shared_ptr<CompilerInstance> CI) = 0;
+
virtual void handleContextHash(std::string Hash) = 0;
};
Index: clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h
===================================================================
--- clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h
+++ clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h
@@ -28,7 +28,17 @@
llvm::function_ref<std::string(const ModuleID &, ModuleOutputKind)>;
/// Graph of modular dependencies.
-using ModuleDepsGraph = std::vector<ModuleDeps>;
+class ModuleDepsGraph {
+ /// This is keeping the scan instance alive so that \c ModuleDeps can use it
+ /// to compute some properties lazily.
+ std::shared_ptr<CompilerInstance> ScanInstance;
+
+public:
+ ModuleDepsGraph(std::shared_ptr<CompilerInstance> ScanInstance = nullptr)
+ : ScanInstance(std::move(ScanInstance)) {}
+
+ std::vector<ModuleDeps> MDs;
+};
/// The full dependencies and module graph for a specific input.
struct TranslationUnitDeps {
@@ -170,6 +180,10 @@
ContextHash = std::move(Hash);
}
+ void handleScanInstance(std::shared_ptr<CompilerInstance> CI) override {
+ ScanInstance = std::move(CI);
+ }
+
TranslationUnitDeps takeTranslationUnitDeps();
ModuleDepsGraph takeModuleGraphDeps();
@@ -180,6 +194,7 @@
std::vector<ModuleID> DirectModuleDeps;
std::vector<Command> Commands;
std::string ContextHash;
+ std::shared_ptr<CompilerInstance> ScanInstance;
std::vector<std::string> OutputPaths;
const llvm::DenseSet<ModuleID> &AlreadySeen;
};
_______________________________________________
cfe-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
