llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-llvm-support @llvm/pr-subscribers-clang-driver Author: Naveen Seth Hanig (naveen-seth) <details> <summary>Changes</summary> This patch is part of a series to support driver-managed module builds. It adds support for the discovery of module dependencies and for dependency graph generation from within the driver. The dependency scan and graph support both Clang modules and C++ named modules. The generated dependency graph can be output in the Graphviz format as a remark. This patch follows the discussion in the RFC linked below and links the driver against the following libraries: ``` clangDependencyScanning clangAST clangFrontend clangSerialization clangLex ``` RFC discussing linking the driver against additional libraries: https://discourse.llvm.org/t/rfc-driver-link-the-driver-against-clangdependencyscanning-clangast-clangfrontend-clangserialization-and-clanglex RFC for driver-managed module builds: https://discourse.llvm.org/t/rfc-modules-support-simple-c-20-modules-use-from-the-clang-driver-without-a-build-system --- Patch is 66.40 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/152770.diff 13 Files Affected: - (modified) clang/include/clang/Basic/DiagnosticDriverKinds.td (+12) - (modified) clang/include/clang/Basic/DiagnosticGroups.td (+1) - (added) clang/include/clang/Driver/DependencyScanner.h (+388) - (modified) clang/include/clang/Driver/Driver.h (+32) - (modified) clang/include/clang/Driver/Options.td (+7) - (modified) clang/include/clang/Lex/DependencyDirectivesScanner.h (+7) - (modified) clang/lib/Driver/CMakeLists.txt (+3) - (added) clang/lib/Driver/DependencyScanner.cpp (+696) - (modified) clang/lib/Driver/Driver.cpp (+84) - (modified) clang/lib/Lex/DependencyDirectivesScanner.cpp (+50) - (added) clang/test/Driver/modules-driver-cxx20-module-usage-scanner.cpp (+192) - (added) clang/test/Driver/modules-driver-dependency-graph.cpp (+88) - (modified) llvm/include/llvm/Support/GraphWriter.h (+2) ``````````diff diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td index 0f17f4aa761ea..68532ba58a03b 100644 --- a/clang/include/clang/Basic/DiagnosticDriverKinds.td +++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td @@ -581,6 +581,18 @@ def err_drv_reduced_module_output_overrided : Warning< "please consider use '-fmodule-output=' to specify the output file for reduced BMI explicitly">, InGroup<DiagGroup<"reduced-bmi-output-overrided">>; +def remark_found_cxx20_module_usage : Remark< + "found C++20 module usage in file '%0'">, + InGroup<ModulesDriver>; +def remark_performing_driver_managed_module_build : Remark< + "performing driver managed module build">, + InGroup<ModulesDriver>; +def err_failed_dependency_scan : Error < + "failed to perform dependency scan">; +def remark_module_dependency_graph : Remark< + "printing module dependency graph">, + InGroup<ModulesDriver>; + def warn_drv_delayed_template_parsing_after_cxx20 : Warning< "-fdelayed-template-parsing is deprecated after C++20">, InGroup<DiagGroup<"delayed-template-parsing-in-cxx20">>; diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td index ccb18aa37447e..78726ecc869db 100644 --- a/clang/include/clang/Basic/DiagnosticGroups.td +++ b/clang/include/clang/Basic/DiagnosticGroups.td @@ -628,6 +628,7 @@ def ModuleConflict : DiagGroup<"module-conflict">; def ModuleFileExtension : DiagGroup<"module-file-extension">; def ModuleIncludeDirectiveTranslation : DiagGroup<"module-include-translation">; def ModuleMap : DiagGroup<"module-map">; +def ModulesDriver : DiagGroup<"modules-driver">; def RoundTripCC1Args : DiagGroup<"round-trip-cc1-args">; def NewlineEOF : DiagGroup<"newline-eof">; def Nullability : DiagGroup<"nullability">; diff --git a/clang/include/clang/Driver/DependencyScanner.h b/clang/include/clang/Driver/DependencyScanner.h new file mode 100644 index 0000000000000..a2d56aba462ca --- /dev/null +++ b/clang/include/clang/Driver/DependencyScanner.h @@ -0,0 +1,388 @@ +#ifndef LLVM_CLANG_DRIVER_DEPENDENCYSCANNER_H +#define LLVM_CLANG_DRIVER_DEPENDENCYSCANNER_H + +#include "clang/Driver/Driver.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" +#include "llvm/ADT/DirectedGraph.h" +#include "llvm/Support/DOTGraphTraits.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/GraphWriter.h" + +namespace clang { +class SourceManager; +class CharSourceRange; +class DiagnosticsEngine; +} // namespace clang + +namespace llvm::opt { +class DerivedArgList; +} // namespace llvm::opt + +namespace clang { +namespace driver { +namespace dependencies { + +using clang::tooling::dependencies::TranslationUnitDeps; + +//===----------------------------------------------------------------------===// +// Dependency Scan +//===----------------------------------------------------------------------===// + +class DependencyScanError : public llvm::ErrorInfo<DependencyScanError> { +public: + static char ID; + + void log(llvm::raw_ostream &OS) const override { + OS << "error while performing dependency scan\n"; + } + + std::error_code convertToErrorCode() const override { + return llvm::errc::not_supported; + } +}; + +/// Performs a full dependency scan for the given driver command line and +/// returns all scan results or an error on scanning failure. +/// +/// \param ClangProgramPath Path to the clang executable +/// \param Diags The calling driver's diagnostics engine +/// \param Args The calling driver's command line arguments +/// +/// \returns The scan results for all inputs, or an error if scanning fails +llvm::Expected<SmallVector<TranslationUnitDeps, 0>> +scanModuleDependencies(llvm::StringRef ClangProgramPath, + clang::DiagnosticsEngine &Diags, + const llvm::opt::DerivedArgList &Args); + +//===----------------------------------------------------------------------===// +// Module Dependency Graph +//===----------------------------------------------------------------------===// + +class MDGNode; +class MDGEdge; +using MDGNodeBase = llvm::DGNode<MDGNode, MDGEdge>; +using MDGEdgeBase = llvm::DGEdge<MDGNode, MDGEdge>; +using MDGBase = llvm::DirectedGraph<MDGNode, MDGEdge>; + +/// Base class for module dependency graph nodes. +/// +/// Represents a node in the ModuleDepGraph, which can be a translation unit +/// which doesn't provide any module, a Clang module, or a C++ named module. +class MDGNode : public MDGNodeBase { +public: + enum class NodeKind { + ClangModule, + NonModuleTU, + CXXNamedModule, + }; + + using Command = tooling::dependencies::Command; + + MDGNode(const NodeKind K) : Kind(K) {} + MDGNode(const NodeKind K, std::vector<Command> Commands) + : Kind(K), Commands(Commands) {} + + virtual ~MDGNode() = 0; + + NodeKind getKind() const { return Kind; } + + ArrayRef<Command> getCommands() const { return Commands; } + +private: + const NodeKind Kind; + +protected: + std::vector<Command> Commands; +}; + +/// ClangModuleNode - represents a Clang module in the ModuleDepGraph. +class ClangModuleNode : public MDGNode { +public: + ClangModuleNode(StringRef ModuleName) + : MDGNode(NodeKind::ClangModule), ModuleName(ModuleName) {} + ~ClangModuleNode() = default; + + static bool classof(const MDGNode *N) { + return N->getKind() == NodeKind::ClangModule; + } + + StringRef getModuleName() const { return ModuleName; } + + void setCommands(std::vector<tooling::dependencies::Command> Commands) { + this->Commands = Commands; + } + +private: + std::string ModuleName; +}; + +/// NonModuleTUNode - represents a regular TU which doesn't provide any module, +/// in the ModuleDepGraph. +class NonModuleTUNode : public MDGNode { +public: + NonModuleTUNode(StringRef InputFile) + : MDGNode(NodeKind::NonModuleTU), InputFile(InputFile) {} + ~NonModuleTUNode() override = default; + + static bool classof(const MDGNode *N) { + return N->getKind() == NodeKind::NonModuleTU; + } + + StringRef getInputFile() const { return InputFile; } + +private: + const std::string InputFile; +}; + +/// CXXNamedModuleNode - represents a C++ named module node in the +/// ModuleDepGraph. +/// +/// Unresolved nodes are those discovered as imports but missing a module +/// definition. +class CXXNamedModuleNode : public MDGNode { +public: + CXXNamedModuleNode(StringRef ModuleName, StringRef InputFile = "") + : MDGNode(NodeKind::CXXNamedModule), InputFile(InputFile), + ModuleName(ModuleName) {} + ~CXXNamedModuleNode() = default; + + static bool classof(const MDGNode *N) { + return N->getKind() == NodeKind::CXXNamedModule; + } + + StringRef getInputFile() const { return InputFile; } + + StringRef getModuleName() const { return ModuleName; } + + bool isUnresolved() const { return InputFile.empty(); } + + void setInputFile(StringRef FilePath) { this->InputFile = FilePath; } + + void setCommands(std::vector<tooling::dependencies::Command> Commands) { + this->Commands = Commands; + } + +private: + std::string InputFile; + std::string ModuleName; +}; + +/// MDGEdge - represents an import relationship, directed from the importing +/// unit to the imported unit. +class MDGEdge : public MDGEdgeBase { +public: + MDGEdge() = delete; + MDGEdge(MDGNode &N) : MDGEdgeBase(N) {} +}; + +class ModuleDepGraphBuilder; + +/// ModuleDepGraph - A directed graph that represents dependency relationships +/// from dependency scan results, with ownership of its nodes and edges. +class ModuleDepGraph : public MDGBase { + friend ModuleDepGraphBuilder; + + template <typename NodeTy, typename... Args> + NodeTy *MakeWithBumpAlloc(Args &&...args); + + llvm::BumpPtrAllocator BumpPtrAlloc; +}; + +/// Fully constructs a ModuleDepGraph from the dependency scan results. +/// +/// \param ScanResults The list of scan results. +/// \param Inputs The calling drivers list of input list in its original order. +/// \param Path to the clang executable. +ModuleDepGraph +buildModuleDepGraph(SmallVectorImpl<TranslationUnitDeps> &&ScanResults, + clang::driver::Driver::InputList Inputs, + StringRef ClangProgramPath); + +} // namespace dependencies +} // namespace driver +} // namespace clang + +//===----------------------------------------------------------------------===// +// Module Dependency Graph: GraphTraits specialization +//===----------------------------------------------------------------------===// + +namespace llvm { +/// non-const versions of the GraphTrait specializations for MDG +template <> struct GraphTraits<clang::driver::dependencies::MDGNode *> { + using NodeRef = clang::driver::dependencies::MDGNode *; + + static NodeRef MDGGetTargetNode(clang::driver::dependencies::MDGEdgeBase *P) { + return &P->getTargetNode(); + } + + // Provide a mapped iterator so that the GraphTrait-based implementations can + // find the target nodes without having to explicitly go through the edges. + using ChildIteratorType = + mapped_iterator<clang::driver::dependencies::MDGNode::iterator, + decltype(&MDGGetTargetNode)>; + using ChildEdgeIteratorType = clang::driver::dependencies::MDGNode::iterator; + + static NodeRef getEntryNode(NodeRef N) { return N; } + static ChildIteratorType child_begin(NodeRef N) { + return ChildIteratorType(N->begin(), &MDGGetTargetNode); + } + static ChildIteratorType child_end(NodeRef N) { + return ChildIteratorType(N->end(), &MDGGetTargetNode); + } + + static ChildEdgeIteratorType child_edge_begin(NodeRef N) { + return N->begin(); + } + static ChildEdgeIteratorType child_edge_end(NodeRef N) { return N->end(); } +}; + +template <> +struct GraphTraits<clang::driver::dependencies::ModuleDepGraph *> + : public GraphTraits<clang::driver::dependencies::MDGNode *> { + using nodes_iterator = clang::driver::dependencies::ModuleDepGraph::iterator; + static NodeRef getEntryNode(clang::driver::dependencies::ModuleDepGraph *G) { + return *G->begin(); + } + static nodes_iterator + nodes_begin(clang::driver::dependencies::ModuleDepGraph *G) { + return G->begin(); + } + static nodes_iterator + nodes_end(clang::driver::dependencies::ModuleDepGraph *G) { + return G->end(); + } +}; + +/// const versions of the GraphTrait specializations for MDG +template <> struct GraphTraits<const clang::driver::dependencies::MDGNode *> { + using NodeRef = const clang::driver::dependencies::MDGNode *; + + static NodeRef + MDGGetTargetNode(const clang::driver::dependencies::MDGEdgeBase *P) { + return &P->getTargetNode(); + } + + using ChildIteratorType = + mapped_iterator<clang::driver::dependencies::MDGNode::const_iterator, + decltype(&MDGGetTargetNode)>; + using ChildEdgeIteratorType = + clang::driver::dependencies::MDGNode::const_iterator; + + static NodeRef getEntryNode(NodeRef N) { return N; } + static ChildIteratorType child_begin(NodeRef N) { + return ChildIteratorType(N->begin(), &MDGGetTargetNode); + } + static ChildIteratorType child_end(NodeRef N) { + return ChildIteratorType(N->end(), &MDGGetTargetNode); + } + + static ChildEdgeIteratorType child_edge_begin(NodeRef N) { + return N->begin(); + } + static ChildEdgeIteratorType child_edge_end(NodeRef N) { return N->end(); } +}; + +template <> +struct GraphTraits<const clang::driver::dependencies::ModuleDepGraph *> + : public GraphTraits<const clang::driver::dependencies::MDGNode *> { + using nodes_iterator = + clang::driver::dependencies::ModuleDepGraph::const_iterator; + + static NodeRef + getEntryNode(const clang::driver::dependencies::ModuleDepGraph *G) { + return *G->begin(); + } + static nodes_iterator + nodes_begin(const clang::driver::dependencies::ModuleDepGraph *G) { + return G->begin(); + } + static nodes_iterator + nodes_end(const clang::driver::dependencies::ModuleDepGraph *G) { + return G->end(); + } +}; + +//===----------------------------------------------------------------------===// +// Module Dependency Graph: DOTGraphTraits & GraphWriter specializations +//===----------------------------------------------------------------------===// + +template <> +struct DOTGraphTraits<const clang::driver::dependencies::ModuleDepGraph *> + : public DefaultDOTGraphTraits { + DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {} + + static StringRef + getGraphName(const clang::driver::dependencies::ModuleDepGraph *MDG) { + return "Module Dependency Graph"; + } + + static StringRef + getNodeKindLabel(const clang::driver::dependencies::NonModuleTUNode *N) { + return "Non-Module TU"; + } + + static StringRef + getNodeKindLabel(const clang::driver::dependencies::ClangModuleNode *N) { + return "Clang Module"; + } + + static StringRef + getNodeKindLabel(const clang::driver::dependencies::CXXNamedModuleNode *N) { + return "C++ Named Module"; + } + + static std::string + getNodeIdentifierLabel(const clang::driver::dependencies::MDGNode *N, + const clang::driver::dependencies::ModuleDepGraph *G) { + using namespace clang::driver::dependencies; + if (const auto *ClangModule = dyn_cast<ClangModuleNode>(N)) + return (Twine(getNodeKindLabel(ClangModule)) + " '" + + ClangModule->getModuleName() + "'") + .str(); + if (const auto *CXXNamedModule = dyn_cast<CXXNamedModuleNode>(N)) + return (Twine(getNodeKindLabel(CXXNamedModule)) + " '" + + CXXNamedModule->getModuleName() + "'") + .str(); + if (const auto *NonModuleTU = dyn_cast<NonModuleTUNode>(N)) + return (Twine(getNodeKindLabel(NonModuleTU)) + " '" + + NonModuleTU->getInputFile() + "'") + .str(); + llvm_unreachable("Unhandled MDGNode kind!"); + } + + static std::string + getGraphProperties(const clang::driver::dependencies::ModuleDepGraph *G) { + return "\tnode [shape=Mrecord];\n\tedge [dir=\"back\"];\n"; + } +}; + +template <> +class GraphWriter<const clang::driver::dependencies::ModuleDepGraph *> + : public GraphWriterBase< + const clang::driver::dependencies::ModuleDepGraph *, + GraphWriter<const clang::driver::dependencies::ModuleDepGraph *>> { +public: + using GraphType = const clang::driver::dependencies::ModuleDepGraph *; + using Base = GraphWriterBase<GraphType, GraphWriter<GraphType>>; + + GraphWriter(raw_ostream &o, const GraphType &g, bool SN) : Base(o, g, SN) {} + + void writeNodes(); + +private: + using Base::DOTTraits; + using Base::GTraits; + using Base::NodeRef; + + void writeNodeDeclarations(ArrayRef<NodeRef> Nodes); + void writeNodeDeclaration(NodeRef Node); + void writeNodeRelations(ArrayRef<NodeRef> Nodes); + void writeNodeRelation(NodeRef Node); + + DenseMap<NodeRef, std::string> NodeIDLabels; +}; + +} // namespace llvm + +#endif diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h index 4d32552b7f85f..b9b187ada8add 100644 --- a/clang/include/clang/Driver/Driver.h +++ b/clang/include/clang/Driver/Driver.h @@ -512,6 +512,9 @@ class Driver { /// BuildActions - Construct the list of actions to perform for the /// given arguments, which are only done for a single architecture. + /// If the compilation is an explicit module build, delegates to + /// BuildDriverManagedModuleBuildActions. Otherwise, BuildDefaultActions is + /// used. /// /// \param C - The compilation that is being built. /// \param Args - The input arguments. @@ -796,6 +799,35 @@ class Driver { /// compilation based on which -f(no-)?lto(=.*)? option occurs last. void setLTOMode(const llvm::opt::ArgList &Args); + /// BuildDefaultActions - Constructs the list of actions to perform + /// for the provided arguments, which are only done for a single architecture. + /// + /// \param C - The compilation that is being built. + /// \param Args - The input arguments. + /// \param Actions - The list to store the resulting actions onto. + void BuildDefaultActions(Compilation &C, llvm::opt::DerivedArgList &Args, + const InputList &Inputs, ActionList &Actions) const; + + /// BuildDriverManagedModuleBuildActions - Performs a dependency + /// scan and constructs the list of actions to perform for dependency order + /// and the provided arguments. This is only done for a single a architecture. + /// + /// \param C - The compilation that is being built. + /// \param Args - The input arguments. + /// \param Actions - The list to store the resulting actions onto. + void BuildDriverManagedModuleBuildActions(Compilation &C, + llvm::opt::DerivedArgList &Args, + const InputList &Inputs, + ActionList &Actions) const; + + /// Scans the leading lines of the C++ source inputs to detect C++20 module + /// usage. + /// + /// \returns True if module usage is detected, false otherwise, or an error on + /// read failure. + llvm::ErrorOr<bool> + ScanInputsForCXX20ModulesUsage(const InputList &Inputs) const; + /// Retrieves a ToolChain for a particular \p Target triple. /// /// Will cache ToolChains for the life of the driver object, and create them diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 6aab43c9ed57f..ae8ddb0c2de88 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -3296,6 +3296,13 @@ defm modules_reduced_bmi : BoolOption<"f", "modules-reduced-bmi", PosFlag<SetTrue, [], [ClangOption, CC1Option], "Generate the reduced BMI">>; +def fmodules_driver : Flag<["-"], "fmodules-driver">, + Group<f_Group>, Visibility<[ClangOption]>, + HelpText<"Enable support for driver managed module builds (experimental)">; +def fno_modules_driver : Flag<["-"], "fno-modules-driver">, + Group<f_Group>, Visibility<[ClangOption]>, + HelpText<"Disable support for driver managed module builds (experimental)">; + def experimental_modules_reduced_bmi : Flag<["-"], "fexperimental-modules-reduced-bmi">, Group<f_Group>, Visibility<[ClangOption, CC1Option]>, Alias<fmodules_reduced_bmi>; diff --git a/clang/include/clang/Lex/DependencyDirectivesScanner.h b/clang/include/clang/Lex/DependencyDirectivesScanner.h index f9fec3998ca53..c0b742d652a03 100644 --- a/clang/include/clang/Lex/DependencyDirectivesScanner.h +++ b/clang/include/clang/Lex/DependencyDirectivesScanner.h @@ -135,6 +135,13 @@ void printDependencyDirectivesAsSource( ArrayRef<dependency_directives_scan::Directive> Directives, llvm::raw_ostream &OS); +/// Scan an input source buffer for C++20 named module usage. +/// +/// \param Source The input source buffer. +/// +/// \returns true if any C++20 named modules related directive was found. +bool scanInputForCXX20ModulesUsage(StringRef Source); + /// Functor that returns the dependency directives for a given file. class DependencyDirectivesGetter { public: diff --git a/clang/lib/Driver/CMakeLists.txt b/clang/lib/Driver/CMakeLists.txt index 45782cbd9d16d..4d6375633c17a 100644 --- a/clang/lib/Driver/CMakeLists.txt +++ b/clang/lib/Driver/CMakeLists.txt @@ -17,6 +17,7 @@ endif() add_clang_library(clangDriver Action.cpp Compilation.cpp + DependencyScanner.cpp Distro.cpp Driver.cpp DriverOptions.cpp @@ -98,5 +99,7 @@ add_clang_library(clangDriver LINK_LIBS clangBasic + clangDependencyScanning + clangLex ${system_libs} ) diff --git a/clang/lib/Driver/DependencyScanner.cpp b/clang/lib/Driver/DependencyScanner.cpp new file mode 100644 index 0000000000000..3b2dc7af67bcb --- /dev/null +++ b/clang/lib/Driver/DependencyScanner.cpp @@ -0,0 +1,696 @@ +#include "clang/Driver/DependencyScanner.h" +#include "clang/Basic/Diagnostic.h" +#include "clang/Basic/LangOptions.h" +#include "clang/Basic/SourceLocation.h" +#include "clang/Driver/Compilation.h" +#include "clang/Driver/Driver.h" +#include "clang/Driver/Tool.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" +#include "clang/Lex/Lexer.h" +#include "clang/Lex/Preprocessor.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" +#include "llvm/ADT/STLExtras.h" +#inclu... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/152770 _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits