Wow, I'm just seeing this through its use on D7289. This is so great! Is there some way we can draw more attention to this? Docs?
-- Sean Silva On Wed, Nov 6, 2013 at 8:12 PM, Peter Collingbourne <[email protected]> wrote: > Author: pcc > Date: Wed Nov 6 14:12:45 2013 > New Revision: 194164 > > URL: http://llvm.org/viewvc/llvm-project?rev=194164&view=rev > Log: > Introduce ClangTool::buildASTs, and buildASTFromCode. > > These allow clients to retrieve persistent AST objects (ASTUnits) which > can be used in an ad-hoc manner after parsing. > > To accommodate this change, the code for processing a CompilerInvocation > using a FrontendAction has been factored out to FrontendActionFactory, and > a new base class, ToolAction, has been introduced, allowing the tool to do > arbitrary things with each CompilerInvocation. This change was necessary > because ASTUnit does not use the FrontendAction interface directly. > > This change also causes the FileManager in ClangTool to use shared > ownership. > This will become necessary because ASTUnit takes shared ownership of > FileManager (ClangTool's FileManager is currently unused by ASTUnit; this > is a FIXME). As shown in the tests, any client of ToolInvocation will > need to be modified to use shared ownership for FileManager. > > Differential Revision: http://llvm-reviews.chandlerc.com/D2097 > > Modified: > cfe/trunk/include/clang/Tooling/Tooling.h > cfe/trunk/lib/Tooling/Tooling.cpp > cfe/trunk/unittests/Tooling/ToolingTest.cpp > > Modified: cfe/trunk/include/clang/Tooling/Tooling.h > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Tooling/Tooling.h?rev=194164&r1=194163&r2=194164&view=diff > > ============================================================================== > --- cfe/trunk/include/clang/Tooling/Tooling.h (original) > +++ cfe/trunk/include/clang/Tooling/Tooling.h Wed Nov 6 14:12:45 2013 > @@ -53,14 +53,33 @@ class FrontendAction; > > namespace tooling { > > +/// \brief Interface to process a clang::CompilerInvocation. > +/// > +/// If your tool is based on FrontendAction, you should be deriving from > +/// FrontendActionFactory instead. > +class ToolAction { > +public: > + virtual ~ToolAction(); > + > + /// \brief Perform an action for an invocation. > + virtual bool runInvocation(clang::CompilerInvocation *Invocation, > + FileManager *Files) = 0; > +}; > + > /// \brief Interface to generate clang::FrontendActions. > /// > /// Having a factory interface allows, for example, a new FrontendAction > to be > -/// created for each translation unit processed by ClangTool. > -class FrontendActionFactory { > +/// created for each translation unit processed by ClangTool. This class > is > +/// also a ToolAction which uses the FrontendActions created by create() > to > +/// process each translation unit. > +class FrontendActionFactory : public ToolAction { > public: > virtual ~FrontendActionFactory(); > > + /// \brief Invokes the compiler with a FrontendAction created by > create(). > + bool runInvocation(clang::CompilerInvocation *Invocation, > + FileManager *Files); > + > /// \brief Returns a new clang::FrontendAction. > /// > /// The caller takes ownership of the returned action. > @@ -133,6 +152,26 @@ bool runToolOnCodeWithArgs(clang::Fronte > const std::vector<std::string> &Args, > const Twine &FileName = "input.cc"); > > +/// \brief Builds an AST for 'Code'. > +/// > +/// \param Code C++ code. > +/// \param FileName The file name which 'Code' will be mapped as. > +/// > +/// \return The resulting AST or null if an error occurred. > +ASTUnit *buildASTFromCode(const Twine &Code, > + const Twine &FileName = "input.cc"); > + > +/// \brief Builds an AST for 'Code' with additional flags. > +/// > +/// \param Code C++ code. > +/// \param Args Additional flags to pass on. > +/// \param FileName The file name which 'Code' will be mapped as. > +/// > +/// \return The resulting AST or null if an error occurred. > +ASTUnit *buildASTFromCodeWithArgs(const Twine &Code, > + const std::vector<std::string> &Args, > + const Twine &FileName = "input.cc"); > + > /// \brief Utility to run a FrontendAction in a single clang invocation. > class ToolInvocation { > public: > @@ -145,9 +184,19 @@ class ToolInvocation { > /// \param ToolAction The action to be executed. Class takes ownership. > /// \param Files The FileManager used for the execution. Class does not > take > /// ownership. > - ToolInvocation(ArrayRef<std::string> CommandLine, FrontendAction > *ToolAction, > + ToolInvocation(ArrayRef<std::string> CommandLine, FrontendAction > *FAction, > FileManager *Files); > > + /// \brief Create a tool invocation. > + /// > + /// \param CommandLine The command line arguments to clang. > + /// \param Action The action to be executed. > + /// \param Files The FileManager used for the execution. > + ToolInvocation(ArrayRef<std::string> CommandLine, ToolAction *Action, > + FileManager *Files); > + > + ~ToolInvocation(); > + > /// \brief Map a virtual file to be used while running the tool. > /// > /// \param FilePath The path at which the content will be mapped. > @@ -167,7 +216,8 @@ class ToolInvocation { > clang::CompilerInvocation *Invocation); > > std::vector<std::string> CommandLine; > - OwningPtr<FrontendAction> ToolAction; > + ToolAction *Action; > + bool OwnsAction; > FileManager *Files; > // Maps <file name> -> <file content>. > llvm::StringMap<StringRef> MappedFileContents; > @@ -216,23 +266,25 @@ class ClangTool { > /// \brief Clear the command line arguments adjuster chain. > void clearArgumentsAdjusters(); > > - /// Runs a frontend action over all files specified in the command line. > + /// Runs an action over all files specified in the command line. > /// > - /// \param ActionFactory Factory generating the frontend actions. The > function > - /// takes ownership of this parameter. A new action is generated for > every > - /// processed translation unit. > - virtual int run(FrontendActionFactory *ActionFactory); > + /// \param Action Tool action. > + int run(ToolAction *Action); > + > + /// \brief Create an AST for each file specified in the command line and > + /// append them to ASTs. > + int buildASTs(std::vector<ASTUnit *> &ASTs); > > /// \brief Returns the file manager used in the tool. > /// > /// The file manager is shared between all translation units. > - FileManager &getFiles() { return Files; } > + FileManager &getFiles() { return *Files; } > > private: > // We store compile commands as pair (file name, compile command). > std::vector< std::pair<std::string, CompileCommand> > CompileCommands; > > - FileManager Files; > + llvm::IntrusiveRefCntPtr<FileManager> Files; > // Contains a list of pairs (<file name>, <file content>). > std::vector< std::pair<StringRef, StringRef> > MappedFileContents; > > > Modified: cfe/trunk/lib/Tooling/Tooling.cpp > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Tooling/Tooling.cpp?rev=194164&r1=194163&r2=194164&view=diff > > ============================================================================== > --- cfe/trunk/lib/Tooling/Tooling.cpp (original) > +++ cfe/trunk/lib/Tooling/Tooling.cpp Wed Nov 6 14:12:45 2013 > @@ -13,9 +13,11 @@ > > > //===----------------------------------------------------------------------===// > > #include "clang/Tooling/Tooling.h" > +#include "clang/AST/ASTConsumer.h" > #include "clang/Driver/Compilation.h" > #include "clang/Driver/Driver.h" > #include "clang/Driver/Tool.h" > +#include "clang/Frontend/ASTUnit.h" > #include "clang/Frontend/CompilerInstance.h" > #include "clang/Frontend/FrontendDiagnostic.h" > #include "clang/Frontend/TextDiagnosticPrinter.h" > @@ -38,6 +40,8 @@ > namespace clang { > namespace tooling { > > +ToolAction::~ToolAction() {} > + > FrontendActionFactory::~FrontendActionFactory() {} > > // FIXME: This file contains structural duplication with other parts of > the > @@ -104,18 +108,26 @@ bool runToolOnCode(clang::FrontendAction > ToolAction, Code, std::vector<std::string>(), FileName); > } > > +static std::vector<std::string> > +getSyntaxOnlyToolArgs(const std::vector<std::string> &ExtraArgs, > + StringRef FileName) { > + std::vector<std::string> Args; > + Args.push_back("clang-tool"); > + Args.push_back("-fsyntax-only"); > + Args.insert(Args.end(), ExtraArgs.begin(), ExtraArgs.end()); > + Args.push_back(FileName.str()); > + return Args; > +} > + > bool runToolOnCodeWithArgs(clang::FrontendAction *ToolAction, const Twine > &Code, > const std::vector<std::string> &Args, > const Twine &FileName) { > SmallString<16> FileNameStorage; > StringRef FileNameRef = > FileName.toNullTerminatedStringRef(FileNameStorage); > - std::vector<std::string> Commands; > - Commands.push_back("clang-tool"); > - Commands.push_back("-fsyntax-only"); > - Commands.insert(Commands.end(), Args.begin(), Args.end()); > - Commands.push_back(FileNameRef.data()); > - FileManager Files((FileSystemOptions())); > - ToolInvocation Invocation(Commands, ToolAction, &Files); > + llvm::IntrusiveRefCntPtr<FileManager> Files( > + new FileManager(FileSystemOptions())); > + ToolInvocation Invocation(getSyntaxOnlyToolArgs(Args, FileNameRef), > ToolAction, > + Files.getPtr()); > > SmallString<1024> CodeStorage; > Invocation.mapVirtualFile(FileNameRef, > @@ -138,10 +150,33 @@ std::string getAbsolutePath(StringRef Fi > return AbsolutePath.str(); > } > > -ToolInvocation::ToolInvocation( > - ArrayRef<std::string> CommandLine, FrontendAction *ToolAction, > - FileManager *Files) > - : CommandLine(CommandLine.vec()), ToolAction(ToolAction), > Files(Files) { > +namespace { > + > +class SingleFrontendActionFactory : public FrontendActionFactory { > + FrontendAction *Action; > + > +public: > + SingleFrontendActionFactory(FrontendAction *Action) : Action(Action) {} > + > + FrontendAction *create() { return Action; } > +}; > + > +} > + > +ToolInvocation::ToolInvocation(ArrayRef<std::string> CommandLine, > + ToolAction *Action, FileManager *Files) > + : CommandLine(CommandLine.vec()), Action(Action), OwnsAction(false), > + Files(Files) {} > + > +ToolInvocation::ToolInvocation(ArrayRef<std::string> CommandLine, > + FrontendAction *FAction, FileManager > *Files) > + : CommandLine(CommandLine.vec()), > + Action(new SingleFrontendActionFactory(FAction)), OwnsAction(true), > + Files(Files) {} > + > +ToolInvocation::~ToolInvocation() { > + if (OwnsAction) > + delete Action; > } > > void ToolInvocation::mapVirtualFile(StringRef FilePath, StringRef > Content) { > @@ -175,6 +210,14 @@ bool ToolInvocation::run() { > } > OwningPtr<clang::CompilerInvocation> Invocation( > newInvocation(&Diagnostics, *CC1Args)); > + for (llvm::StringMap<StringRef>::const_iterator > + It = MappedFileContents.begin(), End = > MappedFileContents.end(); > + It != End; ++It) { > + // Inject the code as the given file name into the preprocessor > options. > + const llvm::MemoryBuffer *Input = > + llvm::MemoryBuffer::getMemBuffer(It->getValue()); > + Invocation->getPreprocessorOpts().addRemappedFile(It->getKey(), > Input); > + } > return runInvocation(BinaryName, Compilation.get(), Invocation.take()); > } > > @@ -189,16 +232,20 @@ bool ToolInvocation::runInvocation( > llvm::errs() << "\n"; > } > > + return Action->runInvocation(Invocation, Files); > +} > + > +bool FrontendActionFactory::runInvocation(CompilerInvocation *Invocation, > + FileManager *Files) { > // Create a compiler instance to handle the actual work. > clang::CompilerInstance Compiler; > Compiler.setInvocation(Invocation); > Compiler.setFileManager(Files); > - // FIXME: What about LangOpts? > > - // ToolAction can have lifetime requirements for Compiler or its > members, and > - // we need to ensure it's deleted earlier than Compiler. So we pass it > to an > - // OwningPtr declared after the Compiler variable. > - OwningPtr<FrontendAction> ScopedToolAction(ToolAction.take()); > + // The FrontendAction can have lifetime requirements for Compiler or its > + // members, and we need to ensure it's deleted earlier than Compiler. > So we > + // pass it to an OwningPtr declared after the Compiler variable. > + OwningPtr<FrontendAction> ScopedToolAction(create()); > > // Create the compilers actual diagnostics engine. > Compiler.createDiagnostics(); > @@ -206,32 +253,16 @@ bool ToolInvocation::runInvocation( > return false; > > Compiler.createSourceManager(*Files); > - addFileMappingsTo(Compiler.getSourceManager()); > > const bool Success = Compiler.ExecuteAction(*ScopedToolAction); > > - Compiler.resetAndLeakFileManager(); > Files->clearStatCaches(); > return Success; > } > > -void ToolInvocation::addFileMappingsTo(SourceManager &Sources) { > - for (llvm::StringMap<StringRef>::const_iterator > - It = MappedFileContents.begin(), End = > MappedFileContents.end(); > - It != End; ++It) { > - // Inject the code as the given file name into the preprocessor > options. > - const llvm::MemoryBuffer *Input = > - llvm::MemoryBuffer::getMemBuffer(It->getValue()); > - // FIXME: figure out what '0' stands for. > - const FileEntry *FromFile = Files->getVirtualFile( > - It->getKey(), Input->getBufferSize(), 0); > - Sources.overrideFileContents(FromFile, Input); > - } > -} > - > ClangTool::ClangTool(const CompilationDatabase &Compilations, > ArrayRef<std::string> SourcePaths) > - : Files((FileSystemOptions())) { > + : Files(new FileManager(FileSystemOptions())) { > ArgsAdjusters.push_back(new ClangStripOutputAdjuster()); > ArgsAdjusters.push_back(new ClangSyntaxOnlyAdjuster()); > for (unsigned I = 0, E = SourcePaths.size(); I != E; ++I) { > @@ -274,7 +305,7 @@ void ClangTool::clearArgumentsAdjusters( > ArgsAdjusters.clear(); > } > > -int ClangTool::run(FrontendActionFactory *ActionFactory) { > +int ClangTool::run(ToolAction *Action) { > // Exists solely for the purpose of lookup of the resource path. > // This just needs to be some symbol in the binary. > static int StaticSymbol; > @@ -309,7 +340,7 @@ int ClangTool::run(FrontendActionFactory > DEBUG({ > llvm::dbgs() << "Processing: " << File << ".\n"; > }); > - ToolInvocation Invocation(CommandLine, ActionFactory->create(), > &Files); > + ToolInvocation Invocation(CommandLine, Action, Files.getPtr()); > for (int I = 0, E = MappedFileContents.size(); I != E; ++I) { > Invocation.mapVirtualFile(MappedFileContents[I].first, > MappedFileContents[I].second); > @@ -323,5 +354,58 @@ int ClangTool::run(FrontendActionFactory > return ProcessingFailed ? 1 : 0; > } > > +namespace { > + > +class ASTBuilderAction : public ToolAction { > + std::vector<ASTUnit *> &ASTs; > + > +public: > + ASTBuilderAction(std::vector<ASTUnit *> &ASTs) : ASTs(ASTs) {} > + > + bool runInvocation(CompilerInvocation *Invocation, > + FileManager *Files) { > + // FIXME: This should use the provided FileManager. > + ASTUnit *AST = ASTUnit::LoadFromCompilerInvocation( > + Invocation, > + > CompilerInstance::createDiagnostics(&Invocation->getDiagnosticOpts())); > + if (!AST) > + return false; > + > + ASTs.push_back(AST); > + return true; > + } > +}; > + > +} > + > +int ClangTool::buildASTs(std::vector<ASTUnit *> &ASTs) { > + ASTBuilderAction Action(ASTs); > + return run(&Action); > +} > + > +ASTUnit *buildASTFromCode(const Twine &Code, const Twine &FileName) { > + return buildASTFromCodeWithArgs(Code, std::vector<std::string>(), > FileName); > +} > + > +ASTUnit *buildASTFromCodeWithArgs(const Twine &Code, > + const std::vector<std::string> &Args, > + const Twine &FileName) { > + SmallString<16> FileNameStorage; > + StringRef FileNameRef = > FileName.toNullTerminatedStringRef(FileNameStorage); > + > + std::vector<ASTUnit *> ASTs; > + ASTBuilderAction Action(ASTs); > + ToolInvocation Invocation(getSyntaxOnlyToolArgs(Args, FileNameRef), > &Action, 0); > + > + SmallString<1024> CodeStorage; > + Invocation.mapVirtualFile(FileNameRef, > + Code.toNullTerminatedStringRef(CodeStorage)); > + if (!Invocation.run()) > + return 0; > + > + assert(ASTs.size() == 1); > + return ASTs[0]; > +} > + > } // end namespace tooling > } // end namespace clang > > Modified: cfe/trunk/unittests/Tooling/ToolingTest.cpp > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/unittests/Tooling/ToolingTest.cpp?rev=194164&r1=194163&r2=194164&view=diff > > ============================================================================== > --- cfe/trunk/unittests/Tooling/ToolingTest.cpp (original) > +++ cfe/trunk/unittests/Tooling/ToolingTest.cpp Wed Nov 6 14:12:45 2013 > @@ -10,12 +10,14 @@ > #include "clang/AST/ASTConsumer.h" > #include "clang/AST/DeclCXX.h" > #include "clang/AST/DeclGroup.h" > +#include "clang/Frontend/ASTUnit.h" > #include "clang/Frontend/CompilerInstance.h" > #include "clang/Frontend/FrontendAction.h" > #include "clang/Frontend/FrontendActions.h" > #include "clang/Tooling/CompilationDatabase.h" > #include "clang/Tooling/Tooling.h" > #include "gtest/gtest.h" > +#include "llvm/ADT/STLExtras.h" > #include <string> > > namespace clang { > @@ -83,6 +85,18 @@ class FindClassDeclXConsumer : public cl > private: > bool *FoundClassDeclX; > }; > +bool FindClassDeclX(ASTUnit *AST) { > + for (std::vector<Decl *>::iterator i = AST->top_level_begin(), > + e = AST->top_level_end(); > + i != e; ++i) { > + if (CXXRecordDecl* Record = dyn_cast<clang::CXXRecordDecl>(*i)) { > + if (Record->getName() == "X") { > + return true; > + } > + } > + } > + return false; > +} > } // end namespace > > TEST(runToolOnCode, FindsClassDecl) { > @@ -97,6 +111,16 @@ TEST(runToolOnCode, FindsClassDecl) { > EXPECT_FALSE(FoundClassDeclX); > } > > +TEST(buildASTFromCode, FindsClassDecl) { > + OwningPtr<ASTUnit> AST(buildASTFromCode("class X;")); > + ASSERT_TRUE(AST.get()); > + EXPECT_TRUE(FindClassDeclX(AST.get())); > + > + AST.reset(buildASTFromCode("class Y;")); > + ASSERT_TRUE(AST.get()); > + EXPECT_FALSE(FindClassDeclX(AST.get())); > +} > + > TEST(newFrontendActionFactory, CreatesFrontendActionFactoryFromType) { > OwningPtr<FrontendActionFactory> Factory( > newFrontendActionFactory<SyntaxOnlyAction>()); > @@ -119,13 +143,15 @@ TEST(newFrontendActionFactory, CreatesFr > } > > TEST(ToolInvocation, TestMapVirtualFile) { > - clang::FileManager Files((clang::FileSystemOptions())); > + IntrusiveRefCntPtr<clang::FileManager> Files( > + new clang::FileManager(clang::FileSystemOptions())); > std::vector<std::string> Args; > Args.push_back("tool-executable"); > Args.push_back("-Idef"); > Args.push_back("-fsyntax-only"); > Args.push_back("test.cpp"); > - clang::tooling::ToolInvocation Invocation(Args, new SyntaxOnlyAction, > &Files); > + clang::tooling::ToolInvocation Invocation(Args, new SyntaxOnlyAction, > + Files.getPtr()); > Invocation.mapVirtualFile("test.cpp", "#include <abc>\n"); > Invocation.mapVirtualFile("def/abc", "\n"); > EXPECT_TRUE(Invocation.run()); > @@ -136,13 +162,15 @@ TEST(ToolInvocation, TestVirtualModulesC > // mapped module.map is found on the include path. In the future, > expand this > // test to run a full modules enabled compilation, so we make sure we > can > // rerun modules compilations with a virtual file system. > - clang::FileManager Files((clang::FileSystemOptions())); > + IntrusiveRefCntPtr<clang::FileManager> Files( > + new clang::FileManager(clang::FileSystemOptions())); > std::vector<std::string> Args; > Args.push_back("tool-executable"); > Args.push_back("-Idef"); > Args.push_back("-fsyntax-only"); > Args.push_back("test.cpp"); > - clang::tooling::ToolInvocation Invocation(Args, new SyntaxOnlyAction, > &Files); > + clang::tooling::ToolInvocation Invocation(Args, new SyntaxOnlyAction, > + Files.getPtr()); > Invocation.mapVirtualFile("test.cpp", "#include <abc>\n"); > Invocation.mapVirtualFile("def/abc", "\n"); > // Add a module.map file in the include directory of our header, so we > trigger > @@ -254,5 +282,23 @@ TEST(ClangToolTest, ArgumentAdjusters) { > EXPECT_FALSE(Found); > } > > +TEST(ClangToolTest, BuildASTs) { > + FixedCompilationDatabase Compilations("/", std::vector<std::string>()); > + > + std::vector<std::string> Sources; > + Sources.push_back("/a.cc"); > + Sources.push_back("/b.cc"); > + ClangTool Tool(Compilations, Sources); > + > + Tool.mapVirtualFile("/a.cc", "void a() {}"); > + Tool.mapVirtualFile("/b.cc", "void b() {}"); > + > + std::vector<ASTUnit *> ASTs; > + EXPECT_EQ(0, Tool.buildASTs(ASTs)); > + EXPECT_EQ(2u, ASTs.size()); > + > + llvm::DeleteContainerPointers(ASTs); > +} > + > } // end namespace tooling > } // end namespace clang > > > _______________________________________________ > cfe-commits mailing list > [email protected] > http://lists.cs.uiuc.edu/mailman/listinfo/cfe-commits >
_______________________________________________ cfe-commits mailing list [email protected] http://lists.cs.uiuc.edu/mailman/listinfo/cfe-commits
