On Fri Jan 30 2015 at 1:18:46 PM Sean Silva <[email protected]> wrote:
> Wow, I'm just seeing this through its use on D7289. This is so great! Is > there some way we can draw more attention to this? Docs? > Awesome! A volunteer :) Yes, we definitely could need more docs here - best in the form of small tutorials ;) > > > -- Sean Silva > > On Wed, Nov 6, 2013 at 8:12 PM, Peter Collingbourne <[email protected]> > wrote: > >> Author: pcc >> Date: Wed Nov 6 14:12:45 2013 >> New Revision: 194164 >> >> URL: http://llvm.org/viewvc/llvm-project?rev=194164&view=rev >> Log: >> Introduce ClangTool::buildASTs, and buildASTFromCode. >> >> These allow clients to retrieve persistent AST objects (ASTUnits) which >> can be used in an ad-hoc manner after parsing. >> >> To accommodate this change, the code for processing a CompilerInvocation >> using a FrontendAction has been factored out to FrontendActionFactory, and >> a new base class, ToolAction, has been introduced, allowing the tool to do >> arbitrary things with each CompilerInvocation. This change was necessary >> because ASTUnit does not use the FrontendAction interface directly. >> >> This change also causes the FileManager in ClangTool to use shared >> ownership. >> This will become necessary because ASTUnit takes shared ownership of >> FileManager (ClangTool's FileManager is currently unused by ASTUnit; this >> is a FIXME). As shown in the tests, any client of ToolInvocation will >> need to be modified to use shared ownership for FileManager. >> >> Differential Revision: http://llvm-reviews.chandlerc.com/D2097 >> >> Modified: >> cfe/trunk/include/clang/Tooling/Tooling.h >> cfe/trunk/lib/Tooling/Tooling.cpp >> cfe/trunk/unittests/Tooling/ToolingTest.cpp >> >> Modified: cfe/trunk/include/clang/Tooling/Tooling.h >> URL: >> http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Tooling/Tooling.h?rev=194164&r1=194163&r2=194164&view=diff >> >> ============================================================================== >> --- cfe/trunk/include/clang/Tooling/Tooling.h (original) >> +++ cfe/trunk/include/clang/Tooling/Tooling.h Wed Nov 6 14:12:45 2013 >> @@ -53,14 +53,33 @@ class FrontendAction; >> >> namespace tooling { >> >> +/// \brief Interface to process a clang::CompilerInvocation. >> +/// >> +/// If your tool is based on FrontendAction, you should be deriving from >> +/// FrontendActionFactory instead. >> +class ToolAction { >> +public: >> + virtual ~ToolAction(); >> + >> + /// \brief Perform an action for an invocation. >> + virtual bool runInvocation(clang::CompilerInvocation *Invocation, >> + FileManager *Files) = 0; >> +}; >> + >> /// \brief Interface to generate clang::FrontendActions. >> /// >> /// Having a factory interface allows, for example, a new FrontendAction >> to be >> -/// created for each translation unit processed by ClangTool. >> -class FrontendActionFactory { >> +/// created for each translation unit processed by ClangTool. This >> class is >> +/// also a ToolAction which uses the FrontendActions created by create() >> to >> +/// process each translation unit. >> +class FrontendActionFactory : public ToolAction { >> public: >> virtual ~FrontendActionFactory(); >> >> + /// \brief Invokes the compiler with a FrontendAction created by >> create(). >> + bool runInvocation(clang::CompilerInvocation *Invocation, >> + FileManager *Files); >> + >> /// \brief Returns a new clang::FrontendAction. >> /// >> /// The caller takes ownership of the returned action. >> @@ -133,6 +152,26 @@ bool runToolOnCodeWithArgs(clang::Fronte >> const std::vector<std::string> &Args, >> const Twine &FileName = "input.cc"); >> >> +/// \brief Builds an AST for 'Code'. >> +/// >> +/// \param Code C++ code. >> +/// \param FileName The file name which 'Code' will be mapped as. >> +/// >> +/// \return The resulting AST or null if an error occurred. >> +ASTUnit *buildASTFromCode(const Twine &Code, >> + const Twine &FileName = "input.cc"); >> + >> +/// \brief Builds an AST for 'Code' with additional flags. >> +/// >> +/// \param Code C++ code. >> +/// \param Args Additional flags to pass on. >> +/// \param FileName The file name which 'Code' will be mapped as. >> +/// >> +/// \return The resulting AST or null if an error occurred. >> +ASTUnit *buildASTFromCodeWithArgs(const Twine &Code, >> + const std::vector<std::string> &Args, >> + const Twine &FileName = "input.cc"); >> + >> /// \brief Utility to run a FrontendAction in a single clang invocation. >> class ToolInvocation { >> public: >> @@ -145,9 +184,19 @@ class ToolInvocation { >> /// \param ToolAction The action to be executed. Class takes ownership. >> /// \param Files The FileManager used for the execution. Class does >> not take >> /// ownership. >> - ToolInvocation(ArrayRef<std::string> CommandLine, FrontendAction >> *ToolAction, >> + ToolInvocation(ArrayRef<std::string> CommandLine, FrontendAction >> *FAction, >> FileManager *Files); >> >> + /// \brief Create a tool invocation. >> + /// >> + /// \param CommandLine The command line arguments to clang. >> + /// \param Action The action to be executed. >> + /// \param Files The FileManager used for the execution. >> + ToolInvocation(ArrayRef<std::string> CommandLine, ToolAction *Action, >> + FileManager *Files); >> + >> + ~ToolInvocation(); >> + >> /// \brief Map a virtual file to be used while running the tool. >> /// >> /// \param FilePath The path at which the content will be mapped. >> @@ -167,7 +216,8 @@ class ToolInvocation { >> clang::CompilerInvocation *Invocation); >> >> std::vector<std::string> CommandLine; >> - OwningPtr<FrontendAction> ToolAction; >> + ToolAction *Action; >> + bool OwnsAction; >> FileManager *Files; >> // Maps <file name> -> <file content>. >> llvm::StringMap<StringRef> MappedFileContents; >> @@ -216,23 +266,25 @@ class ClangTool { >> /// \brief Clear the command line arguments adjuster chain. >> void clearArgumentsAdjusters(); >> >> - /// Runs a frontend action over all files specified in the command >> line. >> + /// Runs an action over all files specified in the command line. >> /// >> - /// \param ActionFactory Factory generating the frontend actions. The >> function >> - /// takes ownership of this parameter. A new action is generated for >> every >> - /// processed translation unit. >> - virtual int run(FrontendActionFactory *ActionFactory); >> + /// \param Action Tool action. >> + int run(ToolAction *Action); >> + >> + /// \brief Create an AST for each file specified in the command line >> and >> + /// append them to ASTs. >> + int buildASTs(std::vector<ASTUnit *> &ASTs); >> >> /// \brief Returns the file manager used in the tool. >> /// >> /// The file manager is shared between all translation units. >> - FileManager &getFiles() { return Files; } >> + FileManager &getFiles() { return *Files; } >> >> private: >> // We store compile commands as pair (file name, compile command). >> std::vector< std::pair<std::string, CompileCommand> > CompileCommands; >> >> - FileManager Files; >> + llvm::IntrusiveRefCntPtr<FileManager> Files; >> // Contains a list of pairs (<file name>, <file content>). >> std::vector< std::pair<StringRef, StringRef> > MappedFileContents; >> >> >> Modified: cfe/trunk/lib/Tooling/Tooling.cpp >> URL: >> http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Tooling/Tooling.cpp?rev=194164&r1=194163&r2=194164&view=diff >> >> ============================================================================== >> --- cfe/trunk/lib/Tooling/Tooling.cpp (original) >> +++ cfe/trunk/lib/Tooling/Tooling.cpp Wed Nov 6 14:12:45 2013 >> @@ -13,9 +13,11 @@ >> >> >> //===----------------------------------------------------------------------===// >> >> #include "clang/Tooling/Tooling.h" >> +#include "clang/AST/ASTConsumer.h" >> #include "clang/Driver/Compilation.h" >> #include "clang/Driver/Driver.h" >> #include "clang/Driver/Tool.h" >> +#include "clang/Frontend/ASTUnit.h" >> #include "clang/Frontend/CompilerInstance.h" >> #include "clang/Frontend/FrontendDiagnostic.h" >> #include "clang/Frontend/TextDiagnosticPrinter.h" >> @@ -38,6 +40,8 @@ >> namespace clang { >> namespace tooling { >> >> +ToolAction::~ToolAction() {} >> + >> FrontendActionFactory::~FrontendActionFactory() {} >> >> // FIXME: This file contains structural duplication with other parts of >> the >> @@ -104,18 +108,26 @@ bool runToolOnCode(clang::FrontendAction >> ToolAction, Code, std::vector<std::string>(), FileName); >> } >> >> +static std::vector<std::string> >> +getSyntaxOnlyToolArgs(const std::vector<std::string> &ExtraArgs, >> + StringRef FileName) { >> + std::vector<std::string> Args; >> + Args.push_back("clang-tool"); >> + Args.push_back("-fsyntax-only"); >> + Args.insert(Args.end(), ExtraArgs.begin(), ExtraArgs.end()); >> + Args.push_back(FileName.str()); >> + return Args; >> +} >> + >> bool runToolOnCodeWithArgs(clang::FrontendAction *ToolAction, const >> Twine &Code, >> const std::vector<std::string> &Args, >> const Twine &FileName) { >> SmallString<16> FileNameStorage; >> StringRef FileNameRef = >> FileName.toNullTerminatedStringRef(FileNameStorage); >> - std::vector<std::string> Commands; >> - Commands.push_back("clang-tool"); >> - Commands.push_back("-fsyntax-only"); >> - Commands.insert(Commands.end(), Args.begin(), Args.end()); >> - Commands.push_back(FileNameRef.data()); >> - FileManager Files((FileSystemOptions())); >> - ToolInvocation Invocation(Commands, ToolAction, &Files); >> + llvm::IntrusiveRefCntPtr<FileManager> Files( >> + new FileManager(FileSystemOptions())); >> + ToolInvocation Invocation(getSyntaxOnlyToolArgs(Args, FileNameRef), >> ToolAction, >> + Files.getPtr()); >> >> SmallString<1024> CodeStorage; >> Invocation.mapVirtualFile(FileNameRef, >> @@ -138,10 +150,33 @@ std::string getAbsolutePath(StringRef Fi >> return AbsolutePath.str(); >> } >> >> -ToolInvocation::ToolInvocation( >> - ArrayRef<std::string> CommandLine, FrontendAction *ToolAction, >> - FileManager *Files) >> - : CommandLine(CommandLine.vec()), ToolAction(ToolAction), >> Files(Files) { >> +namespace { >> + >> +class SingleFrontendActionFactory : public FrontendActionFactory { >> + FrontendAction *Action; >> + >> +public: >> + SingleFrontendActionFactory(FrontendAction *Action) : Action(Action) {} >> + >> + FrontendAction *create() { return Action; } >> +}; >> + >> +} >> + >> +ToolInvocation::ToolInvocation(ArrayRef<std::string> CommandLine, >> + ToolAction *Action, FileManager *Files) >> + : CommandLine(CommandLine.vec()), Action(Action), OwnsAction(false), >> + Files(Files) {} >> + >> +ToolInvocation::ToolInvocation(ArrayRef<std::string> CommandLine, >> + FrontendAction *FAction, FileManager >> *Files) >> + : CommandLine(CommandLine.vec()), >> + Action(new SingleFrontendActionFactory(FAction)), OwnsAction(true), >> + Files(Files) {} >> + >> +ToolInvocation::~ToolInvocation() { >> + if (OwnsAction) >> + delete Action; >> } >> >> void ToolInvocation::mapVirtualFile(StringRef FilePath, StringRef >> Content) { >> @@ -175,6 +210,14 @@ bool ToolInvocation::run() { >> } >> OwningPtr<clang::CompilerInvocation> Invocation( >> newInvocation(&Diagnostics, *CC1Args)); >> + for (llvm::StringMap<StringRef>::const_iterator >> + It = MappedFileContents.begin(), End = >> MappedFileContents.end(); >> + It != End; ++It) { >> + // Inject the code as the given file name into the preprocessor >> options. >> + const llvm::MemoryBuffer *Input = >> + llvm::MemoryBuffer::getMemBuffer(It->getValue()); >> + Invocation->getPreprocessorOpts().addRemappedFile(It->getKey(), >> Input); >> + } >> return runInvocation(BinaryName, Compilation.get(), Invocation.take()); >> } >> >> @@ -189,16 +232,20 @@ bool ToolInvocation::runInvocation( >> llvm::errs() << "\n"; >> } >> >> + return Action->runInvocation(Invocation, Files); >> +} >> + >> +bool FrontendActionFactory::runInvocation(CompilerInvocation *Invocation, >> + FileManager *Files) { >> // Create a compiler instance to handle the actual work. >> clang::CompilerInstance Compiler; >> Compiler.setInvocation(Invocation); >> Compiler.setFileManager(Files); >> - // FIXME: What about LangOpts? >> >> - // ToolAction can have lifetime requirements for Compiler or its >> members, and >> - // we need to ensure it's deleted earlier than Compiler. So we pass it >> to an >> - // OwningPtr declared after the Compiler variable. >> - OwningPtr<FrontendAction> ScopedToolAction(ToolAction.take()); >> + // The FrontendAction can have lifetime requirements for Compiler or >> its >> + // members, and we need to ensure it's deleted earlier than Compiler. >> So we >> + // pass it to an OwningPtr declared after the Compiler variable. >> + OwningPtr<FrontendAction> ScopedToolAction(create()); >> >> // Create the compilers actual diagnostics engine. >> Compiler.createDiagnostics(); >> @@ -206,32 +253,16 @@ bool ToolInvocation::runInvocation( >> return false; >> >> Compiler.createSourceManager(*Files); >> - addFileMappingsTo(Compiler.getSourceManager()); >> >> const bool Success = Compiler.ExecuteAction(*ScopedToolAction); >> >> - Compiler.resetAndLeakFileManager(); >> Files->clearStatCaches(); >> return Success; >> } >> >> -void ToolInvocation::addFileMappingsTo(SourceManager &Sources) { >> - for (llvm::StringMap<StringRef>::const_iterator >> - It = MappedFileContents.begin(), End = >> MappedFileContents.end(); >> - It != End; ++It) { >> - // Inject the code as the given file name into the preprocessor >> options. >> - const llvm::MemoryBuffer *Input = >> - llvm::MemoryBuffer::getMemBuffer(It->getValue()); >> - // FIXME: figure out what '0' stands for. >> - const FileEntry *FromFile = Files->getVirtualFile( >> - It->getKey(), Input->getBufferSize(), 0); >> - Sources.overrideFileContents(FromFile, Input); >> - } >> -} >> - >> ClangTool::ClangTool(const CompilationDatabase &Compilations, >> ArrayRef<std::string> SourcePaths) >> - : Files((FileSystemOptions())) { >> + : Files(new FileManager(FileSystemOptions())) { >> ArgsAdjusters.push_back(new ClangStripOutputAdjuster()); >> ArgsAdjusters.push_back(new ClangSyntaxOnlyAdjuster()); >> for (unsigned I = 0, E = SourcePaths.size(); I != E; ++I) { >> @@ -274,7 +305,7 @@ void ClangTool::clearArgumentsAdjusters( >> ArgsAdjusters.clear(); >> } >> >> -int ClangTool::run(FrontendActionFactory *ActionFactory) { >> +int ClangTool::run(ToolAction *Action) { >> // Exists solely for the purpose of lookup of the resource path. >> // This just needs to be some symbol in the binary. >> static int StaticSymbol; >> @@ -309,7 +340,7 @@ int ClangTool::run(FrontendActionFactory >> DEBUG({ >> llvm::dbgs() << "Processing: " << File << ".\n"; >> }); >> - ToolInvocation Invocation(CommandLine, ActionFactory->create(), >> &Files); >> + ToolInvocation Invocation(CommandLine, Action, Files.getPtr()); >> for (int I = 0, E = MappedFileContents.size(); I != E; ++I) { >> Invocation.mapVirtualFile(MappedFileContents[I].first, >> MappedFileContents[I].second); >> @@ -323,5 +354,58 @@ int ClangTool::run(FrontendActionFactory >> return ProcessingFailed ? 1 : 0; >> } >> >> +namespace { >> + >> +class ASTBuilderAction : public ToolAction { >> + std::vector<ASTUnit *> &ASTs; >> + >> +public: >> + ASTBuilderAction(std::vector<ASTUnit *> &ASTs) : ASTs(ASTs) {} >> + >> + bool runInvocation(CompilerInvocation *Invocation, >> + FileManager *Files) { >> + // FIXME: This should use the provided FileManager. >> + ASTUnit *AST = ASTUnit::LoadFromCompilerInvocation( >> + Invocation, >> + >> CompilerInstance::createDiagnostics(&Invocation->getDiagnosticOpts())); >> + if (!AST) >> + return false; >> + >> + ASTs.push_back(AST); >> + return true; >> + } >> +}; >> + >> +} >> + >> +int ClangTool::buildASTs(std::vector<ASTUnit *> &ASTs) { >> + ASTBuilderAction Action(ASTs); >> + return run(&Action); >> +} >> + >> +ASTUnit *buildASTFromCode(const Twine &Code, const Twine &FileName) { >> + return buildASTFromCodeWithArgs(Code, std::vector<std::string>(), >> FileName); >> +} >> + >> +ASTUnit *buildASTFromCodeWithArgs(const Twine &Code, >> + const std::vector<std::string> &Args, >> + const Twine &FileName) { >> + SmallString<16> FileNameStorage; >> + StringRef FileNameRef = >> FileName.toNullTerminatedStringRef(FileNameStorage); >> + >> + std::vector<ASTUnit *> ASTs; >> + ASTBuilderAction Action(ASTs); >> + ToolInvocation Invocation(getSyntaxOnlyToolArgs(Args, FileNameRef), >> &Action, 0); >> + >> + SmallString<1024> CodeStorage; >> + Invocation.mapVirtualFile(FileNameRef, >> + Code.toNullTerminatedStringRef(CodeStorage)); >> + if (!Invocation.run()) >> + return 0; >> + >> + assert(ASTs.size() == 1); >> + return ASTs[0]; >> +} >> + >> } // end namespace tooling >> } // end namespace clang >> >> Modified: cfe/trunk/unittests/Tooling/ToolingTest.cpp >> URL: >> http://llvm.org/viewvc/llvm-project/cfe/trunk/unittests/Tooling/ToolingTest.cpp?rev=194164&r1=194163&r2=194164&view=diff >> >> ============================================================================== >> --- cfe/trunk/unittests/Tooling/ToolingTest.cpp (original) >> +++ cfe/trunk/unittests/Tooling/ToolingTest.cpp Wed Nov 6 14:12:45 2013 >> @@ -10,12 +10,14 @@ >> #include "clang/AST/ASTConsumer.h" >> #include "clang/AST/DeclCXX.h" >> #include "clang/AST/DeclGroup.h" >> +#include "clang/Frontend/ASTUnit.h" >> #include "clang/Frontend/CompilerInstance.h" >> #include "clang/Frontend/FrontendAction.h" >> #include "clang/Frontend/FrontendActions.h" >> #include "clang/Tooling/CompilationDatabase.h" >> #include "clang/Tooling/Tooling.h" >> #include "gtest/gtest.h" >> +#include "llvm/ADT/STLExtras.h" >> #include <string> >> >> namespace clang { >> @@ -83,6 +85,18 @@ class FindClassDeclXConsumer : public cl >> private: >> bool *FoundClassDeclX; >> }; >> +bool FindClassDeclX(ASTUnit *AST) { >> + for (std::vector<Decl *>::iterator i = AST->top_level_begin(), >> + e = AST->top_level_end(); >> + i != e; ++i) { >> + if (CXXRecordDecl* Record = dyn_cast<clang::CXXRecordDecl>(*i)) { >> + if (Record->getName() == "X") { >> + return true; >> + } >> + } >> + } >> + return false; >> +} >> } // end namespace >> >> TEST(runToolOnCode, FindsClassDecl) { >> @@ -97,6 +111,16 @@ TEST(runToolOnCode, FindsClassDecl) { >> EXPECT_FALSE(FoundClassDeclX); >> } >> >> +TEST(buildASTFromCode, FindsClassDecl) { >> + OwningPtr<ASTUnit> AST(buildASTFromCode("class X;")); >> + ASSERT_TRUE(AST.get()); >> + EXPECT_TRUE(FindClassDeclX(AST.get())); >> + >> + AST.reset(buildASTFromCode("class Y;")); >> + ASSERT_TRUE(AST.get()); >> + EXPECT_FALSE(FindClassDeclX(AST.get())); >> +} >> + >> TEST(newFrontendActionFactory, CreatesFrontendActionFactoryFromType) { >> OwningPtr<FrontendActionFactory> Factory( >> newFrontendActionFactory<SyntaxOnlyAction>()); >> @@ -119,13 +143,15 @@ TEST(newFrontendActionFactory, CreatesFr >> } >> >> TEST(ToolInvocation, TestMapVirtualFile) { >> - clang::FileManager Files((clang::FileSystemOptions())); >> + IntrusiveRefCntPtr<clang::FileManager> Files( >> + new clang::FileManager(clang::FileSystemOptions())); >> std::vector<std::string> Args; >> Args.push_back("tool-executable"); >> Args.push_back("-Idef"); >> Args.push_back("-fsyntax-only"); >> Args.push_back("test.cpp"); >> - clang::tooling::ToolInvocation Invocation(Args, new SyntaxOnlyAction, >> &Files); >> + clang::tooling::ToolInvocation Invocation(Args, new SyntaxOnlyAction, >> + Files.getPtr()); >> Invocation.mapVirtualFile("test.cpp", "#include <abc>\n"); >> Invocation.mapVirtualFile("def/abc", "\n"); >> EXPECT_TRUE(Invocation.run()); >> @@ -136,13 +162,15 @@ TEST(ToolInvocation, TestVirtualModulesC >> // mapped module.map is found on the include path. In the future, >> expand this >> // test to run a full modules enabled compilation, so we make sure we >> can >> // rerun modules compilations with a virtual file system. >> - clang::FileManager Files((clang::FileSystemOptions())); >> + IntrusiveRefCntPtr<clang::FileManager> Files( >> + new clang::FileManager(clang::FileSystemOptions())); >> std::vector<std::string> Args; >> Args.push_back("tool-executable"); >> Args.push_back("-Idef"); >> Args.push_back("-fsyntax-only"); >> Args.push_back("test.cpp"); >> - clang::tooling::ToolInvocation Invocation(Args, new SyntaxOnlyAction, >> &Files); >> + clang::tooling::ToolInvocation Invocation(Args, new SyntaxOnlyAction, >> + Files.getPtr()); >> Invocation.mapVirtualFile("test.cpp", "#include <abc>\n"); >> Invocation.mapVirtualFile("def/abc", "\n"); >> // Add a module.map file in the include directory of our header, so we >> trigger >> @@ -254,5 +282,23 @@ TEST(ClangToolTest, ArgumentAdjusters) { >> EXPECT_FALSE(Found); >> } >> >> +TEST(ClangToolTest, BuildASTs) { >> + FixedCompilationDatabase Compilations("/", std::vector<std::string>()); >> + >> + std::vector<std::string> Sources; >> + Sources.push_back("/a.cc"); >> + Sources.push_back("/b.cc"); >> + ClangTool Tool(Compilations, Sources); >> + >> + Tool.mapVirtualFile("/a.cc", "void a() {}"); >> + Tool.mapVirtualFile("/b.cc", "void b() {}"); >> + >> + std::vector<ASTUnit *> ASTs; >> + EXPECT_EQ(0, Tool.buildASTs(ASTs)); >> + EXPECT_EQ(2u, ASTs.size()); >> + >> + llvm::DeleteContainerPointers(ASTs); >> +} >> + >> } // end namespace tooling >> } // end namespace clang >> >> >> _______________________________________________ >> cfe-commits mailing list >> [email protected] >> http://lists.cs.uiuc.edu/mailman/listinfo/cfe-commits >> > > _______________________________________________ > cfe-commits mailing list > [email protected] > http://lists.cs.uiuc.edu/mailman/listinfo/cfe-commits >
_______________________________________________ cfe-commits mailing list [email protected] http://lists.cs.uiuc.edu/mailman/listinfo/cfe-commits
