Author: rsmith Date: Mon Jun 5 13:10:11 2017 New Revision: 304726 URL: http://llvm.org/viewvc/llvm-project?rev=304726&view=rev Log: Rather than rejecting attempts to run preprocessor-only actions on AST files, replay the steps taken to create the AST file with the preprocessor-only action installed to produce preprocessed output.
This can be used to produce the preprocessed text for an existing .pch or .pcm file. Modified: cfe/trunk/include/clang/Basic/LangOptions.h cfe/trunk/include/clang/Basic/SourceManager.h cfe/trunk/include/clang/Frontend/ASTUnit.h cfe/trunk/include/clang/Frontend/FrontendAction.h cfe/trunk/lib/Basic/SourceManager.cpp cfe/trunk/lib/Frontend/ASTUnit.cpp cfe/trunk/lib/Frontend/FrontendAction.cpp cfe/trunk/lib/Serialization/ASTReader.cpp cfe/trunk/test/Modules/preprocess-module.cpp Modified: cfe/trunk/include/clang/Basic/LangOptions.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Basic/LangOptions.h?rev=304726&r1=304725&r2=304726&view=diff ============================================================================== --- cfe/trunk/include/clang/Basic/LangOptions.h (original) +++ cfe/trunk/include/clang/Basic/LangOptions.h Mon Jun 5 13:10:11 2017 @@ -58,6 +58,7 @@ public: SOB_Trapping // -ftrapv }; + // FIXME: Unify with TUKind. enum CompilingModuleKind { CMK_None, ///< Not compiling a module interface at all. CMK_ModuleMap, ///< Compiling a module from a module map. Modified: cfe/trunk/include/clang/Basic/SourceManager.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Basic/SourceManager.h?rev=304726&r1=304725&r2=304726&view=diff ============================================================================== --- cfe/trunk/include/clang/Basic/SourceManager.h (original) +++ cfe/trunk/include/clang/Basic/SourceManager.h Mon Jun 5 13:10:11 2017 @@ -722,6 +722,10 @@ public: void clearIDTables(); + /// Initialize this source manager suitably to replay the compilation + /// described by \p Old. Requires that \p Old outlive \p *this. + void initializeForReplay(const SourceManager &Old); + DiagnosticsEngine &getDiagnostics() const { return Diag; } FileManager &getFileManager() const { return FileMgr; } Modified: cfe/trunk/include/clang/Frontend/ASTUnit.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Frontend/ASTUnit.h?rev=304726&r1=304725&r2=304726&view=diff ============================================================================== --- cfe/trunk/include/clang/Frontend/ASTUnit.h (original) +++ cfe/trunk/include/clang/Frontend/ASTUnit.h Mon Jun 5 13:10:11 2017 @@ -51,6 +51,7 @@ class DiagnosticsEngine; class FileEntry; class FileManager; class HeaderSearch; +class InputKind; class MemoryBufferCache; class Preprocessor; class PCHContainerOperations; @@ -305,9 +306,6 @@ private: /// (likely to change while trying to use them). bool UserFilesAreVolatile : 1; - /// \brief The language options used when we load an AST file. - LangOptions ASTFileLangOpts; - static void ConfigureDiags(IntrusiveRefCntPtr<DiagnosticsEngine> Diags, ASTUnit &AST, bool CaptureDiagnostics); @@ -702,6 +700,9 @@ public: /// \brief Determine what kind of translation unit this AST represents. TranslationUnitKind getTranslationUnitKind() const { return TUKind; } + /// \brief Determine the input kind this AST unit represents. + InputKind getInputKind() const; + /// \brief A mapping from a file name to the memory buffer that stores the /// remapped contents of that file. typedef std::pair<std::string, llvm::MemoryBuffer *> RemappedFile; Modified: cfe/trunk/include/clang/Frontend/FrontendAction.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Frontend/FrontendAction.h?rev=304726&r1=304725&r2=304726&view=diff ============================================================================== --- cfe/trunk/include/clang/Frontend/FrontendAction.h (original) +++ cfe/trunk/include/clang/Frontend/FrontendAction.h Mon Jun 5 13:10:11 2017 @@ -176,10 +176,10 @@ public: virtual TranslationUnitKind getTranslationUnitKind() { return TU_Complete; } /// \brief Does this action support use with PCH? - virtual bool hasPCHSupport() const { return !usesPreprocessorOnly(); } + virtual bool hasPCHSupport() const { return true; } /// \brief Does this action support use with AST files? - virtual bool hasASTFileSupport() const { return !usesPreprocessorOnly(); } + virtual bool hasASTFileSupport() const { return true; } /// \brief Does this action support use with IR files? virtual bool hasIRSupport() const { return false; } Modified: cfe/trunk/lib/Basic/SourceManager.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Basic/SourceManager.cpp?rev=304726&r1=304725&r2=304726&view=diff ============================================================================== --- cfe/trunk/lib/Basic/SourceManager.cpp (original) +++ cfe/trunk/lib/Basic/SourceManager.cpp Mon Jun 5 13:10:11 2017 @@ -345,6 +345,41 @@ void SourceManager::clearIDTables() { createExpansionLoc(SourceLocation(),SourceLocation(),SourceLocation(), 1); } +void SourceManager::initializeForReplay(const SourceManager &Old) { + assert(MainFileID.isInvalid() && "expected uninitialized SourceManager"); + + auto CloneContentCache = [&](const ContentCache *Cache) -> ContentCache * { + auto *Clone = new (ContentCacheAlloc.Allocate<ContentCache>()) ContentCache; + Clone->OrigEntry = Cache->OrigEntry; + Clone->ContentsEntry = Cache->ContentsEntry; + Clone->BufferOverridden = Cache->BufferOverridden; + Clone->IsSystemFile = Cache->IsSystemFile; + Clone->IsTransient = Cache->IsTransient; + Clone->replaceBuffer(Cache->getRawBuffer(), /*DoNotFree*/true); + return Clone; + }; + + // Set up our main file ID as a copy of the old source manager's main file. + const SLocEntry &OldMainFile = Old.getSLocEntry(Old.getMainFileID()); + assert(OldMainFile.isFile() && "main file is macro expansion?"); + setMainFileID(createFileID( + CloneContentCache(OldMainFile.getFile().getContentCache()), + SourceLocation(), OldMainFile.getFile().getFileCharacteristic(), 0, 0)); + + // Ensure all SLocEntries are loaded from the external source. + for (unsigned I = 0, N = Old.LoadedSLocEntryTable.size(); I != N; ++I) + if (!Old.SLocEntryLoaded[I]) + Old.loadSLocEntry(I, nullptr); + + // Inherit any content cache data from the old source manager. + for (auto &FileInfo : Old.FileInfos) { + SrcMgr::ContentCache *&Slot = FileInfos[FileInfo.first]; + if (Slot) + continue; + Slot = CloneContentCache(FileInfo.second); + } +} + /// getOrCreateContentCache - Create or return a cached ContentCache for the /// specified file. const ContentCache * Modified: cfe/trunk/lib/Frontend/ASTUnit.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Frontend/ASTUnit.cpp?rev=304726&r1=304725&r2=304726&view=diff ============================================================================== --- cfe/trunk/lib/Frontend/ASTUnit.cpp (original) +++ cfe/trunk/lib/Frontend/ASTUnit.cpp Mon Jun 5 13:10:11 2017 @@ -667,6 +667,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFr ConfigureDiags(Diags, *AST, CaptureDiagnostics); + AST->LangOpts = std::make_shared<LangOptions>(); AST->OnlyLocalDecls = OnlyLocalDecls; AST->CaptureDiagnostics = CaptureDiagnostics; AST->Diagnostics = Diags; @@ -682,7 +683,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFr AST->HeaderInfo.reset(new HeaderSearch(AST->HSOpts, AST->getSourceManager(), AST->getDiagnostics(), - AST->ASTFileLangOpts, + AST->getLangOpts(), /*Target=*/nullptr)); auto PPOpts = std::make_shared<PreprocessorOptions>(); @@ -696,13 +697,13 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFr unsigned Counter; AST->PP = std::make_shared<Preprocessor>( - std::move(PPOpts), AST->getDiagnostics(), AST->ASTFileLangOpts, + std::move(PPOpts), AST->getDiagnostics(), *AST->LangOpts, AST->getSourceManager(), *AST->PCMCache, HeaderInfo, *AST, /*IILookup=*/nullptr, /*OwnsHeaderSearch=*/false); Preprocessor &PP = *AST->PP; - AST->Ctx = new ASTContext(AST->ASTFileLangOpts, AST->getSourceManager(), + AST->Ctx = new ASTContext(*AST->LangOpts, AST->getSourceManager(), PP.getIdentifierTable(), PP.getSelectorTable(), PP.getBuiltinInfo()); ASTContext &Context = *AST->Ctx; @@ -716,7 +717,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFr AllowPCHWithCompilerErrors); AST->Reader->setListener(llvm::make_unique<ASTInfoCollector>( - *AST->PP, Context, AST->ASTFileLangOpts, AST->TargetOpts, AST->Target, + *AST->PP, Context, *AST->LangOpts, AST->TargetOpts, AST->Target, Counter)); // Attach the AST reader to the AST context as an external AST @@ -2879,7 +2880,32 @@ const FileEntry *ASTUnit::getPCHFile() { } bool ASTUnit::isModuleFile() { - return isMainFileAST() && ASTFileLangOpts.isCompilingModule(); + return isMainFileAST() && getLangOpts().isCompilingModule(); +} + +InputKind ASTUnit::getInputKind() const { + auto &LangOpts = getLangOpts(); + + InputKind::Language Lang; + if (LangOpts.OpenCL) + Lang = InputKind::OpenCL; + else if (LangOpts.CUDA) + Lang = InputKind::CUDA; + else if (LangOpts.RenderScript) + Lang = InputKind::RenderScript; + else if (LangOpts.CPlusPlus) + Lang = LangOpts.ObjC1 ? InputKind::ObjCXX : InputKind::CXX; + else + Lang = LangOpts.ObjC1 ? InputKind::ObjC : InputKind::C; + + InputKind::Format Fmt = InputKind::Source; + if (LangOpts.getCompilingModule() == LangOptions::CMK_ModuleMap) + Fmt = InputKind::ModuleMap; + + // We don't know if input was preprocessed. Assume not. + bool PP = false; + + return InputKind(Lang, Fmt, PP); } void ASTUnit::PreambleData::countLines() const { Modified: cfe/trunk/lib/Frontend/FrontendAction.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Frontend/FrontendAction.cpp?rev=304726&r1=304725&r2=304726&view=diff ============================================================================== --- cfe/trunk/lib/Frontend/FrontendAction.cpp (original) +++ cfe/trunk/lib/Frontend/FrontendAction.cpp Mon Jun 5 13:10:11 2017 @@ -387,8 +387,7 @@ static std::error_code collectModuleHead return std::error_code(); } -static bool loadModuleMapForModuleBuild(CompilerInstance &CI, - StringRef Filename, bool IsSystem, +static bool loadModuleMapForModuleBuild(CompilerInstance &CI, bool IsSystem, bool IsPreprocessed, std::string &PresumedModuleMapFile, unsigned &Offset) { @@ -523,7 +522,8 @@ getInputBufferForModule(CompilerInstance } bool FrontendAction::BeginSourceFile(CompilerInstance &CI, - const FrontendInputFile &Input) { + const FrontendInputFile &RealInput) { + FrontendInputFile Input(RealInput); assert(!Instance && "Already processing a source file!"); assert(!Input.isEmpty() && "Unexpected empty filename!"); setCurrentInput(Input); @@ -531,15 +531,69 @@ bool FrontendAction::BeginSourceFile(Com StringRef InputFile = Input.getFile(); bool HasBegunSourceFile = false; + bool ReplayASTFile = Input.getKind().getFormat() == InputKind::Precompiled && + usesPreprocessorOnly(); if (!BeginInvocation(CI)) goto failure; + // If we're replaying the build of an AST file, import it and set up + // the initial state from its build. + if (ReplayASTFile) { + IntrusiveRefCntPtr<DiagnosticsEngine> Diags(&CI.getDiagnostics()); + + // The AST unit populates its own diagnostics engine rather than ours. + IntrusiveRefCntPtr<DiagnosticsEngine> ASTDiags( + new DiagnosticsEngine(Diags->getDiagnosticIDs(), + &Diags->getDiagnosticOptions())); + ASTDiags->setClient(Diags->getClient(), /*OwnsClient*/false); + + std::unique_ptr<ASTUnit> AST = ASTUnit::LoadFromASTFile( + InputFile, CI.getPCHContainerReader(), ASTDiags, CI.getFileSystemOpts(), + CI.getCodeGenOpts().DebugTypeExtRefs); + if (!AST) + goto failure; + + // Options relating to how we treat the input (but not what we do with it) + // are inherited from the AST unit. + CI.getLangOpts() = AST->getLangOpts(); + + // Preload all the module files loaded transitively by the AST unit. + if (auto ASTReader = AST->getASTReader()) { + auto &MM = ASTReader->getModuleManager(); + for (ModuleFile &MF : MM) + if (&MF != &MM.getPrimaryModule()) + CI.getFrontendOpts().ModuleFiles.push_back(MF.FileName); + } + + // Set the shared objects, these are reset when we finish processing the + // file, otherwise the CompilerInstance will happily destroy them. + CI.setFileManager(&AST->getFileManager()); + CI.createSourceManager(CI.getFileManager()); + CI.getSourceManager().initializeForReplay(AST->getSourceManager()); + CI.createPreprocessor(getTranslationUnitKind()); + + // Set up the input file for replay purposes. + auto Kind = AST->getInputKind(); + if (Kind.getFormat() == InputKind::ModuleMap) { + Module *ASTModule = + AST->getPreprocessor().getHeaderSearchInfo().lookupModule( + AST->getLangOpts().CurrentModule, /*AllowSearch*/ false); + Input = FrontendInputFile(ASTModule->PresumedModuleMapFile, Kind); + } else { + auto &SM = CI.getSourceManager(); + FileID ID = SM.getMainFileID(); + if (auto *File = SM.getFileEntryForID(ID)) + Input = FrontendInputFile(File->getName(), Kind); + else + Input = FrontendInputFile(SM.getBuffer(ID), Kind); + } + setCurrentInput(Input, std::move(AST)); + } + // AST files follow a very different path, since they share objects via the // AST unit. if (Input.getKind().getFormat() == InputKind::Precompiled) { - // FIXME: We should not be asserting on bad command-line arguments. - assert(!usesPreprocessorOnly() && - "Attempt to pass AST file to preprocessor only action!"); + assert(!usesPreprocessorOnly() && "this case was handled above"); assert(hasASTFileSupport() && "This action does not have AST file support!"); @@ -680,7 +734,7 @@ bool FrontendAction::BeginSourceFile(Com std::string PresumedModuleMapFile; unsigned OffsetToContents; - if (loadModuleMapForModuleBuild(CI, Input.getFile(), Input.isSystem(), + if (loadModuleMapForModuleBuild(CI, Input.isSystem(), Input.isPreprocessed(), PresumedModuleMapFile, OffsetToContents)) goto failure; @@ -829,14 +883,7 @@ bool FrontendAction::BeginSourceFile(Com // If we failed, reset state since the client will not end up calling the // matching EndSourceFile(). - failure: - if (isCurrentFileAST()) { - CI.setASTContext(nullptr); - CI.setPreprocessor(nullptr); - CI.setSourceManager(nullptr); - CI.setFileManager(nullptr); - } - +failure: if (HasBegunSourceFile) CI.getDiagnosticClient().EndSourceFile(); CI.clearOutputFiles(/*EraseFiles=*/true); @@ -914,6 +961,7 @@ void FrontendAction::EndSourceFile() { CI.resetAndLeakPreprocessor(); CI.resetAndLeakSourceManager(); CI.resetAndLeakFileManager(); + BuryPointer(CurrentASTUnit.release()); } else { CI.setPreprocessor(nullptr); CI.setSourceManager(nullptr); Modified: cfe/trunk/lib/Serialization/ASTReader.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Serialization/ASTReader.cpp?rev=304726&r1=304725&r2=304726&view=diff ============================================================================== --- cfe/trunk/lib/Serialization/ASTReader.cpp (original) +++ cfe/trunk/lib/Serialization/ASTReader.cpp Mon Jun 5 13:10:11 2017 @@ -4918,6 +4918,7 @@ ASTReader::ReadSubmoduleBlock(ModuleFile } CurrentModule->setASTFile(F.File); + CurrentModule->PresumedModuleMapFile = F.ModuleMapPath; } CurrentModule->Kind = ModuleKind; Modified: cfe/trunk/test/Modules/preprocess-module.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Modules/preprocess-module.cpp?rev=304726&r1=304725&r2=304726&view=diff ============================================================================== --- cfe/trunk/test/Modules/preprocess-module.cpp (original) +++ cfe/trunk/test/Modules/preprocess-module.cpp Mon Jun 5 13:10:11 2017 @@ -34,10 +34,15 @@ // RUN: rm %t/fwd.h %t/file.h %t/file2.h %t/module.modulemap // RUN: %clang_cc1 -fmodules -fmodule-name=file -fmodule-file=%t/fwd.pcm -x c++-module-map-cpp-output %t/copy.ii -emit-module -o %t/copy.pcm -// Finally, check that our module contains correct mapping information for the headers. +// Check that our module contains correct mapping information for the headers. // RUN: cp %S/Inputs/preprocess/fwd.h %S/Inputs/preprocess/file.h %S/Inputs/preprocess/file2.h %S/Inputs/preprocess/module.modulemap %t // RUN: %clang_cc1 -fmodules -fmodule-file=%t/copy.pcm %s -I%t -verify -fno-modules-error-recovery -DCOPY -DINCLUDE +// Check that we can preprocess from a .pcm file and that we get the same result as preprocessing from the original sources. +// RUN: %clang_cc1 -fmodules -fmodule-name=file -fmodule-file=%t/fwd.pcm -I%S/Inputs/preprocess -x c++-module-map %S/Inputs/preprocess/module.modulemap -emit-module -o %t/file.pcm +// RUN: %clang_cc1 -fmodules -fmodule-name=file -fmodule-file=%t/fwd.pcm -I%S/Inputs/preprocess %t/file.pcm -E -frewrite-includes -o %t/file.rewrite.ii +// RUN: cmp %t/rewrite.ii %t/file.rewrite.ii + // == module map // CHECK: # 1 "{{.*}}module.modulemap" // CHECK: module file { _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits