On Tue, Nov 30, 2010 at 1:16 AM, Douglas Gregor <[email protected]> wrote: > Author: dgregor > Date: Tue Nov 30 00:16:57 2010 > New Revision: 120396 > > URL: http://llvm.org/viewvc/llvm-project?rev=120396&view=rev > Log: > When using a precompiled preamble with detailed preprocessing records, > trap the serialized preprocessing records (macro definitions, macro > instantiations, macro definitions) from the generation of the > precompiled preamble, then replay those when walking the list of > preprocessed entities. This eliminates a bug where clang_getCursor() > wasn't able to find preprocessed-entity cursors in the preamble. > > Modified: > cfe/trunk/include/clang/Frontend/ASTUnit.h > cfe/trunk/include/clang/Lex/PreprocessingRecord.h > cfe/trunk/include/clang/Serialization/ASTReader.h > cfe/trunk/include/clang/Serialization/ASTWriter.h > cfe/trunk/lib/Frontend/ASTUnit.cpp > cfe/trunk/lib/Serialization/ASTReader.cpp > cfe/trunk/lib/Serialization/ASTWriter.cpp > cfe/trunk/lib/Serialization/GeneratePCH.cpp > cfe/trunk/test/Index/c-index-getCursor-pp.c > cfe/trunk/tools/libclang/CIndex.cpp > > Modified: cfe/trunk/include/clang/Frontend/ASTUnit.h > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Frontend/ASTUnit.h?rev=120396&r1=120395&r2=120396&view=diff > ============================================================================== > --- cfe/trunk/include/clang/Frontend/ASTUnit.h (original) > +++ cfe/trunk/include/clang/Frontend/ASTUnit.h Tue Nov 30 00:16:57 2010 > @@ -110,6 +110,14 @@ > // more scalable search mechanisms. > std::vector<Decl*> TopLevelDecls; > > + /// \brief The list of preprocessed entities which appeared when the > ASTUnit > + /// was loaded. > + /// > + /// FIXME: This is just an optimization hack to avoid deserializing large > + /// parts of a PCH file while performing a walk or search. In the long > term, > + /// we should provide more scalable search mechanisms. > + std::vector<PreprocessedEntity *> PreprocessedEntities; > + > /// The name of the original source file used to generate this ASTUnit. > std::string OriginalSourceFile; > > @@ -215,6 +223,10 @@ > /// declarations parsed within the precompiled preamble. > std::vector<serialization::DeclID> TopLevelDeclsInPreamble; > > + /// \brief A list of the offsets into the precompiled preamble which > + /// correspond to preprocessed entities. > + std::vector<uint64_t> PreprocessedEntitiesInPreamble; > + > /// \brief Whether we should be caching code-completion results. > bool ShouldCacheCodeCompletionResults; > > @@ -317,7 +329,8 @@ > bool AllowRebuild = true, > unsigned MaxLines = > 0); > void RealizeTopLevelDeclsFromPreamble(); > - > + void RealizePreprocessedEntitiesFromPreamble(); > + > public: > class ConcurrencyCheck { > volatile ASTUnit &Self; > @@ -426,6 +439,17 @@ > TopLevelDeclsInPreamble.push_back(D); > } > > + typedef std::vector<PreprocessedEntity *>::iterator pp_entity_iterator; > + > + pp_entity_iterator pp_entity_begin(); > + pp_entity_iterator pp_entity_end(); > + > + /// \brief Add a new preprocessed entity that's stored at the given offset > + /// in the precompiled preamble. > + void addPreprocessedEntityFromPreamble(uint64_t Offset) { > + PreprocessedEntitiesInPreamble.push_back(Offset); > + } > + > /// \brief Retrieve the mapping from File IDs to the preprocessed entities > /// within that file. > PreprocessedEntitiesByFileMap &getPreprocessedEntitiesByFile() { > > Modified: cfe/trunk/include/clang/Lex/PreprocessingRecord.h > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/PreprocessingRecord.h?rev=120396&r1=120395&r2=120396&view=diff > ============================================================================== > --- cfe/trunk/include/clang/Lex/PreprocessingRecord.h (original) > +++ cfe/trunk/include/clang/Lex/PreprocessingRecord.h Tue Nov 30 00:16:57 2010 > @@ -248,6 +248,9 @@ > /// \brief Read any preallocated preprocessed entities from the external > /// source. > virtual void ReadPreprocessedEntities() = 0; > + > + /// \brief Read the preprocessed entity at the given offset. > + virtual PreprocessedEntity *ReadPreprocessedEntity(uint64_t Offset) = 0; > }; > > /// \brief A record of the steps taken while preprocessing a source file, > @@ -302,6 +305,11 @@ > void SetExternalSource(ExternalPreprocessingRecordSource &Source, > unsigned NumPreallocatedEntities); > > + /// \brief Retrieve the external source for preprocessed entities. > + ExternalPreprocessingRecordSource *getExternalSource() const { > + return ExternalSource; > + } > + > unsigned getNumPreallocatedEntities() const { > return NumPreallocatedEntities; > } > > Modified: cfe/trunk/include/clang/Serialization/ASTReader.h > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Serialization/ASTReader.h?rev=120396&r1=120395&r2=120396&view=diff > ============================================================================== > --- cfe/trunk/include/clang/Serialization/ASTReader.h (original) > +++ cfe/trunk/include/clang/Serialization/ASTReader.h Tue Nov 30 00:16:57 2010 > @@ -845,9 +845,12 @@ > /// build prior to including the precompiled header. > const std::string &getSuggestedPredefines() { return SuggestedPredefines; } > > - /// \brief Read preprocessed entities into the > + /// \brief Read preprocessed entities into the preprocessing record. > virtual void ReadPreprocessedEntities(); > > + /// \brief Read the preprocessed entity at the given offset. > + virtual PreprocessedEntity *ReadPreprocessedEntity(uint64_t Offset); > + > void ReadUserDiagnosticMappings(Diagnostic &Diag); > > /// \brief Returns the number of source locations found in the chain. > @@ -1154,7 +1157,7 @@ > Expr *ReadSubExpr(); > > /// \brief Reads the macro record located at the given offset. > - void ReadMacroRecord(PerFileData &F, uint64_t Offset); > + PreprocessedEntity *ReadMacroRecord(PerFileData &F, uint64_t Offset); > > /// \brief Note that the identifier is a macro whose record will be loaded > /// from the given AST file at the given (file-local) offset. > > Modified: cfe/trunk/include/clang/Serialization/ASTWriter.h > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Serialization/ASTWriter.h?rev=120396&r1=120395&r2=120396&view=diff > ============================================================================== > --- cfe/trunk/include/clang/Serialization/ASTWriter.h (original) > +++ cfe/trunk/include/clang/Serialization/ASTWriter.h Tue Nov 30 00:16:57 2010 > @@ -23,6 +23,7 @@ > #include "clang/Sema/SemaConsumer.h" > #include "llvm/ADT/SmallPtrSet.h" > #include "llvm/ADT/SmallVector.h" > +#include "llvm/ADT/DenseMap.h" > #include "llvm/Bitcode/BitstreamWriter.h" > #include <map> > #include <queue> > @@ -37,6 +38,7 @@ > namespace clang { > > class ASTContext; > +class ASTSerializationListener; > class NestedNameSpecifier; > class CXXBaseSpecifier; > class CXXBaseOrMemberInitializer; > @@ -44,6 +46,7 @@ > class MacroDefinition; > class MemorizeStatCalls; > class ASTReader; > +class PreprocessedEntity; > class Preprocessor; > class Sema; > class SourceManager; > @@ -70,6 +73,10 @@ > /// \brief The reader of existing AST files, if we're chaining. > ASTReader *Chain; > > + /// \brief A listener object that receives notifications when certain > + /// entities are serialized. > + ASTSerializationListener *SerializationListener; > + > /// \brief Stores a declaration or a type to be written to the AST file. > class DeclOrType { > public: > @@ -334,6 +341,12 @@ > /// the given bitstream. > ASTWriter(llvm::BitstreamWriter &Stream); > > + /// \brief Set the listener that will receive notification of serialization > + /// events. > + void SetSerializationListener(ASTSerializationListener *Listener) { > + SerializationListener = Listener; > + } > + > /// \brief Write a precompiled header for the given semantic analysis. > /// > /// \param SemaRef a reference to the semantic analysis object that > processed > @@ -573,6 +586,7 @@ > virtual void InitializeSema(Sema &S) { SemaPtr = &S; } > virtual void HandleTranslationUnit(ASTContext &Ctx); > virtual ASTMutationListener *GetASTMutationListener(); > + virtual ASTSerializationListener *GetASTSerializationListener(); > virtual ASTDeserializationListener *GetASTDeserializationListener(); > }; > > > Modified: cfe/trunk/lib/Frontend/ASTUnit.cpp > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Frontend/ASTUnit.cpp?rev=120396&r1=120395&r2=120396&view=diff > ============================================================================== > --- cfe/trunk/lib/Frontend/ASTUnit.cpp (original) > +++ cfe/trunk/lib/Frontend/ASTUnit.cpp Tue Nov 30 00:16:57 2010 > @@ -27,6 +27,7 @@ > #include "clang/Frontend/FrontendOptions.h" > #include "clang/Frontend/Utils.h" > #include "clang/Serialization/ASTReader.h" > +#include "clang/Serialization/ASTSerializationListener.h" > #include "clang/Serialization/ASTWriter.h" > #include "clang/Lex/HeaderSearch.h" > #include "clang/Lex/Preprocessor.h" > @@ -637,10 +638,11 @@ > } > }; > > -class PrecompilePreambleConsumer : public PCHGenerator { > +class PrecompilePreambleConsumer : public PCHGenerator, > + public ASTSerializationListener { > ASTUnit &Unit; > std::vector<Decl *> TopLevelDecls; > - > + > public: > PrecompilePreambleConsumer(ASTUnit &Unit, > const Preprocessor &PP, bool Chaining, > @@ -672,6 +674,15 @@ > > getWriter().getDeclID(TopLevelDecls[I])); > } > } > + > + virtual void SerializedPreprocessedEntity(PreprocessedEntity *Entity, > + uint64_t Offset) { > + Unit.addPreprocessedEntityFromPreamble(Offset); > + } > + > + virtual ASTSerializationListener *GetASTSerializationListener() { > + return this; > + } > }; > > class PrecompilePreambleAction : public ASTFrontendAction { > @@ -757,6 +768,7 @@ > > // Clear out old caches and data. > TopLevelDecls.clear(); > + PreprocessedEntities.clear(); > CleanTemporaryFiles(); > PreprocessedEntitiesByFile.clear(); > > @@ -765,6 +777,7 @@ > StoredDiagnostics.begin() + > NumStoredDiagnosticsFromDriver, > StoredDiagnostics.end()); > TopLevelDeclsInPreamble.clear(); > + PreprocessedEntitiesInPreamble.clear(); > } > > // Create a file manager object to provide access to and cache the > filesystem. > @@ -1237,6 +1250,8 @@ > StoredDiagnostics.end()); > TopLevelDecls.clear(); > TopLevelDeclsInPreamble.clear(); > + PreprocessedEntities.clear(); > + PreprocessedEntitiesInPreamble.clear(); > > // Create a file manager object to provide access to and cache the > filesystem. > Clang.setFileManager(new FileManager(Clang.getFileSystemOpts())); > @@ -1269,6 +1284,8 @@ > llvm::sys::Path(FrontendOpts.OutputFile).eraseFromDisk(); > Preamble.clear(); > TopLevelDeclsInPreamble.clear(); > + PreprocessedEntities.clear(); > + PreprocessedEntitiesInPreamble.clear(); > PreambleRebuildCounter = DefaultPreambleRebuildInterval; > PreprocessorOpts.eraseRemappedFile( > PreprocessorOpts.remapped_file_buffer_end() - > 1); > @@ -1321,6 +1338,55 @@ > TopLevelDecls.insert(TopLevelDecls.begin(), Resolved.begin(), > Resolved.end()); > } > > +void ASTUnit::RealizePreprocessedEntitiesFromPreamble() { > + if (!PP) > + return; > + > + PreprocessingRecord *PPRec = PP->getPreprocessingRecord(); > + if (!PPRec) > + return; > + > + ExternalPreprocessingRecordSource *External = PPRec->getExternalSource(); > + if (!External) > + return; > + > + for (unsigned I = 0, N = PreprocessedEntitiesInPreamble.size(); I != N; > ++I) { > + if (PreprocessedEntity *PE > + = > External->ReadPreprocessedEntity(PreprocessedEntitiesInPreamble[I])) > + PreprocessedEntities.push_back(PE); > + } > + > + if (PreprocessedEntities.empty()) > + return; > + > + PreprocessedEntities.insert(PreprocessedEntities.end(), > + PPRec->begin(true), PPRec->end(true));
Hi, I added a operator< to PreprocessingRecord::iterator otherwise MSVC will refuse to compile the insert line in DEBUG mode. see r135670 _______________________________________________ cfe-commits mailing list [email protected] http://lists.cs.uiuc.edu/mailman/listinfo/cfe-commits
