Author: dblaikie Date: Wed Jun 6 13:52:13 2012 New Revision: 158093 URL: http://llvm.org/viewvc/llvm-project?rev=158093&view=rev Log: Add a -rewrite-includes option, which is similar to -rewrite-macros, but only expands #include directives.
Patch contributed by Lubos Lunak ([email protected]). Review by Matt Beaumont-Gay ([email protected]). Added: cfe/trunk/lib/Rewrite/InclusionRewriter.cpp cfe/trunk/test/Frontend/Inputs/rewrite-includes1.h cfe/trunk/test/Frontend/Inputs/rewrite-includes2.h cfe/trunk/test/Frontend/Inputs/rewrite-includes3.h cfe/trunk/test/Frontend/Inputs/rewrite-includes4.h cfe/trunk/test/Frontend/Inputs/rewrite-includes5.h cfe/trunk/test/Frontend/Inputs/rewrite-includes6.h cfe/trunk/test/Frontend/Inputs/rewrite-includes7.h cfe/trunk/test/Frontend/rewrite-includes.c Modified: cfe/trunk/include/clang/Driver/CC1Options.td cfe/trunk/include/clang/Frontend/FrontendOptions.h cfe/trunk/include/clang/Lex/Preprocessor.h cfe/trunk/include/clang/Rewrite/FrontendActions.h cfe/trunk/include/clang/Rewrite/Rewriters.h cfe/trunk/lib/Frontend/CompilerInvocation.cpp cfe/trunk/lib/FrontendTool/ExecuteCompilerInvocation.cpp cfe/trunk/lib/Lex/Lexer.cpp cfe/trunk/lib/Lex/PPDirectives.cpp cfe/trunk/lib/Lex/Preprocessor.cpp cfe/trunk/lib/Rewrite/CMakeLists.txt cfe/trunk/lib/Rewrite/FrontendActions.cpp Modified: cfe/trunk/include/clang/Driver/CC1Options.td URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Driver/CC1Options.td?rev=158093&r1=158092&r2=158093&view=diff ============================================================================== --- cfe/trunk/include/clang/Driver/CC1Options.td (original) +++ cfe/trunk/include/clang/Driver/CC1Options.td Wed Jun 6 13:52:13 2012 @@ -332,6 +332,8 @@ HelpText<"Rewriter playground">; def rewrite_macros : Flag<"-rewrite-macros">, HelpText<"Expand macros without full preprocessing">; +def rewrite_includes : Flag<"-rewrite-includes">, + HelpText<"Expand includes without full preprocessing">; def migrate : Flag<"-migrate">, HelpText<"Migrate source code">; } Modified: cfe/trunk/include/clang/Frontend/FrontendOptions.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Frontend/FrontendOptions.h?rev=158093&r1=158092&r2=158093&view=diff ============================================================================== --- cfe/trunk/include/clang/Frontend/FrontendOptions.h (original) +++ cfe/trunk/include/clang/Frontend/FrontendOptions.h Wed Jun 6 13:52:13 2012 @@ -43,6 +43,7 @@ PrintPreamble, ///< Print the "preamble" of the input file PrintPreprocessedInput, ///< -E mode. RewriteMacros, ///< Expand macros but not #includes. + RewriteIncludes, ///< Expand #includes but not macros. RewriteObjC, ///< ObjC->C Rewriter. RewriteTest, ///< Rewriter playground RunAnalysis, ///< Run one or more source code analyses. Modified: cfe/trunk/include/clang/Lex/Preprocessor.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/Preprocessor.h?rev=158093&r1=158092&r2=158093&view=diff ============================================================================== --- cfe/trunk/include/clang/Lex/Preprocessor.h (original) +++ cfe/trunk/include/clang/Lex/Preprocessor.h Wed Jun 6 13:52:13 2012 @@ -121,6 +121,13 @@ /// DisableMacroExpansion - True if macro expansion is disabled. bool DisableMacroExpansion : 1; + /// MacroExpansionInDirectivesOverride - Temporarily disables + /// DisableMacroExpansion (i.e. enables expansion) when parsing preprocessor + /// directives. + bool MacroExpansionInDirectivesOverride : 1; + + class ResetMacroExpansionHelper; + /// \brief Whether we have already loaded macros from the external source. mutable bool ReadMacrosFromExternalSource : 1; @@ -643,6 +650,12 @@ while (Result.getKind() == tok::comment); } + /// Disables macro expansion everywhere except for preprocessor directives. + void SetMacroExpansionOnlyInDirectives() { + DisableMacroExpansion = true; + MacroExpansionInDirectivesOverride = true; + } + /// LookAhead - This peeks ahead N tokens and returns that token without /// consuming any tokens. LookAhead(0) returns the next token that would be /// returned by Lex(), LookAhead(1) returns the token after it, etc. This Modified: cfe/trunk/include/clang/Rewrite/FrontendActions.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Rewrite/FrontendActions.h?rev=158093&r1=158092&r2=158093&view=diff ============================================================================== --- cfe/trunk/include/clang/Rewrite/FrontendActions.h (original) +++ cfe/trunk/include/clang/Rewrite/FrontendActions.h Wed Jun 6 13:52:13 2012 @@ -73,6 +73,11 @@ void ExecuteAction(); }; +class RewriteIncludesAction : public PreprocessorFrontendAction { +protected: + void ExecuteAction(); +}; + } // end namespace clang #endif Modified: cfe/trunk/include/clang/Rewrite/Rewriters.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Rewrite/Rewriters.h?rev=158093&r1=158092&r2=158093&view=diff ============================================================================== --- cfe/trunk/include/clang/Rewrite/Rewriters.h (original) +++ cfe/trunk/include/clang/Rewrite/Rewriters.h Wed Jun 6 13:52:13 2012 @@ -18,6 +18,7 @@ namespace clang { class Preprocessor; +class PreprocessorOutputOptions; /// RewriteMacrosInInput - Implement -rewrite-macros mode. void RewriteMacrosInInput(Preprocessor &PP, raw_ostream *OS); @@ -25,6 +26,10 @@ /// DoRewriteTest - A simple test for the TokenRewriter class. void DoRewriteTest(Preprocessor &PP, raw_ostream *OS); +/// RewriteIncludesInInput - Implement -rewrite-includes mode. +void RewriteIncludesInInput(Preprocessor &PP, raw_ostream *OS, + const PreprocessorOutputOptions &Opts); + } // end namespace clang #endif Modified: cfe/trunk/lib/Frontend/CompilerInvocation.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Frontend/CompilerInvocation.cpp?rev=158093&r1=158092&r2=158093&view=diff ============================================================================== --- cfe/trunk/lib/Frontend/CompilerInvocation.cpp (original) +++ cfe/trunk/lib/Frontend/CompilerInvocation.cpp Wed Jun 6 13:52:13 2012 @@ -445,6 +445,7 @@ case frontend::PrintPreamble: return "-print-preamble"; case frontend::PrintPreprocessedInput: return "-E"; case frontend::RewriteMacros: return "-rewrite-macros"; + case frontend::RewriteIncludes: return "-rewrite-includes"; case frontend::RewriteObjC: return "-rewrite-objc"; case frontend::RewriteTest: return "-rewrite-test"; case frontend::RunAnalysis: return "-analyze"; @@ -1435,6 +1436,8 @@ Opts.ProgramAction = frontend::PrintPreprocessedInput; break; case OPT_rewrite_macros: Opts.ProgramAction = frontend::RewriteMacros; break; + case OPT_rewrite_includes: + Opts.ProgramAction = frontend::RewriteIncludes; break; case OPT_rewrite_objc: Opts.ProgramAction = frontend::RewriteObjC; break; case OPT_rewrite_test: Modified: cfe/trunk/lib/FrontendTool/ExecuteCompilerInvocation.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/FrontendTool/ExecuteCompilerInvocation.cpp?rev=158093&r1=158092&r2=158093&view=diff ============================================================================== --- cfe/trunk/lib/FrontendTool/ExecuteCompilerInvocation.cpp (original) +++ cfe/trunk/lib/FrontendTool/ExecuteCompilerInvocation.cpp Wed Jun 6 13:52:13 2012 @@ -73,6 +73,7 @@ case PrintPreamble: return new PrintPreambleAction(); case PrintPreprocessedInput: return new PrintPreprocessedAction(); case RewriteMacros: return new RewriteMacrosAction(); + case RewriteIncludes: return new RewriteIncludesAction(); case RewriteObjC: return new RewriteObjCAction(); case RewriteTest: return new RewriteTestAction(); case RunAnalysis: return new ento::AnalysisAction(); Modified: cfe/trunk/lib/Lex/Lexer.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Lexer.cpp?rev=158093&r1=158092&r2=158093&view=diff ============================================================================== --- cfe/trunk/lib/Lex/Lexer.cpp (original) +++ cfe/trunk/lib/Lex/Lexer.cpp Wed Jun 6 13:52:13 2012 @@ -2022,7 +2022,7 @@ // directly. FormTokenWithChars(Result, CurPtr, tok::comment); - if (!ParsingPreprocessorDirective) + if (!ParsingPreprocessorDirective || LexingRawMode) return true; // If this BCPL-style comment is in a macro definition, transmogrify it into @@ -2626,7 +2626,8 @@ ParsingPreprocessorDirective = false; // Restore comment saving mode, in case it was disabled for directive. - SetCommentRetentionState(PP->getCommentRetentionState()); + if (!LexingRawMode) + SetCommentRetentionState(PP->getCommentRetentionState()); // Since we consumed a newline, we are back at the start of a line. IsAtStartOfLine = true; Modified: cfe/trunk/lib/Lex/PPDirectives.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/PPDirectives.cpp?rev=158093&r1=158092&r2=158093&view=diff ============================================================================== --- cfe/trunk/lib/Lex/PPDirectives.cpp (original) +++ cfe/trunk/lib/Lex/PPDirectives.cpp Wed Jun 6 13:52:13 2012 @@ -553,6 +553,21 @@ // Preprocessor Directive Handling. //===----------------------------------------------------------------------===// +class Preprocessor::ResetMacroExpansionHelper { +public: + ResetMacroExpansionHelper(Preprocessor *pp) + : PP(pp), save(pp->DisableMacroExpansion) { + if (pp->MacroExpansionInDirectivesOverride) + pp->DisableMacroExpansion = false; + } + ~ResetMacroExpansionHelper() { + PP->DisableMacroExpansion = save; + } +private: + Preprocessor *PP; + bool save; +}; + /// HandleDirective - This callback is invoked when the lexer sees a # token /// at the start of a line. This consumes the directive, modifies the /// lexer/preprocessor state, and advances the lexer(s) so that the next token @@ -604,6 +619,10 @@ Diag(Result, diag::ext_embedded_directive); } + // Temporarily enable macro expansion if set so + // and reset to previous state when returning from this function. + ResetMacroExpansionHelper helper(this); + TryAgain: switch (Result.getKind()) { case tok::eod: Modified: cfe/trunk/lib/Lex/Preprocessor.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Preprocessor.cpp?rev=158093&r1=158092&r2=158093&view=diff ============================================================================== --- cfe/trunk/lib/Lex/Preprocessor.cpp (original) +++ cfe/trunk/lib/Lex/Preprocessor.cpp Wed Jun 6 13:52:13 2012 @@ -86,6 +86,7 @@ // Macro expansion is enabled. DisableMacroExpansion = false; + MacroExpansionInDirectivesOverride = false; InMacroArgs = false; InMacroArgPreExpansion = false; NumCachedTokenLexers = 0; Modified: cfe/trunk/lib/Rewrite/CMakeLists.txt URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Rewrite/CMakeLists.txt?rev=158093&r1=158092&r2=158093&view=diff ============================================================================== --- cfe/trunk/lib/Rewrite/CMakeLists.txt (original) +++ cfe/trunk/lib/Rewrite/CMakeLists.txt Wed Jun 6 13:52:13 2012 @@ -6,6 +6,7 @@ FrontendActions.cpp HTMLPrint.cpp HTMLRewrite.cpp + InclusionRewriter.cpp RewriteMacros.cpp RewriteModernObjC.cpp RewriteObjC.cpp Modified: cfe/trunk/lib/Rewrite/FrontendActions.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Rewrite/FrontendActions.cpp?rev=158093&r1=158092&r2=158093&view=diff ============================================================================== --- cfe/trunk/lib/Rewrite/FrontendActions.cpp (original) +++ cfe/trunk/lib/Rewrite/FrontendActions.cpp Wed Jun 6 13:52:13 2012 @@ -181,3 +181,12 @@ DoRewriteTest(CI.getPreprocessor(), OS); } + +void RewriteIncludesAction::ExecuteAction() { + CompilerInstance &CI = getCompilerInstance(); + raw_ostream *OS = CI.createDefaultOutputFile(true, getCurrentFile()); + if (!OS) return; + + RewriteIncludesInInput(CI.getPreprocessor(), OS, + CI.getPreprocessorOutputOpts()); +} Added: cfe/trunk/lib/Rewrite/InclusionRewriter.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Rewrite/InclusionRewriter.cpp?rev=158093&view=auto ============================================================================== --- cfe/trunk/lib/Rewrite/InclusionRewriter.cpp (added) +++ cfe/trunk/lib/Rewrite/InclusionRewriter.cpp Wed Jun 6 13:52:13 2012 @@ -0,0 +1,370 @@ +//===--- InclusionRewriter.cpp - Rewrite includes into their expansions ---===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This code rewrites include invocations into their expansions. This gives you +// a file with all included files merged into it. +// +//===----------------------------------------------------------------------===// + +#include "clang/Rewrite/Rewriters.h" +#include "clang/Lex/Preprocessor.h" +#include "clang/Basic/SourceManager.h" +#include "clang/Frontend/PreprocessorOutputOptions.h" +#include "llvm/Support/raw_ostream.h" + +using namespace clang; +using namespace llvm; + +namespace { + +class InclusionRewriter : public PPCallbacks { + /// Information about which #includes were actually performed, + /// created by preprocessor callbacks. + struct FileChange { + SourceLocation From; + FileID Id; + SrcMgr::CharacteristicKind FileType; + FileChange(SourceLocation From) : From(From) { + } + }; + Preprocessor &PP; //< Used to find inclusion directives. + SourceManager &SM; //< Used to read and manage source files. + raw_ostream &OS; //< The destination stream for rewritten contents. + bool ShowLineMarkers; //< Show #line markers. + bool UseLineDirective; //< Use of line directives or line markers. + typedef std::map<unsigned, FileChange> FileChangeMap; + FileChangeMap FileChanges; /// Tracks which files were included where. + /// Used transitively for building up the FileChanges mapping over the + /// various \c PPCallbacks callbacks. + FileChangeMap::iterator LastInsertedFileChange; +public: + InclusionRewriter(Preprocessor &PP, raw_ostream &OS, bool ShowLineMarkers); + bool Process(FileID FileId, SrcMgr::CharacteristicKind FileType); +private: + virtual void FileChanged(SourceLocation Loc, FileChangeReason Reason, + SrcMgr::CharacteristicKind FileType, + FileID PrevFID); + virtual void FileSkipped(const FileEntry &ParentFile, + const Token &FilenameTok, + SrcMgr::CharacteristicKind FileType); + virtual void InclusionDirective(SourceLocation HashLoc, + const Token &IncludeTok, + StringRef FileName, + bool IsAngled, + const FileEntry *File, + SourceLocation EndLoc, + StringRef SearchPath, + StringRef RelativePath); + void WriteLineInfo(const char *Filename, int Line, + SrcMgr::CharacteristicKind FileType, + StringRef EOL, StringRef Extra = StringRef()); + void OutputContentUpTo(const MemoryBuffer &FromFile, + unsigned &WriteFrom, unsigned WriteTo, + StringRef EOL, int &lines, + bool EnsureNewline = false); + void CommentOutDirective(Lexer &DirectivesLex, const Token &StartToken, + const MemoryBuffer &FromFile, StringRef EOL, + unsigned &NextToWrite, int &Lines); + const FileChange *FindFileChangeLocation(SourceLocation Loc) const; + StringRef NextIdentifierName(Lexer &RawLex, Token &RawToken); +}; + +} // end anonymous namespace + +/// Initializes an InclusionRewriter with a \p PP source and \p OS destination. +InclusionRewriter::InclusionRewriter(Preprocessor &PP, raw_ostream &OS, + bool ShowLineMarkers) + : PP(PP), SM(PP.getSourceManager()), OS(OS), + ShowLineMarkers(ShowLineMarkers), + LastInsertedFileChange(FileChanges.end()) { + // If we're in microsoft mode, use normal #line instead of line markers. + UseLineDirective = PP.getLangOpts().MicrosoftExt; +} + +/// Write appropriate line information as either #line directives or GNU line +/// markers depending on what mode we're in, including the \p Filename and +/// \p Line we are located at, using the specified \p EOL line separator, and +/// any \p Extra context specifiers in GNU line directives. +void InclusionRewriter::WriteLineInfo(const char *Filename, int Line, + SrcMgr::CharacteristicKind FileType, + StringRef EOL, StringRef Extra) { + if (!ShowLineMarkers) + return; + if (UseLineDirective) { + OS << "#line" << ' ' << Line << ' ' << '"' << Filename << '"'; + } else { + // Use GNU linemarkers as described here: + // http://gcc.gnu.org/onlinedocs/cpp/Preprocessor-Output.html + OS << '#' << ' ' << Line << ' ' << '"' << Filename << '"'; + if (!Extra.empty()) + OS << Extra; + if (FileType == SrcMgr::C_System) + // "`3' This indicates that the following text comes from a system header + // file, so certain warnings should be suppressed." + OS << " 3"; + else if (FileType == SrcMgr::C_ExternCSystem) + // as above for `3', plus "`4' This indicates that the following text + // should be treated as being wrapped in an implicit extern "C" block." + OS << " 3 4"; + } + OS << EOL; +} + +/// FileChanged - Whenever the preprocessor enters or exits a #include file +/// it invokes this handler. +void InclusionRewriter::FileChanged(SourceLocation Loc, + FileChangeReason Reason, + SrcMgr::CharacteristicKind NewFileType, + FileID) { + if (Reason != EnterFile) + return; + if (LastInsertedFileChange == FileChanges.end()) + // we didn't reach this file (eg: the main file) via an inclusion directive + return; + LastInsertedFileChange->second.Id = FullSourceLoc(Loc, SM).getFileID(); + LastInsertedFileChange->second.FileType = NewFileType; + LastInsertedFileChange = FileChanges.end(); +} + +/// Called whenever an inclusion is skipped due to canonical header protection +/// macros. +void InclusionRewriter::FileSkipped(const FileEntry &/*ParentFile*/, + const Token &/*FilenameTok*/, + SrcMgr::CharacteristicKind /*FileType*/) { + assert(LastInsertedFileChange != FileChanges.end() && "A file, that wasn't " + "found via an inclusion directive, was skipped"); + FileChanges.erase(LastInsertedFileChange); + LastInsertedFileChange = FileChanges.end(); +} + +/// This should be called whenever the preprocessor encounters include +/// directives. It does not say whether the file has been included, but it +/// provides more information about the directive (hash location instead +/// of location inside the included file). It is assumed that the matching +/// FileChanged() or FileSkipped() is called after this. +void InclusionRewriter::InclusionDirective(SourceLocation HashLoc, + const Token &/*IncludeTok*/, + StringRef /*FileName*/, + bool /*IsAngled*/, + const FileEntry * /*File*/, + SourceLocation /*EndLoc*/, + StringRef /*SearchPath*/, + StringRef /*RelativePath*/) { + assert(LastInsertedFileChange == FileChanges.end() && "Another inclusion " + "directive was found before the previous one was processed"); + std::pair<FileChangeMap::iterator, bool> p = FileChanges.insert( + std::make_pair(HashLoc.getRawEncoding(), FileChange(HashLoc))); + assert(p.second && "Unexpected revisitation of the same include directive"); + LastInsertedFileChange = p.first; +} + +/// Simple lookup for a SourceLocation (specifically one denoting the hash in +/// an inclusion directive) in the map of inclusion information, FileChanges. +const InclusionRewriter::FileChange * +InclusionRewriter::FindFileChangeLocation(SourceLocation Loc) const { + FileChangeMap::const_iterator I = FileChanges.find(Loc.getRawEncoding()); + if (I != FileChanges.end()) + return &I->second; + return NULL; +} + +/// Count the raw \\n characters in the \p Len characters from \p Pos. +inline unsigned CountNewLines(const char *Pos, int Len) { + const char *End = Pos + Len; + unsigned Lines = 0; + --Pos; + while ((Pos = static_cast<const char*>(memchr(Pos + 1, '\n', End - Pos - 1)))) + ++Lines; + return Lines; +} + +/// Detect the likely line ending style of \p FromFile by examining the first +/// newline found within it. +static StringRef DetectEOL(const MemoryBuffer &FromFile) { + // detect what line endings the file uses, so that added content does not mix + // the style + const char *Pos = strchr(FromFile.getBufferStart(), '\n'); + if (Pos == NULL) + return "\n"; + if (Pos + 1 < FromFile.getBufferEnd() && Pos[1] == '\r') + return "\n\r"; + if (Pos - 1 >= FromFile.getBufferStart() && Pos[-1] == '\r') + return "\r\n"; + return "\n"; +} + +/// Writes out bytes from \p FromFile, starting at \p NextToWrite and ending at +/// \p WriteTo - 1. +void InclusionRewriter::OutputContentUpTo(const MemoryBuffer &FromFile, + unsigned &WriteFrom, unsigned WriteTo, + StringRef EOL, int &Line, + bool EnsureNewline) { + if (WriteTo <= WriteFrom) + return; + OS.write(FromFile.getBufferStart() + WriteFrom, WriteTo - WriteFrom); + // count lines manually, it's faster than getPresumedLoc() + Line += CountNewLines(FromFile.getBufferStart() + WriteFrom, + WriteTo - WriteFrom); + if (EnsureNewline) { + char LastChar = FromFile.getBufferStart()[WriteTo - 1]; + if (LastChar != '\n' && LastChar != '\r') + OS << EOL; + } + WriteFrom = WriteTo; +} + +/// Print characters from \p FromFile starting at \p NextToWrite up until the +/// inclusion directive at \p StartToken, then print out the inclusion +/// inclusion directive disabled by a #if directive, updating \p NextToWrite +/// and \p Line to track the number of source lines visited and the progress +/// through the \p FromFile buffer. +void InclusionRewriter::CommentOutDirective(Lexer &DirectiveLex, + const Token &StartToken, + const MemoryBuffer &FromFile, + StringRef EOL, + unsigned &NextToWrite, int &Line) { + OutputContentUpTo(FromFile, NextToWrite, + SM.getFileOffset(StartToken.getLocation()), EOL, Line); + Token DirectiveToken; + do { + DirectiveLex.LexFromRawLexer(DirectiveToken); + } while (!DirectiveToken.is(tok::eod) && DirectiveToken.isNot(tok::eof)); + OS << "#if 0 /* expanded by -rewrite-includes */" << EOL; + OutputContentUpTo(FromFile, NextToWrite, + SM.getFileOffset(DirectiveToken.getLocation()) + DirectiveToken.getLength(), + EOL, Line); + OS << "#endif /* expanded by -rewrite-includes */" << EOL; +} + +/// Find the next identifier in the pragma directive specified by \p RawToken. +StringRef InclusionRewriter::NextIdentifierName(Lexer &RawLex, + Token &RawToken) { + RawLex.LexFromRawLexer(RawToken); + if (RawToken.is(tok::raw_identifier)) + PP.LookUpIdentifierInfo(RawToken); + if (RawToken.is(tok::identifier)) + return RawToken.getIdentifierInfo()->getName(); + return StringRef(); +} + +/// Use a raw lexer to analyze \p FileId, inccrementally copying parts of it +/// and including content of included files recursively. +bool InclusionRewriter::Process(FileID FileId, + SrcMgr::CharacteristicKind FileType) +{ + bool Invalid; + const MemoryBuffer &FromFile = *SM.getBuffer(FileId, &Invalid); + assert(!Invalid && "Invalid FileID while trying to rewrite includes"); + const char *FileName = FromFile.getBufferIdentifier(); + Lexer RawLex(FileId, &FromFile, PP.getSourceManager(), PP.getLangOpts()); + RawLex.SetCommentRetentionState(false); + + StringRef EOL = DetectEOL(FromFile); + + // Per the GNU docs: "1" indicates the start of a new file. + WriteLineInfo(FileName, 1, FileType, EOL, " 1"); + + if (SM.getFileIDSize(FileId) == 0) + return true; + + // The next byte to be copied from the source file + unsigned NextToWrite = 0; + int Line = 1; // The current input file line number. + + Token RawToken; + RawLex.LexFromRawLexer(RawToken); + + // TODO: Consider adding a switch that strips possibly unimportant content, + // such as comments, to reduce the size of repro files. + while (RawToken.isNot(tok::eof)) { + if (RawToken.is(tok::hash) && RawToken.isAtStartOfLine()) { + RawLex.setParsingPreprocessorDirective(true); + Token HashToken = RawToken; + RawLex.LexFromRawLexer(RawToken); + if (RawToken.is(tok::raw_identifier)) + PP.LookUpIdentifierInfo(RawToken); + if (RawToken.is(tok::identifier)) { + switch (RawToken.getIdentifierInfo()->getPPKeywordID()) { + case tok::pp_include: + case tok::pp_include_next: + case tok::pp_import: { + CommentOutDirective(RawLex, HashToken, FromFile, EOL, NextToWrite, + Line); + if (const FileChange *Change = FindFileChangeLocation( + HashToken.getLocation())) { + // now include and recursively process the file + if (Process(Change->Id, Change->FileType)) + // and set lineinfo back to this file, if the nested one was + // actually included + // `2' indicates returning to a file (after having included + // another file. + WriteLineInfo(FileName, Line, FileType, EOL, " 2"); + } else + // fix up lineinfo (since commented out directive changed line + // numbers) for inclusions that were skipped due to header guards + WriteLineInfo(FileName, Line, FileType, EOL); + break; + } + case tok::pp_pragma: { + StringRef Identifier = NextIdentifierName(RawLex, RawToken); + if (Identifier == "clang" || Identifier == "GCC") { + if (NextIdentifierName(RawLex, RawToken) == "system_header") { + // keep the directive in, commented out + CommentOutDirective(RawLex, HashToken, FromFile, EOL, + NextToWrite, Line); + // update our own type + FileType = SM.getFileCharacteristic(RawToken.getLocation()); + WriteLineInfo(FileName, Line, FileType, EOL); + } + } else if (Identifier == "once") { + // keep the directive in, commented out + CommentOutDirective(RawLex, HashToken, FromFile, EOL, + NextToWrite, Line); + WriteLineInfo(FileName, Line, FileType, EOL); + } + break; + } + default: + break; + } + } + RawLex.setParsingPreprocessorDirective(false); + } + RawLex.LexFromRawLexer(RawToken); + } + OutputContentUpTo(FromFile, NextToWrite, + SM.getFileOffset(SM.getLocForEndOfFile(FileId)) + 1, EOL, Line, + /*EnsureNewline*/true); + return true; +} + +/// InclusionRewriterInInput - Implement -rewrite-includes mode. +void clang::RewriteIncludesInInput(Preprocessor &PP, raw_ostream *OS, + const PreprocessorOutputOptions &Opts) { + SourceManager &SM = PP.getSourceManager(); + InclusionRewriter *Rewrite = new InclusionRewriter(PP, *OS, + Opts.ShowLineMarkers); + PP.addPPCallbacks(Rewrite); + + // First let the preprocessor process the entire file and call callbacks. + // Callbacks will record which #include's were actually performed. + PP.EnterMainSourceFile(); + Token Tok; + // Only preprocessor directives matter here, so disable macro expansion + // everywhere else as an optimization. + // TODO: It would be even faster if the preprocessor could be switched + // to a mode where it would parse only preprocessor directives and comments, + // nothing else matters for parsing or processing. + PP.SetMacroExpansionOnlyInDirectives(); + do { + PP.Lex(Tok); + } while (Tok.isNot(tok::eof)); + Rewrite->Process(SM.getMainFileID(), SrcMgr::C_User); + OS->flush(); +} Added: cfe/trunk/test/Frontend/Inputs/rewrite-includes1.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Frontend/Inputs/rewrite-includes1.h?rev=158093&view=auto ============================================================================== --- cfe/trunk/test/Frontend/Inputs/rewrite-includes1.h (added) +++ cfe/trunk/test/Frontend/Inputs/rewrite-includes1.h Wed Jun 6 13:52:13 2012 @@ -0,0 +1,3 @@ +#pragma clang system_header +included_line1 +#include "rewrite-includes2.h" Added: cfe/trunk/test/Frontend/Inputs/rewrite-includes2.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Frontend/Inputs/rewrite-includes2.h?rev=158093&view=auto ============================================================================== --- cfe/trunk/test/Frontend/Inputs/rewrite-includes2.h (added) +++ cfe/trunk/test/Frontend/Inputs/rewrite-includes2.h Wed Jun 6 13:52:13 2012 @@ -0,0 +1 @@ +included_line2 Added: cfe/trunk/test/Frontend/Inputs/rewrite-includes3.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Frontend/Inputs/rewrite-includes3.h?rev=158093&view=auto ============================================================================== --- cfe/trunk/test/Frontend/Inputs/rewrite-includes3.h (added) +++ cfe/trunk/test/Frontend/Inputs/rewrite-includes3.h Wed Jun 6 13:52:13 2012 @@ -0,0 +1 @@ +included_line3 Added: cfe/trunk/test/Frontend/Inputs/rewrite-includes4.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Frontend/Inputs/rewrite-includes4.h?rev=158093&view=auto ============================================================================== --- cfe/trunk/test/Frontend/Inputs/rewrite-includes4.h (added) +++ cfe/trunk/test/Frontend/Inputs/rewrite-includes4.h Wed Jun 6 13:52:13 2012 @@ -0,0 +1 @@ +included_line4 Added: cfe/trunk/test/Frontend/Inputs/rewrite-includes5.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Frontend/Inputs/rewrite-includes5.h?rev=158093&view=auto ============================================================================== --- cfe/trunk/test/Frontend/Inputs/rewrite-includes5.h (added) +++ cfe/trunk/test/Frontend/Inputs/rewrite-includes5.h Wed Jun 6 13:52:13 2012 @@ -0,0 +1 @@ +included_line5 Added: cfe/trunk/test/Frontend/Inputs/rewrite-includes6.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Frontend/Inputs/rewrite-includes6.h?rev=158093&view=auto ============================================================================== --- cfe/trunk/test/Frontend/Inputs/rewrite-includes6.h (added) +++ cfe/trunk/test/Frontend/Inputs/rewrite-includes6.h Wed Jun 6 13:52:13 2012 @@ -0,0 +1,2 @@ +#pragma once +included_line6 Added: cfe/trunk/test/Frontend/Inputs/rewrite-includes7.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Frontend/Inputs/rewrite-includes7.h?rev=158093&view=auto ============================================================================== --- cfe/trunk/test/Frontend/Inputs/rewrite-includes7.h (added) +++ cfe/trunk/test/Frontend/Inputs/rewrite-includes7.h Wed Jun 6 13:52:13 2012 @@ -0,0 +1,4 @@ +#ifndef REWRITE_INCLUDES_7 +#define REWRITE_INCLUDES_7 +included_line7 +#endif Added: cfe/trunk/test/Frontend/rewrite-includes.c URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Frontend/rewrite-includes.c?rev=158093&view=auto ============================================================================== --- cfe/trunk/test/Frontend/rewrite-includes.c (added) +++ cfe/trunk/test/Frontend/rewrite-includes.c Wed Jun 6 13:52:13 2012 @@ -0,0 +1,145 @@ +// RUN: %clang_cc1 -verify -rewrite-includes -DFIRST -I %S/Inputs %s -o - | FileCheck -strict-whitespace %s +// RUN: %clang_cc1 -verify -rewrite-includes -P -DFIRST -I %S/Inputs %s -o - | FileCheck -check-prefix=CHECKNL -strict-whitespace %s +// STARTCOMPARE +#define A(a,b) a ## b +A(1,2) +#include "rewrite-includes1.h" +#ifdef FIRST +#define HEADER "rewrite-includes3.h" +#include HEADER +#else +#include "rewrite-includes4.h" +#endif +#/**/include /**/ "rewrite-includes5.h" /**/ \ + +#include "rewrite-includes6.h" // comment + +#include "rewrite-includes6.h" /* comment + continues */ +#include "rewrite-includes7.h" +#include "rewrite-includes7.h" +// ENDCOMPARE +// CHECK: {{^}}// STARTCOMPARE{{$}} +// CHECK-NEXT: {{^}}#define A(a,b) a ## b{{$}} +// CHECK-NEXT: {{^}}A(1,2){{$}} +// CHECK-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}} +// CHECK-NEXT: {{^}}#include "rewrite-includes1.h"{{$}} +// CHECK-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}} +// CHECK-NEXT: {{^}}# 1 "{{.*}}/Inputs/rewrite-includes1.h" 1{{$}} +// CHECK-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}} +// CHECK-NEXT: {{^}}#pragma clang system_header{{$}} +// CHECK-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}} +// CHECK-NEXT: {{^}}# 2 "{{.*}}/Inputs/rewrite-includes1.h" 3{{$}} +// CHECK-NEXT: {{^}}included_line1{{$}} +// CHECK-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}} +// CHECK-NEXT: {{^}}#include "rewrite-includes2.h"{{$}} +// CHECK-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}} +// CHECK-NEXT: {{^}}# 1 "{{.*}}/Inputs/rewrite-includes2.h" 1 3{{$}} +// CHECK-NEXT: {{^}}included_line2{{$}} +// CHECK-NEXT: {{^}}# 4 "{{.*}}/Inputs/rewrite-includes1.h" 2 3{{$}} +// CHECK-NEXT: {{^}}# 7 "{{.*}}rewrite-includes.c" 2{{$}} +// CHECK-NEXT: {{^}}#ifdef FIRST{{$}} +// CHECK-NEXT: {{^}}#define HEADER "rewrite-includes3.h"{{$}} +// CHECK-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}} +// CHECK-NEXT: {{^}}#include HEADER{{$}} +// CHECK-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}} +// CHECK-NEXT: {{^}}# 1 "{{.*}}/Inputs/rewrite-includes3.h" 1{{$}} +// CHECK-NEXT: {{^}}included_line3{{$}} +// CHECK-NEXT: {{^}}# 10 "{{.*}}rewrite-includes.c" 2{{$}} +// CHECK-NEXT: {{^}}#else{{$}} +// CHECK-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}} +// CHECK-NEXT: {{^}}#include "rewrite-includes4.h"{{$}} +// CHECK-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}} +// CHECK-NEXT: {{^}}# 12 "{{.*}}rewrite-includes.c"{{$}} +// CHECK-NEXT: {{^}}#endif{{$}} +// CHECK-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}} +// CHECK-NEXT: {{^}}#/**/include /**/ "rewrite-includes5.h" /**/ {{\\}}{{$}} +// CHECK-NEXT: {{^}} {{$}} +// CHECK-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}} +// CHECK-NEXT: {{^}}# 1 "{{.*}}/Inputs/rewrite-includes5.h" 1{{$}} +// CHECK-NEXT: {{^}}included_line5{{$}} +// CHECK-NEXT: {{^}}# 15 "{{.*}}rewrite-includes.c" 2{{$}} +// CHECK-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}} +// CHECK-NEXT: {{^}}#include "rewrite-includes6.h" // comment{{$}} +// CHECK-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}} +// CHECK-NEXT: {{^}}# 1 "{{.*}}/Inputs/rewrite-includes6.h" 1{{$}} +// CHECK-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}} +// CHECK-NEXT: {{^}}#pragma once{{$}} +// CHECK-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}} +// CHECK-NEXT: {{^}}# 2 "{{.*}}/Inputs/rewrite-includes6.h"{{$}} +// CHECK-NEXT: {{^}}included_line6{{$}} +// CHECK-NEXT: {{^}}# 16 "{{.*}}rewrite-includes.c" 2{{$}} +// CHECK-NEXT: {{^}} {{$}} +// CHECK-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}} +// CHECK-NEXT: {{^}}#include "rewrite-includes6.h" /* comment{{$}} +// CHECK-NEXT: {{^}} continues */{{$}} +// CHECK-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}} +// CHECK-NEXT: {{^}}# 19 "{{.*}}rewrite-includes.c"{{$}} +// CHECK-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}} +// CHECK-NEXT: {{^}}#include "rewrite-includes7.h"{{$}} +// CHECK-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}} +// CHECK-NEXT: {{^}}# 1 "{{.*}}/Inputs/rewrite-includes7.h" 1{{$}} +// CHECK-NEXT: {{^}}#ifndef REWRITE_INCLUDES_7{{$}} +// CHECK-NEXT: {{^}}#define REWRITE_INCLUDES_7{{$}} +// CHECK-NEXT: {{^}}included_line7{{$}} +// CHECK-NEXT: {{^}}#endif{{$}} +// CHECK-NEXT: {{^}}# 20 "{{.*}}rewrite-includes.c" 2{{$}} +// CHECK-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}} +// CHECK-NEXT: {{^}}#include "rewrite-includes7.h"{{$}} +// CHECK-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}} +// CHECK-NEXT: {{^}}# 21 "{{.*}}rewrite-includes.c"{{$}} +// CHECK-NEXT: {{^}}// ENDCOMPARE{{$}} + +// CHECKNL: {{^}}// STARTCOMPARE{{$}} +// CHECKNL-NEXT: {{^}}#define A(a,b) a ## b{{$}} +// CHECKNL-NEXT: {{^}}A(1,2){{$}} +// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}} +// CHECKNL-NEXT: {{^}}#include "rewrite-includes1.h"{{$}} +// CHECKNL-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}} +// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}} +// CHECKNL-NEXT: {{^}}#pragma clang system_header{{$}} +// CHECKNL-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}} +// CHECKNL-NEXT: {{^}}included_line1{{$}} +// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}} +// CHECKNL-NEXT: {{^}}#include "rewrite-includes2.h"{{$}} +// CHECKNL-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}} +// CHECKNL-NEXT: {{^}}included_line2{{$}} +// CHECKNL-NEXT: {{^}}#ifdef FIRST{{$}} +// CHECKNL-NEXT: {{^}}#define HEADER "rewrite-includes3.h"{{$}} +// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}} +// CHECKNL-NEXT: {{^}}#include HEADER{{$}} +// CHECKNL-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}} +// CHECKNL-NEXT: {{^}}included_line3{{$}} +// CHECKNL-NEXT: {{^}}#else{{$}} +// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}} +// CHECKNL-NEXT: {{^}}#include "rewrite-includes4.h"{{$}} +// CHECKNL-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}} +// CHECKNL-NEXT: {{^}}#endif{{$}} +// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}} +// CHECKNL-NEXT: {{^}}#/**/include /**/ "rewrite-includes5.h" /**/ {{\\}}{{$}} +// CHECKNL-NEXT: {{^}} {{$}} +// CHECKNL-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}} +// CHECKNL-NEXT: {{^}}included_line5{{$}} +// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}} +// CHECKNL-NEXT: {{^}}#include "rewrite-includes6.h" // comment{{$}} +// CHECKNL-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}} +// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}} +// CHECKNL-NEXT: {{^}}#pragma once{{$}} +// CHECKNL-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}} +// CHECKNL-NEXT: {{^}}included_line6{{$}} +// CHECKNL-NEXT: {{^}} {{$}} +// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}} +// CHECKNL-NEXT: {{^}}#include "rewrite-includes6.h" /* comment{{$}} +// CHECKNL-NEXT: {{^}} continues */{{$}} +// CHECKNL-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}} +// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}} +// CHECKNL-NEXT: {{^}}#include "rewrite-includes7.h"{{$}} +// CHECKNL-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}} +// CHECKNL-NEXT: {{^}}#ifndef REWRITE_INCLUDES_7{{$}} +// CHECKNL-NEXT: {{^}}#define REWRITE_INCLUDES_7{{$}} +// CHECKNL-NEXT: {{^}}included_line7{{$}} +// CHECKNL-NEXT: {{^}}#endif{{$}} +// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -rewrite-includes */{{$}} +// CHECKNL-NEXT: {{^}}#include "rewrite-includes7.h"{{$}} +// CHECKNL-NEXT: {{^}}#endif /* expanded by -rewrite-includes */{{$}} +// CHECKNL-NEXT: {{^}}// ENDCOMPARE{{$}} _______________________________________________ cfe-commits mailing list [email protected] http://lists.cs.uiuc.edu/mailman/listinfo/cfe-commits
