[PATCH] D59885: [Lex] Allow to consume tokens while preprocessing

[Ilya Biryukov via Phabricator via cfe-commits]

ilya-biryukov updated this revision to Diff 198663.
ilya-biryukov added a comment.

- Get rid of an accidental change in how LexAfterModuleImport is handled


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D59885/new/

https://reviews.llvm.org/D59885

Files:
  clang/include/clang/Lex/Preprocessor.h
  clang/include/clang/Lex/TokenLexer.h
  clang/lib/Lex/PPCaching.cpp
  clang/lib/Lex/Preprocessor.cpp


Index: clang/lib/Lex/Preprocessor.cpp
===================================================================
--- clang/lib/Lex/Preprocessor.cpp
+++ clang/lib/Lex/Preprocessor.cpp
@@ -636,7 +636,8 @@
       break;
     case CLK_CachingLexer:
       bool IsNewToken;
-      CachingLex(Tok, IsNewToken);
+      bool Report;
+      CachingLex(Tok, IsNewToken, Report);
       break;
     case CLK_LexAfterModuleImport:
       LexAfterModuleImport(Tok);
@@ -877,6 +878,11 @@
 }
 
 void Preprocessor::Lex(Token &Result) {
+  bool Report;
+  Lex(Result, Report);
+}
+
+void Preprocessor::Lex(Token &Result, bool &Report) {
   ++LexLevel;
 
   // We loop here until a lex function returns a token; this avoids recursion.
@@ -885,17 +891,20 @@
   do {
     switch (CurLexerKind) {
     case CLK_Lexer:
+      Report = true;
       ReturnedToken = CurLexer->Lex(Result);
       break;
     case CLK_TokenLexer:
+      Report = CurTokenLexer->isMacroExpansion();
       ReturnedToken = CurTokenLexer->Lex(Result);
       break;
     case CLK_CachingLexer:
-      CachingLex(Result, IsNewToken);
+      CachingLex(Result, IsNewToken, Report);
       ReturnedToken = true;
       break;
     case CLK_LexAfterModuleImport:
       ReturnedToken = LexAfterModuleImport(Result);
+      Report = true;
       break;
     }
   } while (!ReturnedToken);
@@ -952,6 +961,8 @@
 
   LastTokenWasAt = Result.is(tok::at);
   --LexLevel;
+  if (OnToken && LexLevel == 0 && Report)
+    OnToken(Result);
 }
 
 /// Lex a header-name token (including one formed from header-name-tokens if
Index: clang/lib/Lex/PPCaching.cpp
===================================================================
--- clang/lib/Lex/PPCaching.cpp
+++ clang/lib/Lex/PPCaching.cpp
@@ -45,7 +45,7 @@
   recomputeCurLexerKind();
 }
 
-void Preprocessor::CachingLex(Token &Result, bool &IsNewToken) {
+void Preprocessor::CachingLex(Token &Result, bool &IsNewToken, bool &Report) {
   if (!InCachingLexMode())
     return;
 
@@ -56,11 +56,12 @@
   if (CachedLexPos < CachedTokens.size()) {
     Result = CachedTokens[CachedLexPos++];
     IsNewToken = false;
+    Report = false;
     return;
   }
 
   ExitCachingLexMode();
-  Lex(Result);
+  Lex(Result, Report);
 
   if (isBacktrackEnabled()) {
     // Cache the lexed token.
Index: clang/include/clang/Lex/TokenLexer.h
===================================================================
--- clang/include/clang/Lex/TokenLexer.h
+++ clang/include/clang/Lex/TokenLexer.h
@@ -147,6 +147,10 @@
   /// preprocessor directive.
   bool isParsingPreprocessorDirective() const;
 
+  /// Returns true iff the TokenLexer is expanding a macro and not replaying a
+  /// stream of tokens.
+  bool isMacroExpansion() const { return Macro != nullptr; }
+
 private:
   void destroy();
 
Index: clang/include/clang/Lex/Preprocessor.h
===================================================================
--- clang/include/clang/Lex/Preprocessor.h
+++ clang/include/clang/Lex/Preprocessor.h
@@ -33,6 +33,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/FoldingSet.h"
+#include "llvm/ADT/FunctionExtras.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/PointerUnion.h"
@@ -48,8 +49,8 @@
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
-#include <memory>
 #include <map>
+#include <memory>
 #include <string>
 #include <utility>
 #include <vector>
@@ -124,6 +125,7 @@
   friend class VAOptDefinitionContext;
   friend class VariadicMacroScopeGuard;
 
+  llvm::unique_function<void(const clang::Token &)> OnToken;
   std::shared_ptr<PreprocessorOptions> PPOpts;
   DiagnosticsEngine        *Diags;
   LangOptions       &LangOpts;
@@ -997,6 +999,12 @@
   }
   /// \}
 
+  /// Register a function that would be called on each token in the final
+  /// expanded token stream.
+  void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) {
+    OnToken = std::move(F);
+  }
+
   bool isMacroDefined(StringRef Id) {
     return isMacroDefined(&Identifiers.get(Id));
   }
@@ -1341,6 +1349,12 @@
   /// Lex the next token for this preprocessor.
   void Lex(Token &Result);
 
+private:
+  /// Lex the next token and tell whether it should be reported to OnToken
+  /// callback.
+  void Lex(Token &Result, bool &Report);
+
+public:
   /// Lex a token, forming a header-name token if possible.
   bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
 
@@ -2135,7 +2149,7 @@
 
   //===--------------------------------------------------------------------===//
   // Caching stuff.
-  void CachingLex(Token &Result, bool &IsNewToken);
+  void CachingLex(Token &Result, bool &IsNewToken, bool &Report);
 
   bool InCachingLexMode() const {
     // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means

_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits