ilya-biryukov created this revision.
ilya-biryukov added a reviewer: rsmith.
Herald added a project: clang.
By adding a hook to consume all tokens produced by the preprocessor.
The intention of this change is to make it possible to consume the
expanded tokens without re-runnig the preprocessor with minimal changes
to the preprocessor and minimal performance penalty when preprocessing
without recording the tokens.
The added hook is very low-level and reconstructing the expanded token
stream requires more work in the client code, the actual algorithm to
collect the tokens using this hook can be found in the follow-up change.
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D59885
Files:
clang/include/clang/Lex/Preprocessor.h
clang/lib/Lex/Preprocessor.cpp
Index: clang/lib/Lex/Preprocessor.cpp
===================================================================
--- clang/lib/Lex/Preprocessor.cpp
+++ clang/lib/Lex/Preprocessor.cpp
@@ -864,6 +864,7 @@
void Preprocessor::Lex(Token &Result) {
// We loop here until a lex function returns a token; this avoids recursion.
bool ReturnedToken;
+ bool Cached = false;
do {
switch (CurLexerKind) {
case CLK_Lexer:
@@ -875,6 +876,7 @@
case CLK_CachingLexer:
CachingLex(Result);
ReturnedToken = true;
+ Cached = true;
break;
case CLK_LexAfterModuleImport:
LexAfterModuleImport(Result);
@@ -893,6 +895,8 @@
}
LastTokenWasAt = Result.is(tok::at);
+ if (OnToken && ReturnedToken && !Cached)
+ OnToken(Result);
}
/// Lex a header-name token (including one formed from header-name-tokens if
Index: clang/include/clang/Lex/Preprocessor.h
===================================================================
--- clang/include/clang/Lex/Preprocessor.h
+++ clang/include/clang/Lex/Preprocessor.h
@@ -33,6 +33,7 @@
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/FoldingSet.h"
+#include "llvm/ADT/FunctionExtras.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/PointerUnion.h"
@@ -48,8 +49,8 @@
#include <cassert>
#include <cstddef>
#include <cstdint>
-#include <memory>
#include <map>
+#include <memory>
#include <string>
#include <utility>
#include <vector>
@@ -124,6 +125,7 @@
friend class VAOptDefinitionContext;
friend class VariadicMacroScopeGuard;
+ llvm::unique_function<void(const clang::Token &)> OnToken;
std::shared_ptr<PreprocessorOptions> PPOpts;
DiagnosticsEngine *Diags;
LangOptions &LangOpts;
@@ -911,6 +913,14 @@
}
/// \}
+ /// Register a function that would be called on each token seen by the
+ /// preprocessor. This is a very low-level hook, the produced token stream is
+ /// tied to the internals of the preprocessor so interpreting result of the
+ /// callback is hard.
+ void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) {
+ OnToken = std::move(F);
+ }
+
bool isMacroDefined(StringRef Id) {
return isMacroDefined(&Identifiers.get(Id));
}
Index: clang/lib/Lex/Preprocessor.cpp
===================================================================
--- clang/lib/Lex/Preprocessor.cpp
+++ clang/lib/Lex/Preprocessor.cpp
@@ -864,6 +864,7 @@
void Preprocessor::Lex(Token &Result) {
// We loop here until a lex function returns a token; this avoids recursion.
bool ReturnedToken;
+ bool Cached = false;
do {
switch (CurLexerKind) {
case CLK_Lexer:
@@ -875,6 +876,7 @@
case CLK_CachingLexer:
CachingLex(Result);
ReturnedToken = true;
+ Cached = true;
break;
case CLK_LexAfterModuleImport:
LexAfterModuleImport(Result);
@@ -893,6 +895,8 @@
}
LastTokenWasAt = Result.is(tok::at);
+ if (OnToken && ReturnedToken && !Cached)
+ OnToken(Result);
}
/// Lex a header-name token (including one formed from header-name-tokens if
Index: clang/include/clang/Lex/Preprocessor.h
===================================================================
--- clang/include/clang/Lex/Preprocessor.h
+++ clang/include/clang/Lex/Preprocessor.h
@@ -33,6 +33,7 @@
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/FoldingSet.h"
+#include "llvm/ADT/FunctionExtras.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/PointerUnion.h"
@@ -48,8 +49,8 @@
#include <cassert>
#include <cstddef>
#include <cstdint>
-#include <memory>
#include <map>
+#include <memory>
#include <string>
#include <utility>
#include <vector>
@@ -124,6 +125,7 @@
friend class VAOptDefinitionContext;
friend class VariadicMacroScopeGuard;
+ llvm::unique_function<void(const clang::Token &)> OnToken;
std::shared_ptr<PreprocessorOptions> PPOpts;
DiagnosticsEngine *Diags;
LangOptions &LangOpts;
@@ -911,6 +913,14 @@
}
/// \}
+ /// Register a function that would be called on each token seen by the
+ /// preprocessor. This is a very low-level hook, the produced token stream is
+ /// tied to the internals of the preprocessor so interpreting result of the
+ /// callback is hard.
+ void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) {
+ OnToken = std::move(F);
+ }
+
bool isMacroDefined(StringRef Id) {
return isMacroDefined(&Identifiers.get(Id));
}
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
