junaire created this revision.
Herald added a project: All.
junaire requested review of this revision.
Herald added a project: clang.
Herald added a subscriber: cfe-commits.
This patches adds a new annotation token called annot_input_end, which
can be used to capture expression in value printing. It works somewhat
like annot_end_of_module token we already have.
This approach is proposed by Richard Smith, thanks!
Signed-off-by: Jun Zhang <j...@junz.org>
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D145316
Files:
clang/include/clang/Basic/TokenKinds.def
clang/include/clang/Parse/Parser.h
clang/lib/Frontend/PrintPreprocessedOutput.cpp
clang/lib/Interpreter/IncrementalParser.cpp
clang/lib/Lex/PPLexerChange.cpp
clang/lib/Parse/ParseCXXInlineMethods.cpp
clang/lib/Parse/ParseDecl.cpp
clang/lib/Parse/Parser.cpp
Index: clang/lib/Parse/Parser.cpp
===================================================================
--- clang/lib/Parse/Parser.cpp
+++ clang/lib/Parse/Parser.cpp
@@ -320,6 +320,7 @@
case tok::annot_module_begin:
case tok::annot_module_end:
case tok::annot_module_include:
+ case tok::annot_input_end:
// Stop before we change submodules. They generally indicate a "good"
// place to pick up parsing again (except in the special case where
// we're trying to skip to EOF).
@@ -616,8 +617,8 @@
// Skip over the EOF token, flagging end of previous input for incremental
// processing
- if (PP.isIncrementalProcessingEnabled() && Tok.is(tok::eof))
- ConsumeToken();
+ if (PP.isIncrementalProcessingEnabled() && Tok.is(tok::annot_input_end))
+ ConsumeAnyToken();
Result = nullptr;
switch (Tok.getKind()) {
@@ -697,6 +698,7 @@
return false;
case tok::eof:
+ case tok::annot_input_end:
// Check whether -fmax-tokens= was reached.
if (PP.getMaxTokens() != 0 && PP.getTokenCount() > PP.getMaxTokens()) {
PP.Diag(Tok.getLocation(), diag::warn_max_tokens_total)
Index: clang/lib/Parse/ParseDecl.cpp
===================================================================
--- clang/lib/Parse/ParseDecl.cpp
+++ clang/lib/Parse/ParseDecl.cpp
@@ -2061,6 +2061,7 @@
case tok::annot_module_begin:
case tok::annot_module_end:
case tok::annot_module_include:
+ case tok::annot_input_end:
return;
default:
Index: clang/lib/Parse/ParseCXXInlineMethods.cpp
===================================================================
--- clang/lib/Parse/ParseCXXInlineMethods.cpp
+++ clang/lib/Parse/ParseCXXInlineMethods.cpp
@@ -836,6 +836,7 @@
case tok::annot_module_begin:
case tok::annot_module_end:
case tok::annot_module_include:
+ case tok::annot_input_end:
// Ran out of tokens.
return false;
@@ -1242,6 +1243,7 @@
case tok::annot_module_begin:
case tok::annot_module_end:
case tok::annot_module_include:
+ case tok::annot_input_end:
// Ran out of tokens.
return false;
Index: clang/lib/Lex/PPLexerChange.cpp
===================================================================
--- clang/lib/Lex/PPLexerChange.cpp
+++ clang/lib/Lex/PPLexerChange.cpp
@@ -535,13 +535,19 @@
return LeavingSubmodule;
}
}
-
// If this is the end of the main file, form an EOF token.
assert(CurLexer && "Got EOF but no current lexer set!");
const char *EndPos = getCurLexerEndPos();
Result.startToken();
CurLexer->BufferPtr = EndPos;
- CurLexer->FormTokenWithChars(Result, EndPos, tok::eof);
+
+ if (isIncrementalProcessingEnabled()) {
+ CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_input_end);
+ Result.setAnnotationEndLoc(Result.getLocation());
+ Result.setAnnotationValue(nullptr);
+ } else {
+ CurLexer->FormTokenWithChars(Result, EndPos, tok::eof);
+ }
if (isCodeCompletionEnabled()) {
// Inserting the code-completion point increases the source buffer by 1,
Index: clang/lib/Interpreter/IncrementalParser.cpp
===================================================================
--- clang/lib/Interpreter/IncrementalParser.cpp
+++ clang/lib/Interpreter/IncrementalParser.cpp
@@ -158,8 +158,8 @@
LastPTU.TUPart = C.getTranslationUnitDecl();
// Skip previous eof due to last incremental input.
- if (P->getCurToken().is(tok::eof)) {
- P->ConsumeToken();
+ if (P->getCurToken().is(tok::annot_input_end)) {
+ P->ConsumeAnyToken();
// FIXME: Clang does not call ExitScope on finalizing the regular TU, we
// might want to do that around HandleEndOfTranslationUnit.
P->ExitScope();
@@ -259,12 +259,12 @@
Token Tok;
do {
PP.Lex(Tok);
- } while (Tok.isNot(tok::eof));
+ } while (Tok.isNot(tok::annot_input_end));
}
Token AssertTok;
PP.Lex(AssertTok);
- assert(AssertTok.is(tok::eof) &&
+ assert(AssertTok.is(tok::annot_input_end) &&
"Lexer must be EOF when starting incremental parse!");
if (CodeGenerator *CG = getCodeGen(Act.get())) {
Index: clang/lib/Frontend/PrintPreprocessedOutput.cpp
===================================================================
--- clang/lib/Frontend/PrintPreprocessedOutput.cpp
+++ clang/lib/Frontend/PrintPreprocessedOutput.cpp
@@ -663,7 +663,8 @@
// them.
if (Tok.is(tok::eof) ||
(Tok.isAnnotation() && !Tok.is(tok::annot_header_unit) &&
- !Tok.is(tok::annot_module_begin) && !Tok.is(tok::annot_module_end)))
+ !Tok.is(tok::annot_module_begin) && !Tok.is(tok::annot_module_end) &&
+ !Tok.is(tok::annot_input_end)))
return;
// EmittedDirectiveOnThisLine takes priority over RequireSameLine.
@@ -819,6 +820,9 @@
// -traditional-cpp the lexer keeps /all/ whitespace, including comments.
PP.Lex(Tok);
continue;
+ } else if (Tok.is(tok::annot_input_end)) {
+ PP.Lex(Tok);
+ continue;
} else if (Tok.is(tok::eod)) {
// Don't print end of directive tokens, since they are typically newlines
// that mess up our line tracking. These come from unknown pre-processor
Index: clang/include/clang/Parse/Parser.h
===================================================================
--- clang/include/clang/Parse/Parser.h
+++ clang/include/clang/Parse/Parser.h
@@ -692,7 +692,8 @@
bool isEofOrEom() {
tok::TokenKind Kind = Tok.getKind();
return Kind == tok::eof || Kind == tok::annot_module_begin ||
- Kind == tok::annot_module_end || Kind == tok::annot_module_include;
+ Kind == tok::annot_module_end || Kind == tok::annot_module_include ||
+ Kind == tok::annot_input_end;
}
/// Checks if the \p Level is valid for use in a fold expression.
Index: clang/include/clang/Basic/TokenKinds.def
===================================================================
--- clang/include/clang/Basic/TokenKinds.def
+++ clang/include/clang/Basic/TokenKinds.def
@@ -937,6 +937,9 @@
// into the name of a header unit.
ANNOTATION(header_unit)
+// Annotation for end of input in clang-repl.
+ANNOTATION(input_end)
+
#undef PRAGMA_ANNOTATION
#undef ANNOTATION
#undef TESTING_KEYWORD
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
