Author: Macro Terra Date: 2026-08-14T23:44:10+08:00 New Revision: 17c348dc5f44e22ce9f94e5301ad4a131d4b0e32
URL: https://github.com/llvm/llvm-project/commit/17c348dc5f44e22ce9f94e5301ad4a131d4b0e32 DIFF: https://github.com/llvm/llvm-project/commit/17c348dc5f44e22ce9f94e5301ad4a131d4b0e32.diff LOG: [clang][Tooling] Ignore end-of-directive tokens in TokenCollector (#212242) fixes #197652 ### Summary `TokenCollector` receives `tok::eod` from the preprocessor for an end-of-directive marker, but that token has no one-to-one raw spelling. Recording it as an expanded token can therefore make the expanded-to-spelled mapping fail in clangd. Skip `tok::eod` alongside non-module annotation tokens. Keep the existing `annot_module_name` reconstruction path, as it intentionally supplies a spelling. Add a regression test for: ```cpp } #pragma clang __debug dump ``` assisted by codex. Added: Modified: clang/lib/Tooling/Syntax/Tokens.cpp clang/unittests/Tooling/Syntax/TokensTest.cpp Removed: ################################################################################ diff --git a/clang/lib/Tooling/Syntax/Tokens.cpp b/clang/lib/Tooling/Syntax/Tokens.cpp index 927333fda18bb..6b75c17cab67e 100644 --- a/clang/lib/Tooling/Syntax/Tokens.cpp +++ b/clang/lib/Tooling/Syntax/Tokens.cpp @@ -688,11 +688,14 @@ TokenCollector::TokenCollector(Preprocessor &PP) : PP(PP) { this->PP.getLangOpts()); Expanded.push_back( syntax::Token(T.getLocation(), Text.size(), tok::annot_module_name)); - } else if (T.isAnnotation()) { return; - } else { - Expanded.push_back(syntax::Token(T)); } + + // These tokens do not have a one-to-one raw spelling. + if (T.isAnnotation() || T.is(tok::eod)) + return; + + Expanded.push_back(syntax::Token(T)); DEBUG_WITH_TYPE("collect-tokens", llvm::dbgs() << "Token: " << syntax::Token(T).dumpForTests( diff --git a/clang/unittests/Tooling/Syntax/TokensTest.cpp b/clang/unittests/Tooling/Syntax/TokensTest.cpp index 6418cc8f87d9d..ebc640b8d0923 100644 --- a/clang/unittests/Tooling/Syntax/TokensTest.cpp +++ b/clang/unittests/Tooling/Syntax/TokensTest.cpp @@ -1169,6 +1169,15 @@ TEST_F(TokenCollectorTest, Pragmas) { )cpp"); } +TEST_F(TokenCollectorTest, DebugPragmaAtEndOfFile) { + AllowErrors = true; + recordTokens("}\n#pragma clang __debug dump\n"); + + // The end-of-directive token has no spelling and must not be collected. + EXPECT_THAT(Buffer.expandedTokens(), + ElementsAre(Kind(tok::r_brace), Kind(tok::eof))); +} + TEST_F(TokenBufferTest, EofTokenOnBracketDepthLimit) { // Force parser to bail out due to exceeding the bracket depth limit. recordTokens("((;", {"-fbracket-depth=1"}); _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
