https://github.com/yronglin created https://github.com/llvm/llvm-project/pull/198544
None >From 2ea14d4d3b54b1dd4141524a6cb0711bd87aae68 Mon Sep 17 00:00:00 2001 From: yronglin <[email protected]> Date: Tue, 19 May 2026 07:18:52 -0700 Subject: [PATCH] [clang] Handle C++20 annot_module_name in syntax tokens Signed-off-by: yronglin <[email protected]> --- clang/lib/Tooling/Syntax/Tokens.cpp | 13 +++++-- clang/unittests/Tooling/Syntax/TokensTest.cpp | 34 +++++++++++++++++-- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/clang/lib/Tooling/Syntax/Tokens.cpp b/clang/lib/Tooling/Syntax/Tokens.cpp index 9ad8a149675d9..927333fda18bb 100644 --- a/clang/lib/Tooling/Syntax/Tokens.cpp +++ b/clang/lib/Tooling/Syntax/Tokens.cpp @@ -681,8 +681,18 @@ class TokenCollector::CollectPPExpansions : public PPCallbacks { TokenCollector::TokenCollector(Preprocessor &PP) : PP(PP) { // Collect the expanded token stream during preprocessing. PP.setTokenWatcher([this](const clang::Token &T) { - if (T.isAnnotation()) + if (T.is(tok::annot_module_name)) { + auto &SM = this->PP.getSourceManager(); + StringRef Text = Lexer::getSourceText( + CharSourceRange::getTokenRange(T.getAnnotationRange()), SM, + this->PP.getLangOpts()); + Expanded.push_back( + syntax::Token(T.getLocation(), Text.size(), tok::annot_module_name)); + } else if (T.isAnnotation()) { return; + } else { + Expanded.push_back(syntax::Token(T)); + } DEBUG_WITH_TYPE("collect-tokens", llvm::dbgs() << "Token: " << syntax::Token(T).dumpForTests( @@ -690,7 +700,6 @@ TokenCollector::TokenCollector(Preprocessor &PP) : PP(PP) { << "\n" ); - Expanded.push_back(syntax::Token(T)); }); // And locations of macro calls, to properly recover boundaries of those in // case of empty expansions. diff --git a/clang/unittests/Tooling/Syntax/TokensTest.cpp b/clang/unittests/Tooling/Syntax/TokensTest.cpp index 8af9308828c28..ae5001a2f5645 100644 --- a/clang/unittests/Tooling/Syntax/TokensTest.cpp +++ b/clang/unittests/Tooling/Syntax/TokensTest.cpp @@ -124,7 +124,7 @@ class TokenCollectorTest : public ::testing::Test { // Prepare to run a compiler. if (!Diags->getClient()) Diags->setClient(new IgnoringDiagConsumer); - std::vector<const char *> Args = {"tok-test", "-std=c++03", + std::vector<const char *> Args = {"tok-test", LangStandard.c_str(), "-fsyntax-only"}; Args.insert(Args.end(), ExtraArgs.begin(), ExtraArgs.end()); Args.push_back(FileName); @@ -144,8 +144,11 @@ class TokenCollectorTest : public ::testing::Test { this->Buffer = TokenBuffer(*SourceMgr); RecordTokens Recorder(this->Buffer); - ASSERT_TRUE(Compiler.ExecuteAction(Recorder)) - << "failed to run the frontend"; + if (AllowErrors) + Compiler.ExecuteAction(Recorder); + else + ASSERT_TRUE(Compiler.ExecuteAction(Recorder)) + << "failed to run the frontend"; } /// Record the tokens and return a test dump of the resulting buffer. @@ -253,6 +256,8 @@ class TokenCollectorTest : public ::testing::Test { } // Data fields. + std::string LangStandard = "-std=c++03"; + bool AllowErrors = false; DiagnosticOptions DiagOpts; llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags = llvm::makeIntrusiveRefCnt<DiagnosticsEngine>(DiagnosticIDs::create(), @@ -1161,4 +1166,27 @@ TEST_F(TokenBufferTest, EofTokenOnBracketDepthLimit) { EXPECT_EQ(Buffer.expandedTokens().back().kind(), tok::eof); EXPECT_EQ(Buffer.expandedTokens().drop_back().back().kind(), tok::l_paren); } + +TEST_F(TokenCollectorTest, CXX20ModuleImportModule) { + LangStandard = "-std=c++20"; + AllowErrors = true; + + recordTokens("import Non.Existent;\n"); + EXPECT_THAT(Buffer.expandedTokens(), + ElementsAre(Kind(tok::kw_import), Kind(tok::annot_module_name), + Kind(tok::semi), Kind(tok::eof))); +} + +TEST_F(TokenCollectorTest, CXX20ModuleImportPartition) { + LangStandard = "-std=c++20"; + AllowErrors = true; + + recordTokens("export module M.N;\nimport :Non.Existent;\n"); + EXPECT_THAT(Buffer.expandedTokens(), + ElementsAre(Kind(tok::kw_export), Kind(tok::kw_module), + Kind(tok::annot_module_name), Kind(tok::semi), + Kind(tok::kw_import), Kind(tok::colon), + Kind(tok::annot_module_name), Kind(tok::semi), + Kind(tok::eof))); +} } // namespace _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
