https://github.com/yronglin updated https://github.com/llvm/llvm-project/pull/187858
>From 6369995b6ac58ac8cd72eeace7e96d5f30f1c3bd Mon Sep 17 00:00:00 2001 From: yronglin <[email protected]> Date: Sun, 22 Mar 2026 12:12:12 +0800 Subject: [PATCH] [clangd] Handle C++20 annot_module_name token Signed-off-by: yronglin <[email protected]> --- .../clangd/test/non-existent.test | 82 +++++++++++++++++++ clang/lib/Tooling/Syntax/Tokens.cpp | 23 +++++- clang/unittests/Tooling/Syntax/TokensTest.cpp | 36 +++++++- 3 files changed, 135 insertions(+), 6 deletions(-) create mode 100644 clang-tools-extra/clangd/test/non-existent.test diff --git a/clang-tools-extra/clangd/test/non-existent.test b/clang-tools-extra/clangd/test/non-existent.test new file mode 100644 index 0000000000000..dd46ce25bc884 --- /dev/null +++ b/clang-tools-extra/clangd/test/non-existent.test @@ -0,0 +1,82 @@ +# +## reproduce a crash in module processing - seems having a non-existent module imported +## as the last statement in a file causes clangd to crash +## modified from modules.test +# +# Windows have different escaping modes. +# FIXME: We should add one for windows. +# UNSUPPORTED: system-windows +# +# RUN: rm -fr %t +# RUN: mkdir -p %t +# RUN: split-file %s %t +# +# RUN: sed -e "s|DIR|%/t|g" %t/compile_commands.json.tmpl > %t/compile_commands.json.tmp +# RUN: sed -e "s|CLANG_CC|%clang|g" %t/compile_commands.json.tmp > %t/compile_commands.json +# RUN: sed -e "s|DIR|%/t|g" %t/definition.jsonrpc.tmpl > %t/definition.jsonrpc +# +# RUN: clangd -experimental-modules-support -lit-test < %t/definition.jsonrpc + +#--- A.cppm +module; +export module A; + +#--- Use.cpp +module; +export module Use; +import A; +import NonExistent; + +#--- compile_commands.json.tmpl +[ + { + "directory": "DIR", + "command": "CLANG_CC -fprebuilt-module-path=DIR -std=c++20 -o DIR/main.cpp.o DIR/Use.cpp -fmodule-file=A=DIR/A.pcm", + "file": "DIR/Use.cpp" + "output": "DIR/main.cpp.o" + }, + { + "directory": "DIR", + "command": "CLANG_CC -fprebuilt-module-path=DIR --std=c++20 DIR/A.cppm --precompile -o DIR/A.pcm", + "file": "DIR/A.cppm" + "output": "DIR/A.pcm" + } +] + +#--- definition.jsonrpc.tmpl +{ + "jsonrpc": "2.0", + "id": 0, + "method": "initialize", + "params": { + "processId": 123, + "rootPath": "clangd", + "capabilities": { + "textDocument": { + "completion": { + "completionItem": { + "snippetSupport": true + } + } + } + }, + "trace": "off" + } +} +--- +{ + "jsonrpc": "2.0", + "method": "textDocument/didOpen", + "params": { + "textDocument": { + "uri": "file://DIR/Use.cpp", + "languageId": "cpp", + "version": 1, + "text": "module;\nexport module Use;\nimport A;\nimport NonExistent;\n" + } + } +} +--- +{"jsonrpc":"2.0","id":2,"method":"shutdown"} +--- +{"jsonrpc":"2.0","method":"exit"} diff --git a/clang/lib/Tooling/Syntax/Tokens.cpp b/clang/lib/Tooling/Syntax/Tokens.cpp index 260654a0701fd..d27b92e8de875 100644 --- a/clang/lib/Tooling/Syntax/Tokens.cpp +++ b/clang/lib/Tooling/Syntax/Tokens.cpp @@ -681,8 +681,18 @@ class TokenCollector::CollectPPExpansions : public PPCallbacks { TokenCollector::TokenCollector(Preprocessor &PP) : PP(PP) { // Collect the expanded token stream during preprocessing. PP.setTokenWatcher([this](const clang::Token &T) { - if (T.isAnnotation()) + if (T.is(tok::annot_module_name)) { + auto &SM = this->PP.getSourceManager(); + StringRef Text = Lexer::getSourceText( + CharSourceRange::getTokenRange(T.getAnnotationRange()), SM, + this->PP.getLangOpts()); + Expanded.push_back( + syntax::Token(T.getLocation(), Text.size(), tok::annot_module_name)); + } else if (T.isAnnotation()) { return; + } else { + Expanded.push_back(syntax::Token(T)); + } DEBUG_WITH_TYPE("collect-tokens", llvm::dbgs() << "Token: " << syntax::Token(T).dumpForTests( @@ -690,7 +700,6 @@ TokenCollector::TokenCollector(Preprocessor &PP) : PP(PP) { << "\n" ); - Expanded.push_back(syntax::Token(T)); }); // And locations of macro calls, to properly recover boundaries of those in // case of empty expansions. @@ -893,6 +902,16 @@ class TokenCollector::Builder { TokenBuffer TokenCollector::consume() && { PP.setTokenWatcher(nullptr); Collector->disable(); + + /// If the parser hit an module load fatal error, the TokenCollector will not + /// receive an EOF token; we need to add an EOF token to the end of the token + /// sequence. + if (PP.hadModuleLoaderFatalFailure() && + (Expanded.empty() || Expanded.back().kind() != tok::eof)) { + auto &SM = PP.getSourceManager(); + Expanded.push_back( + syntax::Token(SM.getLocForEndOfFile(SM.getMainFileID()), 0, tok::eof)); + } return Builder(std::move(Expanded), std::move(Expansions), PP.getSourceManager(), PP.getLangOpts()) .build(); diff --git a/clang/unittests/Tooling/Syntax/TokensTest.cpp b/clang/unittests/Tooling/Syntax/TokensTest.cpp index 468ca5ddd2c75..03122bf86d250 100644 --- a/clang/unittests/Tooling/Syntax/TokensTest.cpp +++ b/clang/unittests/Tooling/Syntax/TokensTest.cpp @@ -123,8 +123,8 @@ class TokenCollectorTest : public ::testing::Test { // Prepare to run a compiler. if (!Diags->getClient()) Diags->setClient(new IgnoringDiagConsumer); - std::vector<const char *> Args = {"tok-test", "-std=c++03", "-fsyntax-only", - FileName}; + std::vector<const char *> Args = {"tok-test", LangStandard.c_str(), + "-fsyntax-only", FileName}; CreateInvocationOptions CIOpts; CIOpts.Diags = Diags; CIOpts.VFS = FS; @@ -141,8 +141,11 @@ class TokenCollectorTest : public ::testing::Test { this->Buffer = TokenBuffer(*SourceMgr); RecordTokens Recorder(this->Buffer); - ASSERT_TRUE(Compiler.ExecuteAction(Recorder)) - << "failed to run the frontend"; + if (AllowErrors) + Compiler.ExecuteAction(Recorder); + else + ASSERT_TRUE(Compiler.ExecuteAction(Recorder)) + << "failed to run the frontend"; } /// Record the tokens and return a test dump of the resulting buffer. @@ -250,6 +253,8 @@ class TokenCollectorTest : public ::testing::Test { } // Data fields. + std::string LangStandard = "-std=c++03"; + bool AllowErrors = false; DiagnosticOptions DiagOpts; llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags = llvm::makeIntrusiveRefCnt<DiagnosticsEngine>(DiagnosticIDs::create(), @@ -1148,4 +1153,27 @@ TEST_F(TokenCollectorTest, Pragmas) { } )cpp"); } + +TEST_F(TokenCollectorTest, CXX20ModuleImportModule) { + LangStandard = "-std=c++20"; + AllowErrors = true; + + recordTokens("import Non.Existent;\n"); + EXPECT_THAT(Buffer.expandedTokens(), + ElementsAre(Kind(tok::kw_import), Kind(tok::annot_module_name), + Kind(tok::semi), Kind(tok::eof))); +} + +TEST_F(TokenCollectorTest, CXX20ModuleImportPartition) { + LangStandard = "-std=c++20"; + AllowErrors = true; + + recordTokens("export module M.N;\nimport :Non.Existent;\n"); + EXPECT_THAT(Buffer.expandedTokens(), + ElementsAre(Kind(tok::kw_export), Kind(tok::kw_module), + Kind(tok::annot_module_name), Kind(tok::semi), + Kind(tok::kw_import), Kind(tok::colon), + Kind(tok::annot_module_name), Kind(tok::semi), + Kind(tok::eof))); +} } // namespace _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
