https://github.com/yronglin updated https://github.com/llvm/llvm-project/pull/225451
>From faf185a05774c273b9b8cd3b4a00f796156f7579 Mon Sep 17 00:00:00 2001 From: yronglin <[email protected]> Date: Tue, 6 Oct 2026 23:22:46 -0700 Subject: [PATCH] [clang] Fix uses header-name lexing rules when #include is followed by an empty macro expansion Signed-off-by: yronglin <[email protected]> --- clang/docs/ReleaseNotes.md | 2 ++ clang/include/clang/Lex/Lexer.h | 7 +++++ clang/include/clang/Lex/Preprocessor.h | 6 +++++ clang/include/clang/Lex/PreprocessorLexer.h | 2 +- clang/lib/Lex/Lexer.cpp | 26 ++++++++++++++++--- clang/lib/Lex/Preprocessor.cpp | 9 +++++-- clang/lib/Lex/PreprocessorLexer.cpp | 18 ------------- clang/test/CXX/drs/cwg3015.cpp | 14 ++++++++++ clang/test/CXX/drs/cwg30xx.cpp | 2 ++ .../ClangScanDeps/depscan-lex-has-include.c | 11 ++++++++ clang/test/Preprocessor/_Pragma-dependency.c | 8 ++++++ clang/unittests/Lex/LexerTest.cpp | 21 +++++++++++++++ .../Lex/PPDependencyDirectivesTest.cpp | 14 +++++++++- clang/www/cxx_dr_status.html | 2 +- 14 files changed, 116 insertions(+), 26 deletions(-) create mode 100644 clang/test/CXX/drs/cwg3015.cpp diff --git a/clang/docs/ReleaseNotes.md b/clang/docs/ReleaseNotes.md index 08442f74879afc..5578bc4c6db41b 100644 --- a/clang/docs/ReleaseNotes.md +++ b/clang/docs/ReleaseNotes.md @@ -751,6 +751,8 @@ features cannot lower the translation-unit ABI level; by a using-declaration (e.g. `using Base::operator auto;`). Such a conversion function is now also diagnosed if it is deleted. (#GH189146) +- Fix incorrect uses header-name lexing rules when ``#include`` is followed by an empty macro expansion. (#GH223781) + - A workaround that was introduced to fix an issue with the `<format>` header present in some versions of libstdc++15 has been extended to support preprocessed input. Previously, splitting the preprocessing and compilation step would result in the fix not being applied. (#GH160314) diff --git a/clang/include/clang/Lex/Lexer.h b/clang/include/clang/Lex/Lexer.h index b3d8ccdd2d0e30..5037aec7640f0d 100644 --- a/clang/include/clang/Lex/Lexer.h +++ b/clang/include/clang/Lex/Lexer.h @@ -202,6 +202,9 @@ class Lexer : public PreprocessorLexer { /// return the tok::eof token. This implicitly involves the preprocessor. bool Lex(Token &Result); + /// Lex a token, producing a header-name token if possible. + bool LexIncludeFilename(Token &FilenameTok) override; + private: /// Called when the preprocessor is in 'dependency scanning lexing mode'. bool LexDependencyDirectiveToken(Token &Result); @@ -639,6 +642,10 @@ class Lexer : public PreprocessorLexer { /// bool LexTokenInternal(Token &Result); + /// Start a new token, applying the pending start-of-line and leading-space + /// flags. + void InitToken(Token &Result); + bool CheckUnicodeWhitespace(Token &Result, uint32_t C, const char *CurPtr); bool LexUnicodeIdentifierStart(Token &Result, uint32_t C, const char *CurPtr); diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h index 31b68a0fd0670e..809fcef77ee9a7 100644 --- a/clang/include/clang/Lex/Preprocessor.h +++ b/clang/include/clang/Lex/Preprocessor.h @@ -3217,6 +3217,12 @@ class Preprocessor { static bool CLK_DependencyDirectivesLexer(Preprocessor &P, Token &Result) { return P.CurLexer->LexDependencyDirectiveToken(Result); } + static bool CLK_HeaderNameLexer(Preprocessor &P, Token &Result) { + // Header-name rules only apply to the first attempt. If it doesn't produce + // a token (e.g. an empty macro was expanded), retry with the normal lexer. + P.recomputeCurLexerKind(); + return P.CurLexer->LexIncludeFilename(Result); + } }; /// Abstract base class that describes a handler that will receive diff --git a/clang/include/clang/Lex/PreprocessorLexer.h b/clang/include/clang/Lex/PreprocessorLexer.h index d71fe708ab20a2..f7746331b0adc1 100644 --- a/clang/include/clang/Lex/PreprocessorLexer.h +++ b/clang/include/clang/Lex/PreprocessorLexer.h @@ -130,7 +130,7 @@ class PreprocessorLexer { // Misc. lexing methods. /// Lex a token, producing a header-name token if possible. - void LexIncludeFilename(Token &FilenameTok); + virtual bool LexIncludeFilename(Token &FilenameTok) = 0; /// Inform the lexer whether or not we are currently lexing a /// preprocessor directive. diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp index 0abc6a56c44383..c8f9fb8754a250 100644 --- a/clang/lib/Lex/Lexer.cpp +++ b/clang/lib/Lex/Lexer.cpp @@ -3811,9 +3811,7 @@ void Lexer::PropagateLineStartLeadingSpaceInfo(Token &Result) { // Note that this doesn't affect IsAtPhysicalStartOfLine. } -bool Lexer::Lex(Token &Result) { - assert(!isDependencyDirectivesLexer()); - +void Lexer::InitToken(Token &Result) { // Start a new token. Result.startToken(); @@ -3837,6 +3835,12 @@ bool Lexer::Lex(Token &Result) { Result.setFlag(Token::LeadingEmptyMacro); HasLeadingEmptyMacro = false; } +} + +bool Lexer::Lex(Token &Result) { + assert(!isDependencyDirectivesLexer()); + + InitToken(Result); bool isRawLex = isLexingRawMode(); (void) isRawLex; @@ -4660,6 +4664,22 @@ bool Lexer::LexTokenInternal(Token &Result) { goto LexStart; } +/// Lex a token, producing a header-name token if the next token starts with +/// '<' or '"'. Otherwise, lex a normal token. +bool Lexer::LexIncludeFilename(Token &FilenameTok) { + bool returnedToken = false; + ParsingFilename = true; + if (isDependencyDirectivesLexer()) + returnedToken = LexDependencyDirectiveToken(FilenameTok); + else { + InitToken(FilenameTok); + returnedToken = LexTokenInternal(FilenameTok); + } + + ParsingFilename = false; + return returnedToken; +} + const char *Lexer::convertDependencyDirectiveToken( const dependency_directives_scan::Token &DDTok, Token &Result) { const char *TokPtr = BufferStart + DDTok.Offset; diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp index 84907adc5d744a..0298f30d85e2ef 100644 --- a/clang/lib/Lex/Preprocessor.cpp +++ b/clang/lib/Lex/Preprocessor.cpp @@ -1125,8 +1125,13 @@ bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) { // __has_include(__has_include)) if (CurPPLexer->ParsingFilename) LexUnexpandedToken(FilenameTok); - else - CurPPLexer->LexIncludeFilename(FilenameTok); + else { + CurLexerCallback = CLK_HeaderNameLexer; + if (AllowMacroExpansion) + Lex(FilenameTok); + else + LexUnexpandedToken(FilenameTok); + } } else { Lex(FilenameTok); } diff --git a/clang/lib/Lex/PreprocessorLexer.cpp b/clang/lib/Lex/PreprocessorLexer.cpp index 3f483c56f8b126..dc4408d0e8df55 100644 --- a/clang/lib/Lex/PreprocessorLexer.cpp +++ b/clang/lib/Lex/PreprocessorLexer.cpp @@ -26,24 +26,6 @@ PreprocessorLexer::PreprocessorLexer(Preprocessor *pp, FileID fid) InitialNumSLocEntries = pp->getSourceManager().local_sloc_entry_size(); } -/// After the preprocessor has parsed a \#include, lex and -/// (potentially) macro expand the filename. -void PreprocessorLexer::LexIncludeFilename(Token &FilenameTok) { - assert(ParsingFilename == false && "reentered LexIncludeFilename"); - - // We are now parsing a filename! - ParsingFilename = true; - - // Lex the filename. - if (LexingRawMode) - IndirectLex(FilenameTok); - else - PP->Lex(FilenameTok); - - // We should have obtained the filename now. - ParsingFilename = false; -} - /// getFileEntry - Return the FileEntry corresponding to this FileID. Like /// getFileID(), this only works for lexers with attached preprocessors. OptionalFileEntryRef PreprocessorLexer::getFileEntry() const { diff --git a/clang/test/CXX/drs/cwg3015.cpp b/clang/test/CXX/drs/cwg3015.cpp new file mode 100644 index 00000000000000..54cdd42e123689 --- /dev/null +++ b/clang/test/CXX/drs/cwg3015.cpp @@ -0,0 +1,14 @@ +// RUN: %clang_cc1 -std=c++98 -fexceptions -fcxx-exceptions -pedantic-errors -verify-directives -verify %s +// RUN: %clang_cc1 -std=c++11 -fexceptions -fcxx-exceptions -pedantic-errors -verify-directives -verify %s +// RUN: %clang_cc1 -std=c++14 -fexceptions -fcxx-exceptions -pedantic-errors -verify-directives -verify %s +// RUN: %clang_cc1 -std=c++17 -fexceptions -fcxx-exceptions -pedantic-errors -verify-directives -verify %s +// RUN: %clang_cc1 -std=c++20 -fexceptions -fcxx-exceptions -pedantic-errors -verify-directives -verify %s +// RUN: %clang_cc1 -std=c++23 -fexceptions -fcxx-exceptions -pedantic-errors -verify-directives -verify %s +// RUN: %clang_cc1 -std=c++2c -fexceptions -fcxx-exceptions -pedantic-errors -verify-directives -verify %s + +// cwg3015: partial + +// An intervening empty macro prevents header-name tokenization. +#define EMPTY +#include EMPTY "name with \"quotes\".h" +// expected-error@-1 {{'name with \"quotes\".h' file not found}} diff --git a/clang/test/CXX/drs/cwg30xx.cpp b/clang/test/CXX/drs/cwg30xx.cpp index a664fe6b3f073e..1fbc96089d4ce0 100644 --- a/clang/test/CXX/drs/cwg30xx.cpp +++ b/clang/test/CXX/drs/cwg30xx.cpp @@ -22,6 +22,8 @@ void f( } // namespace cwg3005 +// cwg3015 is in cwg3015.cpp + namespace cwg3035 { // cwg3035: no #if __cplusplus >= 201103L static union { diff --git a/clang/test/ClangScanDeps/depscan-lex-has-include.c b/clang/test/ClangScanDeps/depscan-lex-has-include.c index 4d46e8d7d19e26..bd1c1392b5cf19 100644 --- a/clang/test/ClangScanDeps/depscan-lex-has-include.c +++ b/clang/test/ClangScanDeps/depscan-lex-has-include.c @@ -38,6 +38,17 @@ #include <something/something.h> #endif +#undef something +#define EMPTY +#define HEADER something/something.h + +// An empty macro prevents header-name tokenization, allowing HEADER to expand. +#if __has_include(EMPTY <HEADER>) +#include EMPTY <HEADER> +#else +#error header not found after empty macro +#endif + //--- something/something.h //--- error.c diff --git a/clang/test/Preprocessor/_Pragma-dependency.c b/clang/test/Preprocessor/_Pragma-dependency.c index 0b785ac87daaf4..7423c9e1714e98 100644 --- a/clang/test/Preprocessor/_Pragma-dependency.c +++ b/clang/test/Preprocessor/_Pragma-dependency.c @@ -6,6 +6,14 @@ // expected-error@+1 {{expected "FILENAME" or <FILENAME>}} #pragma GCC dependency self +#define EMPTY +// FIXME: The diagnostic location is not correct, it's current in +// #pragma GCC dependency EMPTY "./_Pragma-dependency.c" +// ^ expected "FILENAME" or <FILENAME> +// The real location should be the start of EMPTY. +// expected-error@+1 {{expected "FILENAME" or <FILENAME>}} +#pragma GCC dependency EMPTY "./_Pragma-dependency.c" + #define DO_PRAGMA _Pragma #define STR "GCC dependency \"parse.y\"") // expected-error@+1 {{'parse.y' file not found}} diff --git a/clang/unittests/Lex/LexerTest.cpp b/clang/unittests/Lex/LexerTest.cpp index 8f632ccc5b9315..adc4420ef4fdf8 100644 --- a/clang/unittests/Lex/LexerTest.cpp +++ b/clang/unittests/Lex/LexerTest.cpp @@ -719,6 +719,27 @@ TEST_F(LexerTest, CreatedFIDCountForPredefinedBuffer) { 1U); } +TEST_F(LexerTest, LexIncludeFilenameResetsToken) { + for (const char *Source : + {"\"foo\\\n.h\" \"bar.h\"", "<foo\\\n.h> <bar.h>"}) { + auto Buffer = llvm::MemoryBuffer::getMemBuffer(Source); + FileID FID = SourceMgr.createFileID(std::move(Buffer)); + Lexer L(FID, SourceMgr.getBufferOrFake(FID), SourceMgr, LangOpts); + + Token Tok; + Tok.startToken(); + ASSERT_TRUE(L.LexIncludeFilename(Tok)); + ASSERT_TRUE(Tok.is(tok::header_name)); + EXPECT_TRUE(Tok.needsCleaning()); + + ASSERT_TRUE(L.LexIncludeFilename(Tok)); + ASSERT_TRUE(Tok.is(tok::header_name)); + EXPECT_FALSE(Tok.needsCleaning()); + EXPECT_EQ(Source[0] == '"' ? "\"bar.h\"" : "<bar.h>", + Lexer::getSpelling(Tok, SourceMgr, LangOpts)); + } +} + TEST_F(LexerTest, RawAndNormalLexSameForLineComments) { const llvm::StringLiteral Source = R"cpp( // First line comment. diff --git a/clang/unittests/Lex/PPDependencyDirectivesTest.cpp b/clang/unittests/Lex/PPDependencyDirectivesTest.cpp index 7b11fc163f6eb8..0e3b26ff8711c6 100644 --- a/clang/unittests/Lex/PPDependencyDirectivesTest.cpp +++ b/clang/unittests/Lex/PPDependencyDirectivesTest.cpp @@ -191,6 +191,16 @@ TEST_F(PPDependencyDirectivesTest, HeaderNamesStartingWithCompoundTokens) { "#if __has_include(<<>)\n" "#include <<>\n" "#endif\n" + "#define HEADER \"=\"\n" + "#define HAS_INCLUDE(x) __has_include(x)\n" + "#if HAS_INCLUDE(HEADER)\n" + "#include HEADER\n" + "#endif\n" + "#define EMPTY\n" + "#define NAME =\n" + "#if __has_include(EMPTY <NAME>)\n" + "#include EMPTY <NAME>\n" + "#endif\n" "#include \\\n<=>"; SmallVector<dependency_directives_scan::Token> Tokens; @@ -246,12 +256,14 @@ TEST_F(PPDependencyDirectivesTest, HeaderNamesStartingWithCompoundTokens) { PP.EnterMainSourceFile(); PP.LexTokensUntilEOF(); + EXPECT_FALSE(Diags.hasErrorOccurred()); SmallVector<std::string> IncludedFilesSlash; for (StringRef IncludedFile : IncludedFiles) IncludedFilesSlash.push_back( llvm::sys::path::convert_to_slash(IncludedFile)); SmallVector<std::string> ExpectedIncludes{ - "/source/main.c", "/source/=", "/source/<", "/source/="}; + "/source/main.c", "/source/=", "/source/<", + "/source/=", "/source/=", "/source/="}; EXPECT_EQ(IncludedFilesSlash, ExpectedIncludes); } diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html index e7679da30d5c29..9c42ac51bb6f62 100755 --- a/clang/www/cxx_dr_status.html +++ b/clang/www/cxx_dr_status.html @@ -20922,7 +20922,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2> <td>[<a href="https://wg21.link/cpp.include";>cpp.include</a>]</td> <td>CD7</td> <td>Handling of <I>header-name</I>s for <TT>#include</TT> and <TT>#embed</TT></td> - <td class="unknown" align="center">Unknown</td> + <td class="partial" align="center">Clang 24</td> </tr> <tr id="3016"> <td><a href="https://cplusplus.github.io/CWG/issues/3016.html";>3016</a></td> _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
