https://github.com/ChenMiaoi created https://github.com/llvm/llvm-project/pull/236408
When returning whitespace tokens, the lexer does not propagate leading-space information to the following token. Preserve this information in whitespace retention mode, accounting for whether the whitespace ends with a newline. Avoid inserting an extra space in traditional preprocessor output when the previous token already supplies whitespace. >From 954eaf22352755ea5105ab466bb1268e35d9135f Mon Sep 17 00:00:00 2001 From: Chen Miao <[email protected]> Date: Wed, 28 Oct 2026 20:59:25 +0800 Subject: [PATCH] [clang][Lex] Preserve leading-space flags when retaining whitespace When returning whitespace tokens, the lexer does not propagate leading-space information to the following token. Preserve this information in whitespace retention mode, accounting for whether the whitespace ends with a newline. Avoid inserting an extra space in traditional preprocessor output when the previous token already supplies whitespace. --- .../lib/Frontend/PrintPreprocessedOutput.cpp | 15 +++++- clang/lib/Lex/Lexer.cpp | 4 +- clang/test/Preprocessor/traditional-cpp.c | 8 ++++ clang/unittests/Lex/LexerTest.cpp | 46 +++++++++++++++++++ 4 files changed, 69 insertions(+), 4 deletions(-) diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp index ab5ac8a4a69c6d..00c122ed7a56c9 100644 --- a/clang/lib/Frontend/PrintPreprocessedOutput.cpp +++ b/clang/lib/Frontend/PrintPreprocessedOutput.cpp @@ -792,6 +792,17 @@ void PrintPPOutputPPCallbacks::HandleWhitespaceBeforeTok(const Token &Tok, !Tok.is(tok::annot_embed) && !Tok.is(tok::annot_module_name))) return; + bool HasLeadingSpace = Tok.hasLeadingSpace(); + if (HasLeadingSpace && PP.getLangOpts().TraditionalCPP && + PrevTok.is(tok::unknown) && PrevTok.getLength()) { + // Traditional preprocessing emits whitespace as tokens. Do not add another + // space if the previous token already supplied it. + SmallString<128> SpellingBuffer; + StringRef Spelling = PP.getSpelling(PrevTok, SpellingBuffer); + if (!Spelling.empty() && isWhitespace(Spelling.back())) + HasLeadingSpace = false; + } + // EmittedDirectiveOnThisLine takes priority over RequireSameLine. if ((!RequireSameLine || EmittedDirectiveOnThisLine) && MoveToLine(Tok, /*RequireStartOfLine=*/EmittedDirectiveOnThisLine)) { @@ -808,7 +819,7 @@ void PrintPPOutputPPCallbacks::HandleWhitespaceBeforeTok(const Token &Tok, // expect leading white space, if a macro expansion in column 1 starts // with an empty macro argument, or an empty nested macro expansion. In // this case, move the token to column 2. - if (ColNo == 1 && Tok.hasLeadingSpace()) + if (ColNo == 1 && HasLeadingSpace) ColNo = 2; // This hack prevents stuff like: @@ -831,7 +842,7 @@ void PrintPPOutputPPCallbacks::HandleWhitespaceBeforeTok(const Token &Tok, // whitespace-minimization mode // - The whitespace is necessary to keep the tokens apart and there is not // already a newline between them - if (RequireSpace || (!MinimizeWhitespace && Tok.hasLeadingSpace()) || + if (RequireSpace || (!MinimizeWhitespace && HasLeadingSpace) || ((EmittedTokensOnThisLine || EmittedDirectiveOnThisLine) && AvoidConcat(PrevPrevTok, PrevTok, Tok))) *OS << ' '; diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp index b659780bac593a..8a3676d94503f7 100644 --- a/clang/lib/Lex/Lexer.cpp +++ b/clang/lib/Lex/Lexer.cpp @@ -2663,7 +2663,7 @@ bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr) { IsAtStartOfLine = true; IsAtPhysicalStartOfLine = true; } - // FIXME: The next token will not have LeadingSpace set. + HasLeadingSpace = !isVerticalWhitespace(CurPtr[-1]); return true; } @@ -3875,7 +3875,7 @@ bool Lexer::LexTokenInternal(Token &Result) { // whitespace. if (isKeepWhitespaceMode()) { FormTokenWithChars(Result, CurPtr, tok::unknown); - // FIXME: The next token will not have LeadingSpace set. + HasLeadingSpace = true; return true; } diff --git a/clang/test/Preprocessor/traditional-cpp.c b/clang/test/Preprocessor/traditional-cpp.c index f311be0bb59360..23fddb1cdc4147 100644 --- a/clang/test/Preprocessor/traditional-cpp.c +++ b/clang/test/Preprocessor/traditional-cpp.c @@ -29,6 +29,14 @@ foo // bar indented! tab separated values +/* CHECK: {{^}}a b{{$}} + * CHECK: {{^}}a b{{$}} + * CHECK: {{^}} indented_with_spaces{{$}} + */ +a b +a b + indented_with_spaces + #define bracket(x) >>>x<<< bracket(| spaces |) /* CHECK: {{^}}>>>| spaces |<<<{{$}} diff --git a/clang/unittests/Lex/LexerTest.cpp b/clang/unittests/Lex/LexerTest.cpp index 8f632ccc5b9315..8ca1abbe826c90 100644 --- a/clang/unittests/Lex/LexerTest.cpp +++ b/clang/unittests/Lex/LexerTest.cpp @@ -719,6 +719,52 @@ TEST_F(LexerTest, CreatedFIDCountForPredefinedBuffer) { 1U); } +TEST_F(LexerTest, LeadingSpaceWithWhitespaceRetention) { + const struct { + const char *Whitespace; + bool LeadingSpace; + bool StartOfLine; + } Cases[] = { + {"", false, false}, {" ", true, false}, + {"\t", true, false}, {" \t\f\v", true, false}, + {"\n", false, true}, {"\r", false, true}, + {"\r\n", false, true}, {"\n ", true, true}, + {"\r\n\t", true, true}, {" \n", false, true}, + {" \n \n", false, true}, {" \n \n\t", true, true}, + }; + + for (const auto &Case : Cases) { + for (bool KeepWhitespace : {false, true}) { + SCOPED_TRACE(Case.Whitespace); + SCOPED_TRACE(KeepWhitespace); + auto Buffer = llvm::MemoryBuffer::getMemBufferCopy( + std::string("a+") + Case.Whitespace + "b+c"); + Lexer L(SourceLocation(), LangOpts, Buffer->getBufferStart(), + Buffer->getBufferStart(), Buffer->getBufferEnd()); + L.SetKeepWhitespaceMode(KeepWhitespace); + + for (unsigned I = 0; I != 5; ++I) { + Token T; + do { + L.LexFromRawLexer(T); + if (T.is(tok::unknown)) { + ASSERT_TRUE(KeepWhitespace); + ASSERT_GT(T.getLength(), 0u); + } + } while (T.is(tok::unknown)); + + ASSERT_EQ(T.getKind(), + I == 1 || I == 3 ? tok::plus : tok::raw_identifier); + EXPECT_EQ(T.hasLeadingSpace(), I == 2 && Case.LeadingSpace); + EXPECT_EQ(T.isAtStartOfLine(), I == 0 || (I == 2 && Case.StartOfLine)); + } + Token T; + L.LexFromRawLexer(T); + EXPECT_TRUE(T.is(tok::eof)); + } + } +} + TEST_F(LexerTest, RawAndNormalLexSameForLineComments) { const llvm::StringLiteral Source = R"cpp( // First line comment. _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
