Author: owenca Date: 2026-06-19T19:45:25-07:00 New Revision: 092858485f00e6786da8a88c0c4d462247e64571
URL: https://github.com/llvm/llvm-project/commit/092858485f00e6786da8a88c0c4d462247e64571 DIFF: https://github.com/llvm/llvm-project/commit/092858485f00e6786da8a88c0c4d462247e64571.diff LOG: [clang-format][NFC] Clean up FormatTokenLexer (#203825) Added: Modified: clang/lib/Format/FormatTokenLexer.cpp clang/lib/Format/FormatTokenLexer.h Removed: ################################################################################ diff --git a/clang/lib/Format/FormatTokenLexer.cpp b/clang/lib/Format/FormatTokenLexer.cpp index 92571c012bdb2..0cf01875af833 100644 --- a/clang/lib/Format/FormatTokenLexer.cpp +++ b/clang/lib/Format/FormatTokenLexer.cpp @@ -33,8 +33,7 @@ FormatTokenLexer::FormatTokenLexer( LangOpts(getFormattingLangOpts(Style)), SourceMgr(SourceMgr), ID(ID), Style(Style), IdentTable(IdentTable), Keywords(IdentTable), Encoding(Encoding), Allocator(Allocator), FirstInLineIndex(0), - FormattingDisabled(false), FormatOffRegex(Style.OneLineFormatOffRegex), - MacroBlockBeginRegex(Style.MacroBlockBegin), + FormattingDisabled(false), MacroBlockBeginRegex(Style.MacroBlockBegin), MacroBlockEndRegex(Style.MacroBlockEnd), VerilogProtectedBlock(false) { Lex = std::make_unique<Lexer>(ID, SourceMgr.getBufferOrFake(ID), SourceMgr, LangOpts); @@ -88,12 +87,14 @@ FormatTokenLexer::FormatTokenLexer( ArrayRef<FormatToken *> FormatTokenLexer::lex() { assert(Tokens.empty()); assert(FirstInLineIndex == 0); + enum { FO_None, FO_CurrentLine, FO_NextLine } FormatOff = FO_None; + llvm::Regex FormatOffRegex(Style.OneLineFormatOffRegex); do { Tokens.push_back(getNextToken()); + auto &Tok = *Tokens.back(); - const auto NewlinesBefore = Tok.NewlinesBefore; - switch (FormatOff) { + switch (const auto NewlinesBefore = Tok.NewlinesBefore; FormatOff) { case FO_NextLine: if (NewlinesBefore > 1) { FormatOff = FO_None; @@ -125,13 +126,16 @@ ArrayRef<FormatToken *> FormatTokenLexer::lex() { } } } + if (Style.isJavaScript()) { tryParseJSRegexLiteral(); handleTemplateStrings(); } else if (Style.isTextProto()) { tryParsePythonComment(); } + tryMergePreviousTokens(); + if (Style.isCSharp()) { // This needs to come after tokens have been merged so that C# // string literals are correctly identified. @@ -140,9 +144,11 @@ ArrayRef<FormatToken *> FormatTokenLexer::lex() { handleTableGenMultilineString(); handleTableGenNumericLikeIdentifier(); } + if (Tokens.back()->NewlinesBefore > 0 || Tokens.back()->IsMultiline) FirstInLineIndex = Tokens.size() - 1; } while (Tokens.back()->isNot(tok::eof)); + if (Style.InsertNewlineAtEOF) { auto &TokEOF = *Tokens.back(); if (TokEOF.NewlinesBefore == 0) { @@ -150,6 +156,7 @@ ArrayRef<FormatToken *> FormatTokenLexer::lex() { TokEOF.OriginalColumn = 0; } } + return Tokens; } diff --git a/clang/lib/Format/FormatTokenLexer.h b/clang/lib/Format/FormatTokenLexer.h index 9f5b735efe1d0..3f8c6ba15173d 100644 --- a/clang/lib/Format/FormatTokenLexer.h +++ b/clang/lib/Format/FormatTokenLexer.h @@ -135,7 +135,6 @@ class FormatTokenLexer { TemplateNames, TypeNames, VariableTemplates; bool FormattingDisabled; - llvm::Regex FormatOffRegex; // For one line. llvm::Regex MacroBlockBeginRegex; llvm::Regex MacroBlockEndRegex; _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
