https://github.com/Sirraide created https://github.com/llvm/llvm-project/pull/229254
There are quite a lot of places in Clang where we create a new `Token` variable and then call `startToken()`, `setKind()` etc. This patch introduces `Token::Create()` to simplify creating tokens manually, as well as `CreateAnnotation()` and `CreateEof()` because creating annotation tokens and EOF tokens is a fairly common operation. I made these factory functions rather than constructors because there are a few places where we allocate uninitialised tokens. A refactor that moves `startToken()` into a default constructor of `Token` would be possible I think, but that would require some more changes, and I think this is already enough of an improvement. Assisted-by: Codex >From 173c2a538ed41b9f0efc0499bfc05c314933b060 Mon Sep 17 00:00:00 2001 From: Ambrose Leeb <[email protected]> Date: Tue, 6 Oct 2026 00:27:48 +0200 Subject: [PATCH] [Clang] [NFC] Introduce factory functions for `Token`. There are quite a lot of places in Clang where we create a new `Token` variable and then call `startToken()`, `setKind()` etc. This patch introduces `Token::Create()` to simplify creating tokens manually, as well as `CreateAnnotation()` and `CreateEof()` because creating annotation tokens and EOF tokens is a fairly common operation. I made these factory functions rather than constructors because there are a few places where we allocate uninitialised tokens. A refactor that moves `startToken()` into a default constructor of `Token` would be possible I think, but that would require some more changes, and I think this is already enough of an improvement. Assisted-by: Codex --- clang/include/clang/Lex/Token.h | 28 +++ .../include/clang/Lex/VariadicMacroSupport.h | 3 +- clang/lib/Format/FormatTokenSource.h | 3 +- clang/lib/Lex/Lexer.cpp | 5 +- clang/lib/Lex/PPDirectives.cpp | 19 +- clang/lib/Lex/PPMacroExpansion.cpp | 28 +-- clang/lib/Lex/Pragma.cpp | 16 +- clang/lib/Lex/TokenLexer.cpp | 6 +- clang/lib/Parse/ParseCXXInlineMethods.cpp | 26 +-- clang/lib/Parse/ParseDecl.cpp | 11 +- clang/lib/Parse/ParseDeclCXX.cpp | 21 +- clang/lib/Parse/ParseExpr.cpp | 18 +- clang/lib/Parse/ParseExprCXX.cpp | 6 +- clang/lib/Parse/ParseObjc.cpp | 6 +- clang/lib/Parse/ParsePragma.cpp | 220 ++++++------------ clang/lib/Parse/Parser.cpp | 3 +- clang/lib/Rewrite/TokenRewriter.cpp | 8 +- 17 files changed, 145 insertions(+), 282 deletions(-) diff --git a/clang/include/clang/Lex/Token.h b/clang/include/clang/Lex/Token.h index 9904b271c200e..681e2eead75cf 100644 --- a/clang/include/clang/Lex/Token.h +++ b/clang/include/clang/Lex/Token.h @@ -192,6 +192,34 @@ class Token { Loc = SourceLocation().getRawEncoding(); } + static Token Create(tok::TokenKind Kind, SourceLocation Loc, + unsigned Length = 0) { + Token Tok; + Tok.startToken(); + Tok.setKind(Kind); + Tok.setLocation(Loc); + Tok.setLength(Length); + return Tok; + } + + static Token CreateAnnotation(tok::TokenKind Kind, SourceRange Range, + void *Value = nullptr) { + assert(tok::isAnnotation(Kind) && "Expected an annotation token kind"); + Token Tok; + Tok.startToken(); + Tok.setKind(Kind); + Tok.setAnnotationRange(Range); + Tok.setAnnotationValue(Value); + return Tok; + } + + static Token CreateEof(SourceLocation Loc = SourceLocation(), + const void *Data = nullptr) { + Token Tok = Create(tok::eof, Loc); + Tok.setEofData(Data); + return Tok; + } + bool hasPtrData() const { return PtrData != nullptr; } IdentifierInfo *getIdentifierInfo() const { diff --git a/clang/include/clang/Lex/VariadicMacroSupport.h b/clang/include/clang/Lex/VariadicMacroSupport.h index cf86a00c6d663..51eaffec2a838 100644 --- a/clang/include/clang/Lex/VariadicMacroSupport.h +++ b/clang/include/clang/Lex/VariadicMacroSupport.h @@ -157,8 +157,7 @@ namespace clang { : VAOptDefinitionContext(PP), LeadingSpaceForStringifiedToken(false), StringifyBefore(false), CharifyBefore(false), BeginsWithPlaceholder(false), EndsWithPlaceholder(false) { - SyntheticEOFToken.startToken(); - SyntheticEOFToken.setKind(tok::eof); + SyntheticEOFToken = Token::CreateEof(); } void reset() { diff --git a/clang/lib/Format/FormatTokenSource.h b/clang/lib/Format/FormatTokenSource.h index 8f00e5f4582c6..c49c98ee1f8b1 100644 --- a/clang/lib/Format/FormatTokenSource.h +++ b/clang/lib/Format/FormatTokenSource.h @@ -195,8 +195,7 @@ class ScopedMacroState : public FormatTokenSource { : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken), PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource), Token(nullptr), PreviousToken(nullptr) { - FakeEOF.Tok.startToken(); - FakeEOF.Tok.setKind(tok::eof); + FakeEOF.Tok = Token::CreateEof(); TokenSource = this; Line.Level = 0; Line.InPPDirective = true; diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp index 4762b38e67c00..e6a6ffcf328bd 100644 --- a/clang/lib/Lex/Lexer.cpp +++ b/clang/lib/Lex/Lexer.cpp @@ -4663,11 +4663,8 @@ bool Lexer::LexTokenInternal(Token &Result) { const char *Lexer::convertDependencyDirectiveToken( const dependency_directives_scan::Token &DDTok, Token &Result) { const char *TokPtr = BufferStart + DDTok.Offset; - Result.startToken(); - Result.setLocation(getSourceLocation(TokPtr)); - Result.setKind(DDTok.Kind); + Result = Token::Create(DDTok.Kind, getSourceLocation(TokPtr), DDTok.Length); Result.setFlag((Token::TokenFlags)DDTok.Flags); - Result.setLength(DDTok.Length); if (Result.is(tok::raw_identifier)) Result.setRawIdentifierData(TokPtr); else if (Result.isLiteral()) diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp index f1e9aaa72ff04..efcc94a05e525 100644 --- a/clang/lib/Lex/PPDirectives.cpp +++ b/clang/lib/Lex/PPDirectives.cpp @@ -1997,11 +1997,7 @@ void Preprocessor::EnterAnnotationToken(SourceRange Range, // FIXME: Produce this as the current token directly, rather than // allocating a new token for it. auto Tok = std::make_unique<Token[]>(1); - Tok[0].startToken(); - Tok[0].setKind(Kind); - Tok[0].setLocation(Range.getBegin()); - Tok[0].setAnnotationEndLoc(Range.getEnd()); - Tok[0].setAnnotationValue(AnnotationVal); + Tok[0] = Token::CreateAnnotation(Kind, Range, AnnotationVal); EnterTokenStream(std::move(Tok), 1, true, /*IsReinject*/ false); } @@ -4063,10 +4059,7 @@ void Preprocessor::HandleEmbedDirectiveImpl( Data->BinaryData = BinaryContents; Data->FileName = FileName; - Toks[CurIdx].startToken(); - Toks[CurIdx].setKind(tok::annot_embed); - Toks[CurIdx].setAnnotationRange(HashLoc); - Toks[CurIdx++].setAnnotationValue(Data); + Toks[CurIdx++] = Token::CreateAnnotation(tok::annot_embed, HashLoc, Data); // Now add the suffix tokens, if any. if (Params.MaybeSuffixParam) { @@ -4292,12 +4285,8 @@ void Preprocessor::HandleCXXImportDirective(Token ImportTok) { case ImportAction::ModuleBegin: // Let the parser know we're textually entering the module. - DirToks.emplace_back(); - DirToks.back().startToken(); - DirToks.back().setKind(tok::annot_module_begin); - DirToks.back().setLocation(SemiLoc); - DirToks.back().setAnnotationEndLoc(SemiLoc); - DirToks.back().setAnnotationValue(Action.ModuleForHeader); + DirToks.push_back(Token::CreateAnnotation( + tok::annot_module_begin, SemiLoc, Action.ModuleForHeader)); [[fallthrough]]; case ImportAction::ModuleImport: diff --git a/clang/lib/Lex/PPMacroExpansion.cpp b/clang/lib/Lex/PPMacroExpansion.cpp index f31ee308b4cda..085ec2eff8381 100644 --- a/clang/lib/Lex/PPMacroExpansion.cpp +++ b/clang/lib/Lex/PPMacroExpansion.cpp @@ -676,10 +676,8 @@ static bool GenerateNewArgTokens(Preprocessor &PP, // Add left paren if (FoundSeparatorToken) { - TempToken.startToken(); - TempToken.setKind(tok::l_paren); - TempToken.setLocation(ArgStartIterator->getLocation()); - TempToken.setLength(0); + TempToken = + Token::Create(tok::l_paren, ArgStartIterator->getLocation()); NewTokens.push_back(TempToken); } @@ -689,10 +687,7 @@ static bool GenerateNewArgTokens(Preprocessor &PP, // Add right paren and store the paren locations in ParenHints if (FoundSeparatorToken) { SourceLocation Loc = PP.getLocForEndOfToken((I - 1)->getLocation()); - TempToken.startToken(); - TempToken.setKind(tok::r_paren); - TempToken.setLocation(Loc); - TempToken.setLength(0); + TempToken = Token::Create(tok::r_paren, Loc); NewTokens.push_back(TempToken); ParenHints.push_back(SourceRange(ArgStartIterator->getLocation(), Loc)); @@ -856,11 +851,7 @@ MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName, } // Add a marker EOF token to the end of the token list for this argument. - Token EOFTok; - EOFTok.startToken(); - EOFTok.setKind(tok::eof); - EOFTok.setLocation(Tok.getLocation()); - EOFTok.setLength(0); + Token EOFTok = Token::CreateEof(Tok.getLocation()); ArgTokens.push_back(EOFTok); ++NumActuals; if (!ContainsCodeCompletionTok && NumFixedArgsLeft != 0) @@ -917,11 +908,7 @@ MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName, if (ContainsCodeCompletionTok) { // Recover from not-fully-formed macro invocation during code-completion. - Token EOFTok; - EOFTok.startToken(); - EOFTok.setKind(tok::eof); - EOFTok.setLocation(Tok.getLocation()); - EOFTok.setLength(0); + Token EOFTok = Token::CreateEof(Tok.getLocation()); for (; NumActuals < MinArgsExpected; ++NumActuals) ArgTokens.push_back(EOFTok); } @@ -980,10 +967,7 @@ MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName, // Add a marker EOF token to the end of the token list for this argument. SourceLocation EndLoc = Tok.getLocation(); - Tok.startToken(); - Tok.setKind(tok::eof); - Tok.setLocation(EndLoc); - Tok.setLength(0); + Tok = Token::CreateEof(EndLoc); ArgTokens.push_back(Tok); // If we expect two arguments, add both as empty. diff --git a/clang/lib/Lex/Pragma.cpp b/clang/lib/Lex/Pragma.cpp index 9e9f1d21980e3..e955bc065bf5c 100644 --- a/clang/lib/Lex/Pragma.cpp +++ b/clang/lib/Lex/Pragma.cpp @@ -1115,19 +1115,15 @@ struct PragmaDebugHandler : public PragmaHandler { LLVM_BUILTIN_TRAP; } else if (II->isStr("parser_crash")) { if (!PP.getPreprocessorOpts().DisablePragmaDebugCrash) { - Token Crasher; - Crasher.startToken(); - Crasher.setKind(tok::annot_pragma_parser_crash); - Crasher.setAnnotationRange(SourceRange(Tok.getLocation())); + Token Crasher = Token::CreateAnnotation(tok::annot_pragma_parser_crash, + Tok.getLocation()); PP.EnterToken(Crasher, /*IsReinject*/ false); } } else if (II->isStr("sleep")) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); } else if (II->isStr("dump")) { - Token DumpAnnot; - DumpAnnot.startToken(); - DumpAnnot.setKind(tok::annot_pragma_dump); - DumpAnnot.setAnnotationRange(SourceRange(Tok.getLocation())); + Token DumpAnnot = + Token::CreateAnnotation(tok::annot_pragma_dump, Tok.getLocation()); PP.EnterToken(DumpAnnot, /*IsReinject*/false); } else if (II->isStr("diag_mapping")) { Token DiagName; @@ -1287,9 +1283,7 @@ struct PragmaDebugHandler : public PragmaHandler { SourceLocation NameLoc = Tok.getLocation(); MutableArrayRef<Token> Toks( PP.getPreprocessorAllocator().Allocate<Token>(1), 1); - Toks[0].startToken(); - Toks[0].setKind(tok::annot_pragma_captured); - Toks[0].setLocation(NameLoc); + Toks[0] = Token::CreateAnnotation(tok::annot_pragma_captured, NameLoc); PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, /*IsReinject=*/false); diff --git a/clang/lib/Lex/TokenLexer.cpp b/clang/lib/Lex/TokenLexer.cpp index 9b18c31e19eb1..8589946a12db9 100644 --- a/clang/lib/Lex/TokenLexer.cpp +++ b/clang/lib/Lex/TokenLexer.cpp @@ -838,11 +838,9 @@ bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream, // Common paste case: identifier+identifier = identifier. Avoid creating // a lexer and other overhead. PP.IncrementPasteCounter(true); - Result.startToken(); - Result.setKind(tok::raw_identifier); + Result = + Token::Create(tok::raw_identifier, ResultTokLoc, LHSLen + RHSLen); Result.setRawIdentifierData(ResultTokStrPtr); - Result.setLocation(ResultTokLoc); - Result.setLength(LHSLen+RHSLen); } else { PP.IncrementPasteCounter(false); diff --git a/clang/lib/Parse/ParseCXXInlineMethods.cpp b/clang/lib/Parse/ParseCXXInlineMethods.cpp index 35dbec8dfb2f7..c1dfe6c5ef1b0 100644 --- a/clang/lib/Parse/ParseCXXInlineMethods.cpp +++ b/clang/lib/Parse/ParseCXXInlineMethods.cpp @@ -254,11 +254,7 @@ void Parser::ParseCXXNonStaticMemberInitializer(Decl *VarD) { // Store an artificial EOF token to ensure that we don't run off the end of // the initializer when we come to parse it. - Token Eof; - Eof.startToken(); - Eof.setKind(tok::eof); - Eof.setLocation(Tok.getLocation()); - Eof.setEofData(VarD); + Token Eof = Token::CreateEof(Tok.getLocation(), VarD); Toks.push_back(Eof); } @@ -402,11 +398,8 @@ void Parser::ParseLexedMethodDeclaration(LateParsedMethodDeclaration &LM) { // Mark the end of the default argument so that we know when to stop when // we parse it later on. Token LastDefaultArgToken = Toks->back(); - Token DefArgEnd; - DefArgEnd.startToken(); - DefArgEnd.setKind(tok::eof); - DefArgEnd.setLocation(LastDefaultArgToken.getEndLoc()); - DefArgEnd.setEofData(Param); + Token DefArgEnd = + Token::CreateEof(LastDefaultArgToken.getEndLoc(), Param); Toks->push_back(DefArgEnd); // Parse the default argument from its saved token stream. @@ -482,11 +475,8 @@ void Parser::ParseLexedMethodDeclaration(LateParsedMethodDeclaration &LM) { // Add the 'stop' token. Token LastExceptionSpecToken = Toks->back(); - Token ExceptionSpecEnd; - ExceptionSpecEnd.startToken(); - ExceptionSpecEnd.setKind(tok::eof); - ExceptionSpecEnd.setLocation(LastExceptionSpecToken.getEndLoc()); - ExceptionSpecEnd.setEofData(LM.Method); + Token ExceptionSpecEnd = + Token::CreateEof(LastExceptionSpecToken.getEndLoc(), LM.Method); Toks->push_back(ExceptionSpecEnd); // Parse the default argument from its saved token stream. @@ -580,11 +570,7 @@ void Parser::ParseLexedMethodDef(LexedMethod &LM) { assert(!LM.Toks.empty() && "Empty body!"); Token LastBodyToken = LM.Toks.back(); - Token BodyEnd; - BodyEnd.startToken(); - BodyEnd.setKind(tok::eof); - BodyEnd.setLocation(LastBodyToken.getEndLoc()); - BodyEnd.setEofData(LM.D); + Token BodyEnd = Token::CreateEof(LastBodyToken.getEndLoc(), LM.D); LM.Toks.push_back(BodyEnd); // Append the current token at the end of the new token stream so that it // doesn't get lost. diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp index 8af0ee43bcc33..794148e6c98f4 100644 --- a/clang/lib/Parse/ParseDecl.cpp +++ b/clang/lib/Parse/ParseDecl.cpp @@ -4881,11 +4881,7 @@ void Parser::ParseStructDeclaration( ParsedAttributes Parser::ParseLexedAttributeTokens(LateParsedAttribute &LPA) { // Create a fake EOF so that attribute parsing won't go off the end of the // attribute. - Token AttrEnd; - AttrEnd.startToken(); - AttrEnd.setKind(tok::eof); - AttrEnd.setLocation(Tok.getLocation()); - AttrEnd.setEofData(LPA.Toks.data()); + Token AttrEnd = Token::CreateEof(Tok.getLocation(), LPA.Toks.data()); LPA.Toks.push_back(AttrEnd); // Append the current token at the end of the new token stream so that it @@ -8334,10 +8330,7 @@ TypeResult Parser::ParseTypeFromString(StringRef TypeStr, StringRef Context, // Replace the "eod" token with an "eof" token identifying the end of // the provided string. Token &EndToken = Tokens.back(); - EndToken.startToken(); - EndToken.setKind(tok::eof); - EndToken.setLocation(Tok.getLocation()); - EndToken.setEofData(TypeStr.data()); + EndToken = Token::CreateEof(Tok.getLocation(), TypeStr.data()); // Add the current token back. Tokens.push_back(Tok); diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp index 8e0613623b982..ac352c35f2d12 100644 --- a/clang/lib/Parse/ParseDeclCXX.cpp +++ b/clang/lib/Parse/ParseDeclCXX.cpp @@ -3815,9 +3815,7 @@ void Parser::DiagnoseUnexpectedNamespace(NamedDecl *D) { // Push '};' onto the token stream to recover. PP.EnterToken(Tok, /*IsReinject*/ true); - Tok.startToken(); - Tok.setLocation(PP.getLocForEndOfToken(PrevTokLocation)); - Tok.setKind(tok::semi); + Tok = Token::Create(tok::semi, PP.getLocForEndOfToken(PrevTokLocation)); PP.EnterToken(Tok, /*IsReinject*/ true); Tok.setKind(tok::r_brace); @@ -4383,18 +4381,14 @@ void Parser::ParseOpenMPAttributeArgs(const IdentifierInfo *AttrName, // If the attribute is named `directive`, we can consume its argument list // and push the tokens from it into the cached token stream for a new OpenMP // pragma directive. - Token OMPBeginTok; - OMPBeginTok.startToken(); - OMPBeginTok.setKind(tok::annot_attr_openmp); - OMPBeginTok.setLocation(Tok.getLocation()); + Token OMPBeginTok = + Token::CreateAnnotation(tok::annot_attr_openmp, Tok.getLocation()); OpenMPTokens.push_back(OMPBeginTok); ConsumeAndStoreUntil(tok::r_paren, OpenMPTokens, /*StopAtSemi=*/false, /*ConsumeFinalToken*/ false); - Token OMPEndTok; - OMPEndTok.startToken(); - OMPEndTok.setKind(tok::annot_pragma_openmp_end); - OMPEndTok.setLocation(Tok.getLocation()); + Token OMPEndTok = Token::CreateAnnotation(tok::annot_pragma_openmp_end, + Tok.getLocation()); OpenMPTokens.push_back(OMPEndTok); } else { assert(AttrName->isStr("sequence") && @@ -4895,11 +4889,8 @@ void Parser::ParseMicrosoftUuidAttributeArgs(ParsedAttributes &Attrs) { // ActOnStringLiteral() copies the string data into the literal, so it's // ok that the Token points to StrBuffer. Token Toks[1]; - Toks[0].startToken(); - Toks[0].setKind(tok::string_literal); - Toks[0].setLocation(StartLoc); + Toks[0] = Token::Create(tok::string_literal, StartLoc, StrBuffer.size()); Toks[0].setLiteralData(StrBuffer.data()); - Toks[0].setLength(StrBuffer.size()); StringLiteral *UuidString = cast<StringLiteral>(Actions.ActOnUnevaluatedStringLiteral(Toks).get()); ArgExprs.push_back(UuidString); diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp index 10f70dc64d8a2..1e52a2feac2d2 100644 --- a/clang/lib/Parse/ParseExpr.cpp +++ b/clang/lib/Parse/ParseExpr.cpp @@ -341,13 +341,8 @@ Parser::ParseRHSOfBinaryExpression(ExprResult LHS, prec::Level MinPrec) { assert(getLangOpts().Reflection); if (getLangOpts().Blocks) { OpToken.setKind(tok::caret); - Token Caret; - { - Caret.startToken(); - Caret.setKind(tok::caret); - Caret.setLocation(OpToken.getLocation().getLocWithOffset(1)); - Caret.setLength(1); - } + Token Caret = Token::Create( + tok::caret, OpToken.getLocation().getLocWithOffset(1), 1); UnconsumeToken(OpToken); PP.EnterToken(Caret, /*IsReinject=*/true); return ParseRHSOfBinaryExpression(LHS, MinPrec); @@ -3202,15 +3197,10 @@ void Parser::injectEmbedTokens() { Data->BinaryData.size() * 2 - 1); unsigned I = 0; for (auto &Byte : Data->BinaryData) { - Toks[I].startToken(); - Toks[I].setKind(tok::binary_data); - Toks[I].setLocation(Tok.getLocation()); - Toks[I].setLength(1); + Toks[I] = Token::Create(tok::binary_data, Tok.getLocation(), 1); Toks[I].setLiteralData(&Byte); if (I != ((Data->BinaryData.size() - 1) * 2)) { - Toks[I + 1].startToken(); - Toks[I + 1].setKind(tok::comma); - Toks[I + 1].setLocation(Tok.getLocation()); + Toks[I + 1] = Token::Create(tok::comma, Tok.getLocation()); } I += 2; } diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp index bfdad0d501282..7563b5d52620e 100644 --- a/clang/lib/Parse/ParseExprCXX.cpp +++ b/clang/lib/Parse/ParseExprCXX.cpp @@ -3711,11 +3711,7 @@ Parser::ParseCXXAmbiguousParenExpression(ParenParseOption &ExprType, } // Create a fake EOF to mark end of Toks buffer. - Token AttrEnd; - AttrEnd.startToken(); - AttrEnd.setKind(tok::eof); - AttrEnd.setLocation(Tok.getLocation()); - AttrEnd.setEofData(Toks.data()); + Token AttrEnd = Token::CreateEof(Tok.getLocation(), Toks.data()); Toks.push_back(AttrEnd); // The current token should go after the cached tokens. diff --git a/clang/lib/Parse/ParseObjc.cpp b/clang/lib/Parse/ParseObjc.cpp index 7ca08cb2112e6..bdd78c983843d 100644 --- a/clang/lib/Parse/ParseObjc.cpp +++ b/clang/lib/Parse/ParseObjc.cpp @@ -3281,11 +3281,7 @@ void Parser::ParseLexedObjCMethodDefs(LexedMethod &LM, bool parseMethod) { assert(!LM.Toks.empty() && "ParseLexedObjCMethodDef - Empty body!"); // Store an artificial EOF token to ensure that we don't run off the end of // the method's body when we come to parse it. - Token Eof; - Eof.startToken(); - Eof.setKind(tok::eof); - Eof.setEofData(MCDecl); - Eof.setLocation(OrigLoc); + Token Eof = Token::CreateEof(OrigLoc, MCDecl); LM.Toks.push_back(Eof); // Append the current token at the end of the new token stream so that it // doesn't get lost. diff --git a/clang/lib/Parse/ParsePragma.cpp b/clang/lib/Parse/ParsePragma.cpp index f58d2c70b7d98..ae4662f5a5ddc 100644 --- a/clang/lib/Parse/ParsePragma.cpp +++ b/clang/lib/Parse/ParsePragma.cpp @@ -123,12 +123,9 @@ struct PragmaSTDC_FENV_ACCESSHandler : public PragmaHandler { MutableArrayRef<Token> Toks(PP.getPreprocessorAllocator().Allocate<Token>(1), 1); - Toks[0].startToken(); - Toks[0].setKind(tok::annot_pragma_fenv_access); - Toks[0].setLocation(Tok.getLocation()); - Toks[0].setAnnotationEndLoc(Tok.getLocation()); - Toks[0].setAnnotationValue(reinterpret_cast<void*>( - static_cast<uintptr_t>(OOS))); + Toks[0] = Token::CreateAnnotation( + tok::annot_pragma_fenv_access, Tok.getLocation(), + reinterpret_cast<void *>(static_cast<uintptr_t>(OOS))); PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, /*IsReinject=*/false); } @@ -147,11 +144,8 @@ struct PragmaSTDC_CX_LIMITED_RANGEHandler : public PragmaHandler { MutableArrayRef<Token> Toks( PP.getPreprocessorAllocator().Allocate<Token>(1), 1); - Toks[0].startToken(); - Toks[0].setKind(tok::annot_pragma_cx_limited_range); - Toks[0].setLocation(Tok.getLocation()); - Toks[0].setAnnotationEndLoc(Tok.getLocation()); - Toks[0].setAnnotationValue( + Toks[0] = Token::CreateAnnotation( + tok::annot_pragma_cx_limited_range, Tok.getLocation(), reinterpret_cast<void *>(static_cast<uintptr_t>(OOS))); PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, /*IsReinject=*/false); @@ -353,12 +347,10 @@ struct PragmaMSFenvAccessHandler : public PragmaHandler { MutableArrayRef<Token> Toks( PP.getPreprocessorAllocator().Allocate<Token>(1), 1); - Toks[0].startToken(); - Toks[0].setKind(tok::annot_pragma_fenv_access_ms); - Toks[0].setLocation(FirstToken.getLocation()); - Toks[0].setAnnotationEndLoc(Tok.getLocation()); - Toks[0].setAnnotationValue( - reinterpret_cast<void*>(static_cast<uintptr_t>(OOS))); + Toks[0] = Token::CreateAnnotation( + tok::annot_pragma_fenv_access_ms, + SourceRange(FirstToken.getLocation(), Tok.getLocation()), + reinterpret_cast<void *>(static_cast<uintptr_t>(OOS))); PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, /*IsReinject=*/false); } @@ -1260,11 +1252,9 @@ bool Parser::HandlePragmaMSInitSeg(StringRef PragmaName, if (!Section.empty()) { // Pretend the user wrote the appropriate string literal here. Token Toks[1]; - Toks[0].startToken(); - Toks[0].setKind(tok::string_literal); - Toks[0].setLocation(Tok.getLocation()); + Toks[0] = + Token::Create(tok::string_literal, Tok.getLocation(), Section.size()); Toks[0].setLiteralData(Section.data()); - Toks[0].setLength(Section.size()); SegmentName = cast<StringLiteral>(Actions.ActOnStringLiteral(Toks, nullptr).get()); PP.Lex(Tok); @@ -2190,11 +2180,8 @@ void PragmaGCCVisibilityHandler::HandlePragma(Preprocessor &PP, } auto Toks = std::make_unique<Token[]>(1); - Toks[0].startToken(); - Toks[0].setKind(tok::annot_pragma_vis); - Toks[0].setLocation(VisLoc); - Toks[0].setAnnotationEndLoc(EndLoc); - Toks[0].setAnnotationValue( + Toks[0] = Token::CreateAnnotation( + tok::annot_pragma_vis, SourceRange(VisLoc, EndLoc), const_cast<void *>(static_cast<const void *>(VisType))); PP.EnterTokenStream(std::move(Toks), 1, /*DisableMacroExpansion=*/true, /*IsReinject=*/false); @@ -2334,11 +2321,9 @@ void PragmaPackHandler::HandlePragma(Preprocessor &PP, MutableArrayRef<Token> Toks(PP.getPreprocessorAllocator().Allocate<Token>(1), 1); - Toks[0].startToken(); - Toks[0].setKind(tok::annot_pragma_pack); - Toks[0].setLocation(PackLoc); - Toks[0].setAnnotationEndLoc(RParenLoc); - Toks[0].setAnnotationValue(static_cast<void*>(Info)); + Toks[0] = Token::CreateAnnotation(tok::annot_pragma_pack, + SourceRange(PackLoc, RParenLoc), + static_cast<void *>(Info)); PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, /*IsReinject=*/false); } @@ -2377,12 +2362,10 @@ void PragmaMSStructHandler::HandlePragma(Preprocessor &PP, MutableArrayRef<Token> Toks(PP.getPreprocessorAllocator().Allocate<Token>(1), 1); - Toks[0].startToken(); - Toks[0].setKind(tok::annot_pragma_msstruct); - Toks[0].setLocation(MSStructTok.getLocation()); - Toks[0].setAnnotationEndLoc(EndLoc); - Toks[0].setAnnotationValue(reinterpret_cast<void*>( - static_cast<uintptr_t>(Kind))); + Toks[0] = Token::CreateAnnotation( + tok::annot_pragma_msstruct, + SourceRange(MSStructTok.getLocation(), EndLoc), + reinterpret_cast<void *>(static_cast<uintptr_t>(Kind))); PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, /*IsReinject=*/false); } @@ -2511,12 +2494,9 @@ static void ParseAlignPragma(Preprocessor &PP, Token &FirstTok, MutableArrayRef<Token> Toks(PP.getPreprocessorAllocator().Allocate<Token>(1), 1); - Toks[0].startToken(); - Toks[0].setKind(tok::annot_pragma_align); - Toks[0].setLocation(FirstTok.getLocation()); - Toks[0].setAnnotationEndLoc(EndLoc); - Toks[0].setAnnotationValue(reinterpret_cast<void*>( - static_cast<uintptr_t>(Kind))); + Toks[0] = Token::CreateAnnotation( + tok::annot_pragma_align, SourceRange(FirstTok.getLocation(), EndLoc), + reinterpret_cast<void *>(static_cast<uintptr_t>(Kind))); PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, /*IsReinject=*/false); } @@ -2605,9 +2585,8 @@ void PragmaUnusedHandler::HandlePragma(Preprocessor &PP, 2 * Identifiers.size()); for (unsigned i=0; i != Identifiers.size(); i++) { Token &pragmaUnusedTok = Toks[2*i], &idTok = Toks[2*i+1]; - pragmaUnusedTok.startToken(); - pragmaUnusedTok.setKind(tok::annot_pragma_unused); - pragmaUnusedTok.setLocation(UnusedLoc); + pragmaUnusedTok = + Token::CreateAnnotation(tok::annot_pragma_unused, UnusedLoc); idTok = Identifiers[i]; } PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, @@ -2654,10 +2633,9 @@ void PragmaWeakHandler::HandlePragma(Preprocessor &PP, MutableArrayRef<Token> Toks( PP.getPreprocessorAllocator().Allocate<Token>(3), 3); Token &pragmaUnusedTok = Toks[0]; - pragmaUnusedTok.startToken(); - pragmaUnusedTok.setKind(tok::annot_pragma_weakalias); - pragmaUnusedTok.setLocation(WeakLoc); - pragmaUnusedTok.setAnnotationEndLoc(AliasName.getLocation()); + pragmaUnusedTok = + Token::CreateAnnotation(tok::annot_pragma_weakalias, + SourceRange(WeakLoc, AliasName.getLocation())); Toks[1] = WeakName; Toks[2] = AliasName; PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, @@ -2666,10 +2644,7 @@ void PragmaWeakHandler::HandlePragma(Preprocessor &PP, MutableArrayRef<Token> Toks( PP.getPreprocessorAllocator().Allocate<Token>(2), 2); Token &pragmaUnusedTok = Toks[0]; - pragmaUnusedTok.startToken(); - pragmaUnusedTok.setKind(tok::annot_pragma_weak); - pragmaUnusedTok.setLocation(WeakLoc); - pragmaUnusedTok.setAnnotationEndLoc(WeakLoc); + pragmaUnusedTok = Token::CreateAnnotation(tok::annot_pragma_weak, WeakLoc); Toks[1] = WeakName; PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, /*IsReinject=*/false); @@ -2711,10 +2686,9 @@ void PragmaRedefineExtnameHandler::HandlePragma(Preprocessor &PP, MutableArrayRef<Token> Toks(PP.getPreprocessorAllocator().Allocate<Token>(3), 3); Token &pragmaRedefTok = Toks[0]; - pragmaRedefTok.startToken(); - pragmaRedefTok.setKind(tok::annot_pragma_redefine_extname); - pragmaRedefTok.setLocation(RedefLoc); - pragmaRedefTok.setAnnotationEndLoc(AliasName.getLocation()); + pragmaRedefTok = + Token::CreateAnnotation(tok::annot_pragma_redefine_extname, + SourceRange(RedefLoc, AliasName.getLocation())); Toks[1] = RedefName; Toks[2] = AliasName; PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, @@ -2730,12 +2704,9 @@ void PragmaFPContractHandler::HandlePragma(Preprocessor &PP, MutableArrayRef<Token> Toks(PP.getPreprocessorAllocator().Allocate<Token>(1), 1); - Toks[0].startToken(); - Toks[0].setKind(tok::annot_pragma_fp_contract); - Toks[0].setLocation(Tok.getLocation()); - Toks[0].setAnnotationEndLoc(Tok.getLocation()); - Toks[0].setAnnotationValue(reinterpret_cast<void*>( - static_cast<uintptr_t>(OOS))); + Toks[0] = Token::CreateAnnotation( + tok::annot_pragma_fp_contract, Tok.getLocation(), + reinterpret_cast<void *>(static_cast<uintptr_t>(OOS))); PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, /*IsReinject=*/false); } @@ -2793,11 +2764,9 @@ void PragmaOpenCLExtensionHandler::HandlePragma(Preprocessor &PP, Info->second = State; MutableArrayRef<Token> Toks(PP.getPreprocessorAllocator().Allocate<Token>(1), 1); - Toks[0].startToken(); - Toks[0].setKind(tok::annot_pragma_opencl_extension); - Toks[0].setLocation(NameLoc); - Toks[0].setAnnotationValue(static_cast<void*>(Info)); - Toks[0].setAnnotationEndLoc(StateLoc); + Toks[0] = Token::CreateAnnotation(tok::annot_pragma_opencl_extension, + SourceRange(NameLoc, StateLoc), + static_cast<void *>(Info)); PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, /*IsReinject=*/false); @@ -2826,10 +2795,7 @@ template <tok::TokenKind StartTok, tok::TokenKind EndTok, void PragmaSupportHandler<StartTok, EndTok, UnexpectedDiag>::HandlePragma( Preprocessor &PP, PragmaIntroducer Introducer, Token &FirstTok) { SmallVector<Token, 16> Pragma; - Token Tok; - Tok.startToken(); - Tok.setKind(StartTok); - Tok.setLocation(Introducer.Loc); + Token Tok = Token::CreateAnnotation(StartTok, Introducer.Loc); while (Tok.isNot(tok::eod) && Tok.isNot(tok::eof)) { Pragma.push_back(Tok); @@ -2848,9 +2814,7 @@ void PragmaSupportHandler<StartTok, EndTok, UnexpectedDiag>::HandlePragma( } } SourceLocation EodLoc = Tok.getLocation(); - Tok.startToken(); - Tok.setKind(EndTok); - Tok.setLocation(EodLoc); + Tok = Token::CreateAnnotation(EndTok, EodLoc); Pragma.push_back(Tok); auto Toks = std::make_unique<Token[]>(Pragma.size()); @@ -2947,12 +2911,9 @@ void PragmaMSPointersToMembers::HandlePragma(Preprocessor &PP, return; } - Token AnnotTok; - AnnotTok.startToken(); - AnnotTok.setKind(tok::annot_pragma_ms_pointers_to_members); - AnnotTok.setLocation(PointersToMembersLoc); - AnnotTok.setAnnotationEndLoc(EndLoc); - AnnotTok.setAnnotationValue( + Token AnnotTok = Token::CreateAnnotation( + tok::annot_pragma_ms_pointers_to_members, + SourceRange(PointersToMembersLoc, EndLoc), reinterpret_cast<void *>(static_cast<uintptr_t>(RepresentationMethod))); PP.EnterToken(AnnotTok, /*IsReinject=*/true); } @@ -3039,13 +3000,10 @@ void PragmaMSVtorDisp::HandlePragma(Preprocessor &PP, } // Enter the annotation. - Token AnnotTok; - AnnotTok.startToken(); - AnnotTok.setKind(tok::annot_pragma_ms_vtordisp); - AnnotTok.setLocation(VtorDispLoc); - AnnotTok.setAnnotationEndLoc(EndLoc); - AnnotTok.setAnnotationValue(reinterpret_cast<void *>( - static_cast<uintptr_t>((Action << 16) | (Value & 0xFFFF)))); + Token AnnotTok = Token::CreateAnnotation( + tok::annot_pragma_ms_vtordisp, SourceRange(VtorDispLoc, EndLoc), + reinterpret_cast<void *>( + static_cast<uintptr_t>((Action << 16) | (Value & 0xFFFF)))); PP.EnterToken(AnnotTok, /*IsReinject=*/false); } @@ -3053,13 +3011,9 @@ void PragmaMSVtorDisp::HandlePragma(Preprocessor &PP, /// an annotation token. void PragmaMSPragma::HandlePragma(Preprocessor &PP, PragmaIntroducer Introducer, Token &Tok) { - Token EoF, AnnotTok; - EoF.startToken(); - EoF.setKind(tok::eof); - AnnotTok.startToken(); - AnnotTok.setKind(tok::annot_pragma_ms_pragma); - AnnotTok.setLocation(Tok.getLocation()); - AnnotTok.setAnnotationEndLoc(Tok.getLocation()); + Token EoF = Token::CreateEof(); + Token AnnotTok = + Token::CreateAnnotation(tok::annot_pragma_ms_pragma, Tok.getLocation()); SmallVector<Token, 8> TokenVector; // Suck up all of the tokens before the eod. for (; Tok.isNot(tok::eod); PP.Lex(Tok)) { @@ -3197,14 +3151,12 @@ void PragmaFloatControlHandler::HandlePragma(Preprocessor &PP, // Enter the annotation. auto TokenArray = std::make_unique<Token[]>(1); - TokenArray[0].startToken(); - TokenArray[0].setKind(tok::annot_pragma_float_control); - TokenArray[0].setLocation(FloatControlLoc); - TokenArray[0].setAnnotationEndLoc(EndLoc); // Create an encoding of Action and Value by shifting the Action into // the high 16 bits then union with the Kind. - TokenArray[0].setAnnotationValue(reinterpret_cast<void *>( - static_cast<uintptr_t>((Action << 16) | (Kind & 0xFFFF)))); + TokenArray[0] = Token::CreateAnnotation( + tok::annot_pragma_float_control, SourceRange(FloatControlLoc, EndLoc), + reinterpret_cast<void *>( + static_cast<uintptr_t>((Action << 16) | (Kind & 0xFFFF)))); PP.EnterTokenStream(std::move(TokenArray), 1, /*DisableMacroExpansion=*/false, /*IsReinject=*/false); } @@ -3542,12 +3494,9 @@ void PragmaFPHandler::HandlePragma(Preprocessor &PP, return; } - Token FPTok; - FPTok.startToken(); - FPTok.setKind(tok::annot_pragma_fp); - FPTok.setLocation(PragmaName.getLocation()); - FPTok.setAnnotationEndLoc(PragmaName.getLocation()); - FPTok.setAnnotationValue(reinterpret_cast<void *>(AnnotValue)); + Token FPTok = + Token::CreateAnnotation(tok::annot_pragma_fp, PragmaName.getLocation(), + reinterpret_cast<void *>(AnnotValue)); TokenList.push_back(FPTok); auto TokenArray = std::make_unique<Token[]>(TokenList.size()); @@ -3601,11 +3550,8 @@ void PragmaSTDC_FENV_ROUNDHandler::HandlePragma(Preprocessor &PP, MutableArrayRef<Token> Toks(PP.getPreprocessorAllocator().Allocate<Token>(1), 1); - Toks[0].startToken(); - Toks[0].setKind(tok::annot_pragma_fenv_round); - Toks[0].setLocation(Tok.getLocation()); - Toks[0].setAnnotationEndLoc(Tok.getLocation()); - Toks[0].setAnnotationValue( + Toks[0] = Token::CreateAnnotation( + tok::annot_pragma_fenv_round, Tok.getLocation(), reinterpret_cast<void *>(static_cast<uintptr_t>(RM))); PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, /*IsReinject=*/false); @@ -3667,10 +3613,7 @@ static bool ParseLoopHintValue(Preprocessor &PP, Token &Tok, Token PragmaName, PP.Lex(Tok); } - Token EOFTok; - EOFTok.startToken(); - EOFTok.setKind(tok::eof); - EOFTok.setLocation(Tok.getLocation()); + Token EOFTok = Token::CreateEof(Tok.getLocation()); ValueList.push_back(EOFTok); // Terminates expression for parsing. markAsReinjectedForRelexing(ValueList); @@ -3781,12 +3724,10 @@ void PragmaLoopHintHandler::HandlePragma(Preprocessor &PP, return; // Generate the loop hint token. - Token LoopHintTok; - LoopHintTok.startToken(); - LoopHintTok.setKind(tok::annot_pragma_loop_hint); - LoopHintTok.setLocation(Introducer.Loc); - LoopHintTok.setAnnotationEndLoc(PragmaName.getLocation()); - LoopHintTok.setAnnotationValue(static_cast<void *>(Info)); + Token LoopHintTok = Token::CreateAnnotation( + tok::annot_pragma_loop_hint, + SourceRange(Introducer.Loc, PragmaName.getLocation()), + static_cast<void *>(Info)); TokenList.push_back(LoopHintTok); } @@ -3869,11 +3810,10 @@ void PragmaUnrollHintHandler::HandlePragma(Preprocessor &PP, // Generate the hint token. auto TokenArray = std::make_unique<Token[]>(1); - TokenArray[0].startToken(); - TokenArray[0].setKind(tok::annot_pragma_loop_hint); - TokenArray[0].setLocation(Introducer.Loc); - TokenArray[0].setAnnotationEndLoc(PragmaName.getLocation()); - TokenArray[0].setAnnotationValue(static_cast<void *>(Info)); + TokenArray[0] = Token::CreateAnnotation( + tok::annot_pragma_loop_hint, + SourceRange(Introducer.Loc, PragmaName.getLocation()), + static_cast<void *>(Info)); PP.EnterTokenStream(std::move(TokenArray), 1, /*DisableMacroExpansion=*/false, /*IsReinject=*/false); } @@ -4166,10 +4106,7 @@ void PragmaAttributeHandler::HandlePragma(Preprocessor &PP, PP.Lex(Tok); // Terminate the attribute for parsing. - Token EOFTok; - EOFTok.startToken(); - EOFTok.setKind(tok::eof); - EOFTok.setLocation(EndLoc); + Token EOFTok = Token::CreateEof(EndLoc); AttributeTokens.push_back(EOFTok); markAsReinjectedForRelexing(AttributeTokens); @@ -4183,11 +4120,9 @@ void PragmaAttributeHandler::HandlePragma(Preprocessor &PP, // Generate the annotated pragma token. auto TokenArray = std::make_unique<Token[]>(1); - TokenArray[0].startToken(); - TokenArray[0].setKind(tok::annot_pragma_attribute); - TokenArray[0].setLocation(FirstToken.getLocation()); - TokenArray[0].setAnnotationEndLoc(FirstToken.getLocation()); - TokenArray[0].setAnnotationValue(static_cast<void *>(Info)); + TokenArray[0] = Token::CreateAnnotation(tok::annot_pragma_attribute, + FirstToken.getLocation(), + static_cast<void *>(Info)); PP.EnterTokenStream(std::move(TokenArray), 1, /*DisableMacroExpansion=*/false, /*IsReinject=*/false); } @@ -4255,11 +4190,7 @@ void PragmaMaxTokensTotalHandler::HandlePragma(Preprocessor &PP, static void zOSPragmaHandlerHelper(Preprocessor &PP, Token &Tok, tok::TokenKind TokKind) { - Token AnnotTok; - AnnotTok.startToken(); - AnnotTok.setKind(TokKind); - AnnotTok.setLocation(Tok.getLocation()); - AnnotTok.setAnnotationEndLoc(Tok.getLocation()); + Token AnnotTok = Token::CreateAnnotation(TokKind, Tok.getLocation()); SmallVector<Token, 8> TokenVector; // Suck up all of the tokens before the eod. for (; Tok.isNot(tok::eod); PP.Lex(Tok)) { @@ -4267,10 +4198,7 @@ static void zOSPragmaHandlerHelper(Preprocessor &PP, Token &Tok, AnnotTok.setAnnotationEndLoc(Tok.getLocation()); } // Add a sentinel EoF token to the end of the list. - Token EoF; - EoF.startToken(); - EoF.setKind(tok::eof); - EoF.setLocation(Tok.getLocation()); + Token EoF = Token::CreateEof(Tok.getLocation()); TokenVector.push_back(EoF); // We must allocate this array with new because EnterTokenStream is going to // delete it later. diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp index e35666b5d6420..4cc40fc0e1f7a 100644 --- a/clang/lib/Parse/Parser.cpp +++ b/clang/lib/Parse/Parser.cpp @@ -77,8 +77,7 @@ Parser::Parser(Preprocessor &pp, Sema &actions, bool skipFunctionBodies) ParsingGenericAssociationType(false), InMessageExpression(false), ParsingInObjCContainer(false), TemplateParameterDepth(0) { SkipFunctionBodies = pp.isCodeCompletionEnabled() || skipFunctionBodies; - Tok.startToken(); - Tok.setKind(tok::eof); + Tok = Token::CreateEof(); Actions.CurScope = nullptr; NumCachedScopes = 0; CurParsedObjCImpl = nullptr; diff --git a/clang/lib/Rewrite/TokenRewriter.cpp b/clang/lib/Rewrite/TokenRewriter.cpp index b1f4bd251580a..9830b414f74c6 100644 --- a/clang/lib/Rewrite/TokenRewriter.cpp +++ b/clang/lib/Rewrite/TokenRewriter.cpp @@ -85,15 +85,11 @@ TokenRewriter::AddTokenBefore(token_iterator I, const char *Val) { // Plop the string into the scratch buffer, then create a token for this // string. - Token Tok; - Tok.startToken(); const char *Spelling; - Tok.setLocation(ScratchBuf->getToken(Val, Len, Spelling)); - Tok.setLength(Len); - // TODO: Form a whole lexer around this and relex the token! For now, just // set kind to tok::unknown. - Tok.setKind(tok::unknown); + Token Tok = Token::Create(tok::unknown, + ScratchBuf->getToken(Val, Len, Spelling), Len); return AddToken(Tok, RemapIterator(I)); } _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
