From: Owen Avery <[email protected]>
ProcMacroInvocLexer and MacroInvocLexer were only separate because one
stored instances of const_TokenPtr directly, while the other stored
instances of AST::Token (a thin wrapper around const_TokenPtr). This
patch removes ProcMacroInvocLexer and makes MacroInvocLexer directly
store const_TokenPtr instances.
gcc/rust/ChangeLog:
* Make-lang.in (GRS_OBJS): Remove entries.
* ast/rust-ast.cc (DelimTokenTree::parse_to_meta_item): Handle
changes to AttributeParser.
(AttributeParser::AttributeParser): Change signature.
* ast/rust-macro.h (AttributeParser::AttributeParser): Likewise.
* expand/rust-macro-expand.cc
(MacroExpander::expand_eager_invocations): Handle changes to
MacroInvocLexer.
(MacroExpander::parse_proc_macro_output): Use MacroInvocLexer
instead of ProcMacroInvocLexer.
* expand/rust-macro-expand.h: Remove inclusion of
"rust-proc-macro-invoc-lexer.h".
* expand/rust-macro-invoc-lexer.cc
(MacroInvocLexer::peek_token): Handle changes to MacroInvocLexer
internal representation.
(MacroInvocLexer::get_token_slice): Likewise.
(MacroInvocLexer::split_current_token): Likewise and fix
iterator handling.
* expand/rust-macro-invoc-lexer.h (class MacroInvocLexerBase):
Remove and combine with...
(class MacroInvocLexer): ...its now only derived class.
* expand/rust-proc-macro-invoc-lexer.cc: Removed.
* expand/rust-proc-macro-invoc-lexer.h: Removed.
* parse/rust-parse-impl-proc-macro.cc: Removed.
Signed-off-by: Owen Avery <[email protected]>
---
This change was merged into the gccrs repository and is posted here for
upstream visibility and potential drive-by review, as requested by GCC
release managers.
Each commit email contains a link to its details on github from where you can
find the Pull-Request and associated discussions.
Commit on github:
https://github.com/Rust-GCC/gccrs/commit/252649e2acc7b6ac2dc997ed7951f387fb6c5eef
The commit has been mentioned in the following pull-request(s):
- https://github.com/Rust-GCC/gccrs/pull/4430
gcc/rust/Make-lang.in | 2 -
gcc/rust/ast/rust-ast.cc | 11 +++-
gcc/rust/ast/rust-macro.h | 2 +-
gcc/rust/expand/rust-macro-expand.cc | 8 +--
gcc/rust/expand/rust-macro-expand.h | 1 -
gcc/rust/expand/rust-macro-invoc-lexer.cc | 21 +++---
gcc/rust/expand/rust-macro-invoc-lexer.h | 48 +++++++-------
.../expand/rust-proc-macro-invoc-lexer.cc | 65 -------------------
gcc/rust/expand/rust-proc-macro-invoc-lexer.h | 48 --------------
gcc/rust/parse/rust-parse-impl-proc-macro.cc | 34 ----------
10 files changed, 48 insertions(+), 192 deletions(-)
delete mode 100644 gcc/rust/expand/rust-proc-macro-invoc-lexer.cc
delete mode 100644 gcc/rust/expand/rust-proc-macro-invoc-lexer.h
delete mode 100644 gcc/rust/parse/rust-parse-impl-proc-macro.cc
diff --git a/gcc/rust/Make-lang.in b/gcc/rust/Make-lang.in
index c0b2ef7e6..5c8f0d902 100644
--- a/gcc/rust/Make-lang.in
+++ b/gcc/rust/Make-lang.in
@@ -73,7 +73,6 @@ GRS_OBJS = \
rust/rust-lex.o \
rust/rust-cfg-parser.o \
rust/rust-parse.o \
- rust/rust-parse-impl-proc-macro.o \
rust/rust-parse-impl-macro.o \
rust/rust-parse-impl-lexer.o \
rust/rust-ast.o \
@@ -109,7 +108,6 @@ GRS_OBJS = \
rust/rust-derive-hash.o \
rust/rust-proc-macro.o \
rust/rust-macro-invoc-lexer.o \
- rust/rust-proc-macro-invoc-lexer.o \
rust/rust-macro-substitute-ctx.o \
rust/rust-macro-builtins.o \
rust/rust-macro-builtins-helpers.o \
diff --git a/gcc/rust/ast/rust-ast.cc b/gcc/rust/ast/rust-ast.cc
index 9813e7795..30fb5f4d5 100644
--- a/gcc/rust/ast/rust-ast.cc
+++ b/gcc/rust/ast/rust-ast.cc
@@ -3554,7 +3554,12 @@ DelimTokenTree::parse_to_meta_item () const
/* assume top-level delim token tree in attribute - convert all nested ones
* to token stream */
- std::vector<std::unique_ptr<Token>> token_stream = to_token_stream ();
+ std::vector<std::unique_ptr<Token>> token_stream_wrapped = to_token_stream
();
+
+ std::vector<const_TokenPtr> token_stream;
+ token_stream.reserve (token_stream_wrapped.size ());
+ for (auto &tk : token_stream_wrapped)
+ token_stream.push_back (tk->get_tok_ptr ());
AttributeParser parser (std::move (token_stream));
std::vector<std::unique_ptr<MetaItemInner>> meta_items (
@@ -3563,8 +3568,8 @@ DelimTokenTree::parse_to_meta_item () const
return new AttrInputMetaItemContainer (std::move (meta_items));
}
-AttributeParser::AttributeParser (
- std::vector<std::unique_ptr<Token>> token_stream, int stream_start_pos)
+AttributeParser::AttributeParser (std::vector<const_TokenPtr> token_stream,
+ int stream_start_pos)
: lexer (new MacroInvocLexer (std::move (token_stream))),
parser (new Parser<MacroInvocLexer> (*lexer))
{
diff --git a/gcc/rust/ast/rust-macro.h b/gcc/rust/ast/rust-macro.h
index 2fe190e1d..ff6cca5af 100644
--- a/gcc/rust/ast/rust-macro.h
+++ b/gcc/rust/ast/rust-macro.h
@@ -1134,7 +1134,7 @@ private:
std::unique_ptr<Parser<MacroInvocLexer>> parser;
public:
- AttributeParser (std::vector<std::unique_ptr<Token>> token_stream,
+ AttributeParser (std::vector<const_TokenPtr> token_stream,
int stream_start_pos = 0);
~AttributeParser ();
diff --git a/gcc/rust/expand/rust-macro-expand.cc
b/gcc/rust/expand/rust-macro-expand.cc
index aeea4c780..af29be7d7 100644
--- a/gcc/rust/expand/rust-macro-expand.cc
+++ b/gcc/rust/expand/rust-macro-expand.cc
@@ -177,9 +177,9 @@ MacroExpander::expand_eager_invocations
(AST::MacroInvocation &invoc)
// we need to create a clone of the delimited token tree as the lexer
// expects ownership of the tokens
- std::vector<std::unique_ptr<Rust::AST::Token>> dtt_clone;
+ std::vector<const_TokenPtr> dtt_clone;
for (auto &tok : stream)
- dtt_clone.emplace_back (tok->clone_token ());
+ dtt_clone.emplace_back (tok->get_tok_ptr ());
MacroInvocLexer lex (std::move (dtt_clone));
Parser<MacroInvocLexer> parser (lex);
@@ -1212,8 +1212,8 @@ MacroExpander::transcribe_rule (
AST::Fragment
MacroExpander::parse_proc_macro_output (ProcMacro::TokenStream ts)
{
- ProcMacroInvocLexer lex (convert (ts));
- Parser<ProcMacroInvocLexer> parser (lex);
+ MacroInvocLexer lex (convert (ts));
+ Parser<MacroInvocLexer> parser (lex);
std::vector<AST::SingleASTNode> nodes;
switch (peek_context ())
diff --git a/gcc/rust/expand/rust-macro-expand.h
b/gcc/rust/expand/rust-macro-expand.h
index cda4292cd..d843227be 100644
--- a/gcc/rust/expand/rust-macro-expand.h
+++ b/gcc/rust/expand/rust-macro-expand.h
@@ -29,7 +29,6 @@
#include "rust-hir-map.h"
#include "rust-name-resolver.h"
#include "rust-macro-invoc-lexer.h"
-#include "rust-proc-macro-invoc-lexer.h"
#include "rust-token-converter.h"
#include "rust-ast-collector.h"
#include "rust-system.h"
diff --git a/gcc/rust/expand/rust-macro-invoc-lexer.cc
b/gcc/rust/expand/rust-macro-invoc-lexer.cc
index 306a0bee6..717f53dd7 100644
--- a/gcc/rust/expand/rust-macro-invoc-lexer.cc
+++ b/gcc/rust/expand/rust-macro-invoc-lexer.cc
@@ -27,7 +27,7 @@ MacroInvocLexer::peek_token (int n)
if ((offs + n) >= token_stream.size ())
return Token::make (END_OF_FILE, UNDEF_LOCATION);
- return token_stream.at (offs + n)->get_tok_ptr ();
+ return token_stream.at (offs + n);
}
void
@@ -39,14 +39,12 @@ MacroInvocLexer::split_current_token (TokenId new_left,
TokenId new_right)
auto l_tok = Token::make (new_left, current_token->get_locus ());
auto r_tok = Token::make (new_right, current_token->get_locus ());
- token_stream.erase (current_pos);
+ current_pos = token_stream.erase (current_pos);
// `insert` inserts before the specified position, so we insert the right one
// then the left
- token_stream.insert (current_pos,
- std::unique_ptr<AST::Token> (new AST::Token (r_tok)));
- token_stream.insert (current_pos,
- std::unique_ptr<AST::Token> (new AST::Token (l_tok)));
+ current_pos = token_stream.insert (current_pos, r_tok);
+ token_stream.insert (current_pos, l_tok);
}
void
@@ -56,12 +54,12 @@ MacroInvocLexer::split_current_token (std::vector<TokenPtr>
new_tokens)
auto current_pos = token_stream.begin () + offs;
- token_stream.erase (current_pos);
+ current_pos = token_stream.erase (current_pos);
- for (size_t i = 1; i < new_tokens.size (); i++)
+ for (auto &tk : new_tokens)
{
- token_stream.insert (current_pos + i, std::unique_ptr<AST::Token> (
- new AST::Token (new_tokens[i])));
+ current_pos = token_stream.insert (current_pos, std::move (tk));
+ current_pos++;
}
}
@@ -73,7 +71,8 @@ MacroInvocLexer::get_token_slice (size_t start_idx, size_t
end_idx) const
rust_assert (end_idx < token_stream.size ());
for (size_t i = start_idx; i < end_idx; i++)
- slice.emplace_back (token_stream[i]->clone_token ());
+ slice.emplace_back (
+ std::unique_ptr<AST::Token> (new AST::Token (token_stream[i])));
return slice;
}
diff --git a/gcc/rust/expand/rust-macro-invoc-lexer.h
b/gcc/rust/expand/rust-macro-invoc-lexer.h
index 2d285df1d..6743159cf 100644
--- a/gcc/rust/expand/rust-macro-invoc-lexer.h
+++ b/gcc/rust/expand/rust-macro-invoc-lexer.h
@@ -22,40 +22,28 @@
#include "rust-ast.h"
namespace Rust {
-template <class T> class MacroInvocLexerBase
+
+class MacroInvocLexer
{
public:
- MacroInvocLexerBase (std::vector<T> stream)
+ MacroInvocLexer (std::vector<const_TokenPtr> stream)
: offs (0), token_stream (std::move (stream))
{}
+ MacroInvocLexer (const std::vector<std::unique_ptr<AST::Token>> &stream)
+ : offs (0)
+ {
+ token_stream.reserve (stream.size ());
+ for (auto &tk : stream)
+ token_stream.push_back (tk->get_tok_ptr ());
+ }
+
// Advances current token to n + 1 tokens ahead of current position.
void skip_token (int n) { offs += (n + 1); }
// Skips the current token.
void skip_token () { skip_token (0); }
- std::string get_filename () const
- {
- // FIXME
- rust_unreachable ();
- return "FIXME";
- }
-
- size_t get_offs () const { return offs; }
-
-protected:
- size_t offs;
- std::vector<T> token_stream;
-};
-
-class MacroInvocLexer : public MacroInvocLexerBase<std::unique_ptr<AST::Token>>
-{
-public:
- MacroInvocLexer (std::vector<std::unique_ptr<AST::Token>> stream)
- : MacroInvocLexerBase (std::move (stream))
- {}
-
// Returns token n tokens ahead of current position.
const_TokenPtr peek_token (int n);
@@ -71,7 +59,21 @@ public:
std::vector<std::unique_ptr<AST::Token>>
get_token_slice (size_t start_idx, size_t end_idx) const;
+
+ std::string get_filename () const
+ {
+ // FIXME
+ rust_unreachable ();
+ return "FIXME";
+ }
+
+ size_t get_offs () const { return offs; }
+
+protected:
+ size_t offs;
+ std::vector<const_TokenPtr> token_stream;
};
+
} // namespace Rust
#endif // RUST_MACRO_INVOC_LEXER_H
diff --git a/gcc/rust/expand/rust-proc-macro-invoc-lexer.cc
b/gcc/rust/expand/rust-proc-macro-invoc-lexer.cc
deleted file mode 100644
index 0949c717c..000000000
--- a/gcc/rust/expand/rust-proc-macro-invoc-lexer.cc
+++ /dev/null
@@ -1,65 +0,0 @@
-// Copyright (C) 2020-2026 Free Software Foundation, Inc.
-
-// This file is part of GCC.
-
-// GCC is free software; you can redistribute it and/or modify it under
-// the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3, or (at your option) any later
-// version.
-
-// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
-// WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-// for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with GCC; see the file COPYING3. If not see
-// <http://www.gnu.org/licenses/>.
-
-#include "rust-proc-macro-invoc-lexer.h"
-#include "rust-token.h"
-
-namespace Rust {
-
-const_TokenPtr
-ProcMacroInvocLexer::peek_token (int n)
-{
- if ((offs + n) >= token_stream.size ())
- return Token::make (END_OF_FILE, UNDEF_LOCATION);
-
- return token_stream.at (offs + n);
-}
-
-void
-ProcMacroInvocLexer::split_current_token (TokenId new_left, TokenId new_right)
-{
- auto ¤t_token = token_stream.at (offs);
- auto current_pos = token_stream.begin () + offs;
-
- auto l_tok = Token::make (new_left, current_token->get_locus ());
- auto r_tok = Token::make (new_right, current_token->get_locus ());
-
- token_stream.erase (current_pos);
-
- // `insert` inserts before the specified position, so we insert the right one
- // then the left
- token_stream.insert (current_pos, l_tok);
- token_stream.insert (current_pos, r_tok);
-}
-
-void
-ProcMacroInvocLexer::split_current_token (std::vector<TokenPtr> new_tokens)
-{
- rust_assert (new_tokens.size () > 0);
-
- auto current_pos = token_stream.begin () + offs;
-
- token_stream.erase (current_pos);
-
- for (size_t i = 1; i < new_tokens.size (); i++)
- {
- token_stream.insert (current_pos + i, new_tokens[i]);
- }
-}
-
-} // namespace Rust
diff --git a/gcc/rust/expand/rust-proc-macro-invoc-lexer.h
b/gcc/rust/expand/rust-proc-macro-invoc-lexer.h
deleted file mode 100644
index 19542c125..000000000
--- a/gcc/rust/expand/rust-proc-macro-invoc-lexer.h
+++ /dev/null
@@ -1,48 +0,0 @@
-// Copyright (C) 2020-2026 Free Software Foundation, Inc.
-
-// This file is part of GCC.
-
-// GCC is free software; you can redistribute it and/or modify it under
-// the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3, or (at your option) any later
-// version.
-
-// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
-// WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-// for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with GCC; see the file COPYING3. If not see
-// <http://www.gnu.org/licenses/>.
-
-#ifndef RUST_PROC_MACRO_INVOC_LEXER_H
-#define RUST_PROC_MACRO_INVOC_LEXER_H
-
-#include "rust-lex.h"
-#include "rust-macro-invoc-lexer.h"
-
-namespace Rust {
-class ProcMacroInvocLexer : public MacroInvocLexerBase<const_TokenPtr>
-{
-public:
- ProcMacroInvocLexer (std::vector<const_TokenPtr> stream)
- : MacroInvocLexerBase (std::move (stream))
- {}
-
- // Returns token n tokens ahead of current position.
- const_TokenPtr peek_token (int n);
-
- // Peeks the current token.
- const_TokenPtr peek_token () { return peek_token (0); }
-
- // Splits the current token into two. Intended for use with nested generics
- // closes (i.e. T<U<X>> where >> is wrongly lexed as one token). Note that
- // this will only work with "simple" tokens like punctuation.
- void split_current_token (TokenId new_left, TokenId new_right);
-
- void split_current_token (std::vector<TokenPtr> new_tokens);
-};
-} // namespace Rust
-
-#endif /* ! RUST_PROC_MACRO_INVOC_LEXER_H */
diff --git a/gcc/rust/parse/rust-parse-impl-proc-macro.cc
b/gcc/rust/parse/rust-parse-impl-proc-macro.cc
deleted file mode 100644
index 342d3478b..000000000
--- a/gcc/rust/parse/rust-parse-impl-proc-macro.cc
+++ /dev/null
@@ -1,34 +0,0 @@
-// Copyright (C) 2025-2026 Free Software Foundation, Inc.
-
-// This file is part of GCC.
-
-// GCC is free software; you can redistribute it and/or modify it under
-// the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3, or (at your option) any later
-// version.
-
-// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
-// WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-// for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with GCC; see the file COPYING3. If not see
-// <http://www.gnu.org/licenses/>.
-
-#include "rust-parse-impl.hxx"
-#include "rust-proc-macro-invoc-lexer.h"
-
-namespace Rust {
-
-template tl::expected<std::unique_ptr<AST::Item>, Parse::Error::Item>
-Parser<ProcMacroInvocLexer>::parse_item (bool);
-
-template std::unique_ptr<AST::Stmt>
- Parser<ProcMacroInvocLexer>::parse_stmt (ParseRestrictions);
-
-// instantiate entire class (or just more functions) if necessary
-
-// template class Parser<ProcMacroInvocLexer>;
-
-} // namespace Rust
base-commit: c4bc3153bffc32eced40a54a83fe8863729456c8
--
2.52.0
