https://github.com/python/cpython/commit/b48599b8017f896c965e89c402bb70feef49808a
commit: b48599b8017f896c965e89c402bb70feef49808a
branch: main
author: Alex Prengère <2138730+alexpreng...@users.noreply.github.com>
committer: AA-Turner <9087854+aa-tur...@users.noreply.github.com>
date: 2025-05-08T02:28:20+01:00
summary:
gh-133555: Allow regenerating the parser with Python < 3.14 (#133557)
Co-authored-by: Adam Turner <9087854+aa-tur...@users.noreply.github.com>
files:
M Grammar/Tokens
M Tools/peg_generator/pegen/parser_generator.py
diff --git a/Grammar/Tokens b/Grammar/Tokens
index e40a4437afb009..0547e6ed08f79a 100644
--- a/Grammar/Tokens
+++ b/Grammar/Tokens
@@ -1,3 +1,8 @@
+# When adding new tokens, remember to update the PEG generator in
+# Tools/peg_generator/pegen/parser_generator.py
+# This will ensure that older versions of Python can generate a Python parser
+# using "python -m pegen python <GRAMMAR FILE>".
+
ENDMARKER
NAME
NUMBER
diff --git a/Tools/peg_generator/pegen/parser_generator.py
b/Tools/peg_generator/pegen/parser_generator.py
index 6ce0649aefe7ff..52ae743c26b6b8 100644
--- a/Tools/peg_generator/pegen/parser_generator.py
+++ b/Tools/peg_generator/pegen/parser_generator.py
@@ -81,6 +81,11 @@ def __init__(self, rules: Dict[str, Rule], tokens: Set[str]):
self.tokens.add("FSTRING_START")
self.tokens.add("FSTRING_END")
self.tokens.add("FSTRING_MIDDLE")
+ # If python < 3.14 add the virtual tstring tokens
+ if sys.version_info < (3, 14, 0, 'beta', 1):
+ self.tokens.add("TSTRING_START")
+ self.tokens.add("TSTRING_END")
+ self.tokens.add("TSTRING_MIDDLE")
def visit_NameLeaf(self, node: NameLeaf) -> None:
if node.value not in self.rules and node.value not in self.tokens:
_______________________________________________
Python-checkins mailing list -- python-checkins@python.org
To unsubscribe send an email to python-checkins-le...@python.org
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: arch...@mail-archive.com
