https://github.com/python/cpython/commit/9dfef4e5f4ac3c1ce494c48f2476a694c12d72a5 commit: 9dfef4e5f4ac3c1ce494c48f2476a694c12d72a5 branch: main author: Pablo Galindo Salgado <[email protected]> committer: pablogsal <[email protected]> date: 2024-10-29T23:40:12Z summary:
gh-125588: Teach the python PEG generator the new f-string tokens (#125589) Signed-off-by: Pablo Galindo <[email protected]> files: A Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-13-52-48.gh-issue-125588.kCahyO.rst M Lib/test/test_peg_generator/test_pegen.py M Tools/peg_generator/pegen/grammar_parser.py M Tools/peg_generator/pegen/metagrammar.gram M Tools/peg_generator/pegen/parser.py M Tools/peg_generator/pegen/python_generator.py diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py index 54c9dce2d0c90d..d8606521345390 100644 --- a/Lib/test/test_peg_generator/test_pegen.py +++ b/Lib/test/test_peg_generator/test_pegen.py @@ -506,6 +506,14 @@ def test_python_expr(self) -> None: val = eval(code) self.assertEqual(val, 3.0) + def test_f_string_in_action(self) -> None: + grammar = """ + start: n=NAME NEWLINE? $ { f"name -> {n.string}" } + """ + parser_class = make_parser(grammar) + node = parse_string("a", parser_class) + self.assertEqual(node.strip(), "name -> a") + def test_nullable(self) -> None: grammar_source = """ start: sign NUMBER diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-13-52-48.gh-issue-125588.kCahyO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-13-52-48.gh-issue-125588.kCahyO.rst new file mode 100644 index 00000000000000..1d59a9c3c205b8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-13-52-48.gh-issue-125588.kCahyO.rst @@ -0,0 +1,2 @@ +The Python PEG generator can now use f-strings in the grammar actions. Patch +by Pablo Galindo diff --git a/Tools/peg_generator/pegen/grammar_parser.py b/Tools/peg_generator/pegen/grammar_parser.py index bf31fe53263645..2e3a607f7209b0 100644 --- a/Tools/peg_generator/pegen/grammar_parser.py +++ b/Tools/peg_generator/pegen/grammar_parser.py @@ -575,7 +575,7 @@ def target_atoms(self) -> Optional[str]: @memoize def target_atom(self) -> Optional[str]: - # target_atom: "{" ~ target_atoms? "}" | "[" ~ target_atoms? "]" | NAME "*" | NAME | NUMBER | STRING | "?" | ":" | !"}" !"]" OP + # target_atom: "{" ~ target_atoms? "}" | "[" ~ target_atoms? "]" | NAME "*" | NAME | NUMBER | STRING | FSTRING_START | FSTRING_MIDDLE | FSTRING_END | "?" | ":" | !"}" !"]" OP mark = self._mark() cut = False if ( @@ -625,6 +625,21 @@ def target_atom(self) -> Optional[str]: ): return string . string self._reset(mark) + if ( + (fstring_start := self.fstring_start()) + ): + return fstring_start . string + self._reset(mark) + if ( + (fstring_middle := self.fstring_middle()) + ): + return fstring_middle . string + self._reset(mark) + if ( + (fstring_end := self.fstring_end()) + ): + return fstring_end . string + self._reset(mark) if ( (literal := self.expect("?")) ): diff --git a/Tools/peg_generator/pegen/metagrammar.gram b/Tools/peg_generator/pegen/metagrammar.gram index f22c334ca371b2..f484c4781823bc 100644 --- a/Tools/peg_generator/pegen/metagrammar.gram +++ b/Tools/peg_generator/pegen/metagrammar.gram @@ -126,6 +126,9 @@ target_atom[str]: | NAME { name.string } | NUMBER { number.string } | STRING { string.string } + | FSTRING_START { fstring_start.string } + | FSTRING_MIDDLE { fstring_middle.string } + | FSTRING_END { fstring_end.string } | "?" { "?" } | ":" { ":" } | !"}" !"]" OP { op.string } diff --git a/Tools/peg_generator/pegen/parser.py b/Tools/peg_generator/pegen/parser.py index ed0aec9db2443f..692eb9ed2417d7 100644 --- a/Tools/peg_generator/pegen/parser.py +++ b/Tools/peg_generator/pegen/parser.py @@ -205,6 +205,36 @@ def string(self) -> Optional[tokenize.TokenInfo]: return self._tokenizer.getnext() return None + @memoize + def fstring_start(self) -> Optional[tokenize.TokenInfo]: + FSTRING_START = getattr(token, "FSTRING_START") + if not FSTRING_START: + return None + tok = self._tokenizer.peek() + if tok.type == FSTRING_START: + return self._tokenizer.getnext() + return None + + @memoize + def fstring_middle(self) -> Optional[tokenize.TokenInfo]: + FSTRING_MIDDLE = getattr(token, "FSTRING_MIDDLE") + if not FSTRING_MIDDLE: + return None + tok = self._tokenizer.peek() + if tok.type == FSTRING_MIDDLE: + return self._tokenizer.getnext() + return None + + @memoize + def fstring_end(self) -> Optional[tokenize.TokenInfo]: + FSTRING_END = getattr(token, "FSTRING_END") + if not FSTRING_END: + return None + tok = self._tokenizer.peek() + if tok.type == FSTRING_END: + return self._tokenizer.getnext() + return None + @memoize def op(self) -> Optional[tokenize.TokenInfo]: tok = self._tokenizer.peek() diff --git a/Tools/peg_generator/pegen/python_generator.py b/Tools/peg_generator/pegen/python_generator.py index 7057135a9061f6..4bb26480ebc0af 100644 --- a/Tools/peg_generator/pegen/python_generator.py +++ b/Tools/peg_generator/pegen/python_generator.py @@ -99,7 +99,8 @@ def visit_NameLeaf(self, node: NameLeaf) -> Tuple[Optional[str], str]: name = node.value if name == "SOFT_KEYWORD": return "soft_keyword", "self.soft_keyword()" - if name in ("NAME", "NUMBER", "STRING", "OP", "TYPE_COMMENT"): + if name in ("NAME", "NUMBER", "STRING", "OP", "TYPE_COMMENT", + "FSTRING_END", "FSTRING_MIDDLE", "FSTRING_START"): name = name.lower() return name, f"self.{name}()" if name in ("NEWLINE", "DEDENT", "INDENT", "ENDMARKER"): _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3/lists/python-checkins.python.org/ Member address: [email protected]
