Package: src:python-pegen Version: 0.3.0-2 User: [email protected] Usertags: python3.15 Tags: patch, ftbfs, forky, sid Severity: important
Hi! While rebuilding the python related packages against the Python 3.15rc2 version we found that python-pegen fails to build from source [1]. The parser doesn't support Python 3.15. The 3.14 support had been added as a Debian patch, but the PR was never merged upstream. I have sent a new upstream PR for the 3.15 support [2]. I've applied this fix in the sandbox [3] to verify that it builds successfully, please consider applying the patch to support the upcoming 3.15 version. Setting the severity to important for now. Once Python 3.15 is released, it will be added to python3-defaults and this bug will become release critical. Happy hacking, [1]: https://debusine.debian.net/debian/r-python-python3.15/artifact/4440301/ [2]: https://github.com/we-like-parsers/pegen/pull/117 [3]: https://debusine.debian.net/debian/r-python-python3.15/ -- "Can you imagine what I would do if I could do all I can?" -- Sun Tzu Saludos /\/\ /\ >< `/
Description: Add unsupported syntax test for unparenthesized except on Python < 3.14 Verify that targeting py_version < (3, 14) properly rejects unparenthesized multiple exception types. Forwarded: https://github.com/we-like-parsers/pegen/pull/117 Index: python-pegen/tests/python_parser/test_unsupported_syntax.py =================================================================== --- python-pegen.orig/tests/python_parser/test_unsupported_syntax.py +++ python-pegen/tests/python_parser/test_unsupported_syntax.py @@ -197,3 +197,19 @@ def test_generic_class_statement(python_ pp.parse("file") assert "Type parameter lists are" in e.exconly() + + +# unparenthesized except 3.14 [email protected]( + "source", ["try:\n\tpass\nexcept ValueError, IndexError:\n\tpass"] +) +def test_unparenthesized_except(python_parser_cls, source): + temp = io.StringIO(source) + tokengen = tokenize.generate_tokens(temp.readline) + tokenizer = Tokenizer(tokengen, verbose=False) + pp = python_parser_cls(tokenizer, py_version=(3, 13)) + with pytest.raises(SyntaxError) as e: + pp.parse("file") + + assert "multiple exception types must be parenthesized" in e.exconly() +
Description: Remove invalid LOCATIONS argument from ast.Expression in tests In Python 3.15, passing location attributes to ast.Expression raises TypeError because ast.Expression does not accept location fields. Forwarded: https://github.com/we-like-parsers/pegen/pull/117 Index: python-pegen/tests/test_pegen.py =================================================================== --- python-pegen.orig/tests/test_pegen.py +++ python-pegen/tests/test_pegen.py @@ -339,7 +339,7 @@ def test_left_recursive() -> None: def test_python_expr() -> None: grammar = """ - start: expr NEWLINE? $ { ast.Expression(expr, LOCATIONS) } + start: expr NEWLINE? $ { ast.Expression(expr) } expr: ( expr '+' term { ast.BinOp(expr, ast.Add(), term, LOCATIONS) } | expr '-' term { ast.BinOp(expr, ast.Sub(), term, LOCATIONS) } | term { term } @@ -664,7 +664,7 @@ def test_unreachable_implicit3() -> None def test_locations_in_alt_action_and_group() -> None: grammar = """ - start: t=term NEWLINE? $ { ast.Expression(t, LOCATIONS) } + start: t=term NEWLINE? $ { ast.Expression(t) } term: | l=term '*' r=factor { ast.BinOp(l, ast.Mult(), r, LOCATIONS) } | l=term '/' r=factor { ast.BinOp(l, ast.Div(), r, LOCATIONS) }
Description: Fix ast node constructor argument typos in grammar Fix typos in ast.arg (annotations -> annotation), ast.MatchAs (target -> name), and ast.MatchStar (target -> name), which are strictly validated in Python 3.15. Forwarded: https://github.com/we-like-parsers/pegen/pull/117 Index: python-pegen/data/python.gram =================================================================== --- python-pegen.orig/data/python.gram +++ python-pegen/data/python.gram @@ -937,7 +937,7 @@ param_maybe_default[Tuple[ast.arg, Any]] | a=param c=default? tc=TYPE_COMMENT? &')' { (self.set_arg_type_comment(a, tc), c) } param: a=NAME b=annotation? { ast.arg(arg=a.string, annotation=b, LOCATIONS) } param_star_annotation: a=NAME b=star_annotation { - ast.arg(arg=a.string, annotations=b, LOCATIONS) + ast.arg(arg=a.string, annotation=b, LOCATIONS) } annotation: ':' a=expression { a } star_annotation: ':' a=star_expression { a } @@ -1199,7 +1199,7 @@ pattern_capture_target[str]: | !"_" name=NAME !('.' | '(' | '=') { name.string } wildcard_pattern["ast.MatchAs"]: - | "_" { ast.MatchAs(pattern=None, target=None, LOCATIONS) } + | "_" { ast.MatchAs(pattern=None, name=None, LOCATIONS) } value_pattern["ast.MatchValue"]: | attr=attr !('.' | '(' | '=') { ast.MatchValue(value=attr, LOCATIONS) } @@ -1234,7 +1234,7 @@ maybe_star_pattern: star_pattern: | '*' target=pattern_capture_target { ast.MatchStar(name=target, LOCATIONS) } - | '*' wildcard_pattern { ast.MatchStar(target=None, LOCATIONS) } + | '*' wildcard_pattern { ast.MatchStar(name=None, LOCATIONS) } mapping_pattern: | '{' '}' { ast.MatchMapping(keys=[], patterns=[], rest=None, LOCATIONS) }
Description: Support is_lazy attribute in Import and ImportFrom AST nodes for Python 3.15 Python 3.15 added the is_lazy attribute (default 0) to ast.Import and ast.ImportFrom AST nodes. Forwarded: https://github.com/we-like-parsers/pegen/pull/117 Index: python-pegen/data/python.gram =================================================================== --- python-pegen.orig/data/python.gram +++ python-pegen/data/python.gram @@ -713,14 +713,22 @@ import_stmt[ast.Import]: # Import statements # ----------------- -import_name[ast.Import]: 'import' a=dotted_as_names { ast.Import(names=a, LOCATIONS) } +import_name[ast.Import]: 'import' a=dotted_as_names { + ast.Import(names=a, is_lazy=0, LOCATIONS) + if sys.version_info >= (3, 15) else + ast.Import(names=a, LOCATIONS) + } # note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS import_from[ast.ImportFrom]: | 'from' a=('.' | '...')* b=dotted_name 'import' c=import_from_targets { + ast.ImportFrom(module=b, names=c, level=self.extract_import_level(a), is_lazy=0, LOCATIONS) + if sys.version_info >= (3, 15) else ast.ImportFrom(module=b, names=c, level=self.extract_import_level(a), LOCATIONS) } | 'from' a=('.' | '...')+ 'import' b=import_from_targets { + ast.ImportFrom(names=b, level=self.extract_import_level(a), is_lazy=0, LOCATIONS) + if sys.version_info >= (3, 15) else ast.ImportFrom(names=b, level=self.extract_import_level(a), LOCATIONS) if sys.version_info >= (3, 9) else ast.ImportFrom(module=None, names=b, level=self.extract_import_level(a), LOCATIONS)
Description: Add tests for is_lazy import AST nodes on Python 3.15 Verify that parsed ast.Import and ast.ImportFrom nodes have is_lazy set to 0. Forwarded: https://github.com/we-like-parsers/pegen/pull/117 Index: python-pegen/tests/python_parser/test_ast_parsing.py =================================================================== --- python-pegen.orig/tests/python_parser/test_ast_parsing.py +++ python-pegen/tests/python_parser/test_ast_parsing.py @@ -108,3 +108,11 @@ def test_parser(python_parse_file, pytho p = ast.dump(python_parse_file(path), **kwargs) diff = "\n".join(difflib.unified_diff(o.split("\n"), p.split("\n"), "cpython", "python-pegen")) assert not diff + + [email protected](sys.version_info < (3, 15), reason="is_lazy added in Python 3.15+") +def test_lazy_imports_ast(python_parse_str): + tree = python_parse_str("import foo\nfrom bar import baz\n", "exec") + assert tree.body[0].is_lazy == 0 + assert tree.body[1].is_lazy == 0 +
Description: Update syntax error handling tests for Python 3.15 Normalize diagnostic message phrasing changes in Python 3.15 (e.g., parameter vs argument, * argument may appear only once, double starred expression). Forwarded: https://github.com/we-like-parsers/pegen/pull/117 Index: python-pegen/tests/python_parser/test_syntax_error_handling.py =================================================================== --- python-pegen.orig/tests/python_parser/test_syntax_error_handling.py +++ python-pegen/tests/python_parser/test_syntax_error_handling.py @@ -46,7 +46,14 @@ def parse_invalid_syntax( if sys.version_info >= min_python_version: # This fails for Python < 3.10.5 but keeping the fix for a patch version is not # worth it - assert message in py_exc.args[0] + py_msg = py_exc.args[0] + if sys.version_info >= (3, 15): + py_msg = ( + py_msg.replace("parameter", "argument") + .replace("* may appear only once", "* argument may appear only once") + .replace("dict unpacking", "double starred expression") + ) + assert message in py_msg or message in py_exc.args[0] print(str(e.exconly())) assert message in str(e.exconly())
Description: Support unpacking in comprehensions for Python 3.15 Support iterable unpacking and dictionary unpacking in comprehensions (PEP 742 / PEP 646) which is valid syntax in Python 3.15. Forwarded: https://github.com/we-like-parsers/pegen/pull/117 Index: python-pegen/data/python.gram =================================================================== --- python-pegen.orig/data/python.gram +++ python-pegen/data/python.gram @@ -1786,21 +1786,38 @@ for_if_clause[ast.comprehension]: | invalid_for_target listcomp[ast.ListComp]: - | '[' a=named_expression b=for_if_clauses ']' { ast.ListComp(elt=a, generators=b, LOCATIONS) } + | '[' a=(starred_expression | named_expression) b=for_if_clauses ']' { + PY_VERSION >= (3, 15); + ast.ListComp(elt=a, generators=b, LOCATIONS) } + | '[' a=named_expression b=for_if_clauses ']' { + PY_VERSION < (3, 15); + ast.ListComp(elt=a, generators=b, LOCATIONS) } | invalid_comprehension setcomp[ast.SetComp]: - | '{' a=named_expression b=for_if_clauses '}' { ast.SetComp(elt=a, generators=b, LOCATIONS) } + | '{' a=(starred_expression | named_expression) b=for_if_clauses '}' { + PY_VERSION >= (3, 15); + ast.SetComp(elt=a, generators=b, LOCATIONS) } + | '{' a=named_expression b=for_if_clauses '}' { + PY_VERSION < (3, 15); + ast.SetComp(elt=a, generators=b, LOCATIONS) } | invalid_comprehension genexp[ast.GeneratorExp]: + | '(' a=(starred_expression | assignment_expression | expression !':=') b=for_if_clauses ')' { + PY_VERSION >= (3, 15); + ast.GeneratorExp(elt=a, generators=b, LOCATIONS) } | '(' a=( assignment_expression | expression !':=') b=for_if_clauses ')' { - ast.GeneratorExp(elt=a, generators=b, LOCATIONS) - } + PY_VERSION < (3, 15); + ast.GeneratorExp(elt=a, generators=b, LOCATIONS) } | invalid_comprehension dictcomp[ast.DictComp]: - | '{' a=kvpair b=for_if_clauses '}' { ast.DictComp(key=a[0], value=a[1], generators=b, LOCATIONS) } + | '{' '**' a=bitwise_or b=for_if_clauses '}' { + PY_VERSION >= (3, 15); + ast.DictComp(key=a, value=None, generators=b, LOCATIONS) } + | '{' a=kvpair b=for_if_clauses '}' { + ast.DictComp(key=a[0], value=a[1], generators=b, LOCATIONS) } | invalid_dict_comprehension # FUNCTION CALL ARGUMENTS @@ -2100,6 +2117,7 @@ invalid_block[NoReturn]: | NEWLINE !INDENT { self.raise_indentation_error("expected an indented block") } invalid_comprehension[NoReturn]: | ('[' | '(' | '{') a=starred_expression for_if_clauses { + PY_VERSION < (3, 15); self.raise_syntax_error_known_location("iterable unpacking cannot be used in comprehension", a) } | ('[' | '{') a=star_named_expression ',' b=star_named_expressions for_if_clauses { @@ -2114,6 +2132,7 @@ invalid_comprehension[NoReturn]: } invalid_dict_comprehension[NoReturn]: | '{' a='**' bitwise_or for_if_clauses '}' { + PY_VERSION < (3, 15); self.raise_syntax_error_known_location("dict unpacking cannot be used in dict comprehension", a) } invalid_parameters[NoReturn]:
Description: Add tests for comprehension unpacking on Python 3.15 Add test data and test cases for PEP 742 / PEP 646 comprehension unpacking in test_ast_parsing.py, test_syntax_error_handling.py, and test_unsupported_syntax.py. Forwarded: https://github.com/we-like-parsers/pegen/pull/117 Index: python-pegen/tests/python_parser/data/comprehensions_unpacking.py =================================================================== --- /dev/null +++ python-pegen/tests/python_parser/data/comprehensions_unpacking.py @@ -0,0 +1,10 @@ +a = [*k for k in g] + + +b = {*k for k in g} + + +c = (*k for k in g) + + +d = {**k for k in g} Index: python-pegen/tests/python_parser/test_ast_parsing.py =================================================================== --- python-pegen.orig/tests/python_parser/test_ast_parsing.py +++ python-pegen/tests/python_parser/test_ast_parsing.py @@ -23,6 +23,13 @@ import pytest "async.py", "call.py", "comprehensions.py", + pytest.param( + "comprehensions_unpacking.py", + marks=pytest.mark.skipif( + sys.version_info < (3, 15), + reason="Unpacking in comprehensions allowed only in Python 3.15+", + ), + ), "expressions.py", "fstrings.py", "function_def.py", Index: python-pegen/tests/python_parser/test_syntax_error_handling.py =================================================================== --- python-pegen.orig/tests/python_parser/test_syntax_error_handling.py +++ python-pegen/tests/python_parser/test_syntax_error_handling.py @@ -425,9 +425,17 @@ def test_invalid_del_statements( def test_invalid_comprehension( python_parse_file, python_parse_str, tmp_path, source, message, start, end ): - parse_invalid_syntax( - python_parse_file, python_parse_str, tmp_path, source, SyntaxError, message, start, end - ) + if sys.version_info >= (3, 15) and message in ( + "iterable unpacking cannot be used in comprehension", + "dict unpacking cannot be used in dict comprehension", + ): + ast_pegen = python_parse_str(source, "exec") + ast_cpython = ast.parse(source) + assert ast.dump(ast_pegen) == ast.dump(ast_cpython) + else: + parse_invalid_syntax( + python_parse_file, python_parse_str, tmp_path, source, SyntaxError, message, start, end + ) @pytest.mark.parametrize( Index: python-pegen/tests/python_parser/test_unsupported_syntax.py =================================================================== --- python-pegen.orig/tests/python_parser/test_unsupported_syntax.py +++ python-pegen/tests/python_parser/test_unsupported_syntax.py @@ -213,3 +213,25 @@ def test_unparenthesized_except(python_p assert "multiple exception types must be parenthesized" in e.exconly() + +# comprehension unpacking 3.15 [email protected]( + "source, message", + [ + ("[*a for a in b]", "iterable unpacking cannot be used in comprehension"), + ("{*a for a in b}", "iterable unpacking cannot be used in comprehension"), + ("(*a for a in b)", "iterable unpacking cannot be used in comprehension"), + ("{**a for a in b}", "dict unpacking cannot be used in dict comprehension"), + ], +) +def test_comprehension_unpacking(python_parser_cls, source, message): + temp = io.StringIO(source) + tokengen = tokenize.generate_tokens(temp.readline) + tokenizer = Tokenizer(tokengen, verbose=False) + pp = python_parser_cls(tokenizer, py_version=(3, 14)) + with pytest.raises(SyntaxError) as e: + pp.parse("file") + + assert message in e.exconly() + +
