https://github.com/python/cpython/commit/59a691361cb7b1d4fde3b92f98a61490381c62c9 commit: 59a691361cb7b1d4fde3b92f98a61490381c62c9 branch: main author: Serhiy Storchaka <[email protected]> committer: serhiy-storchaka <[email protected]> date: 2026-09-04T14:14:13+03:00 summary:
gh-156894: Fix the position of syntax errors which cover a range (GH-156901) The callers of _PyTokenizer_syntaxerror_known_range() pass columns in bytes, but SyntaxError.offset and end_offset are columns in characters. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> files: A Misc/NEWS.d/next/Core_and_Builtins/2026-09-03-15-20-00.gh-issue-156894.Rt9Bx4.rst M Lib/test/test_exceptions.py M Lib/test/test_tokenize.py M Parser/tokenizer/helpers.c diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index c34cf44d722456c..0c02b38dd3c0a89 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -233,6 +233,23 @@ def test_error_offset_continuation_characters(self): check = self.check check('"\\\n"(1 for c in I,\\\n\\', 2, 2) + def testSyntaxErrorRange(self): + # gh-156894: the position was reported in bytes, not in characters, + # for the errors which cover a range + for source, offset, end_offset in [ + ('abcd = 00010', 8, 11), + ('\u03b1\u03b2\u03b3\u03b4 = 00010', 8, 11), + ('a\u0301b\u0308c\u20d7d\u1ab0 = 00010', 12, 15), + ("abcd = ub'a'", 8, 10), + ("\u03b1\u03b2\u03b3\u03b4 = ub'a'", 8, 10), + ("a\u0301b\u0308c\u20d7d\u1ab0 = ub'a'", 12, 14), + ]: + with self.subTest(source=source): + with self.assertRaises(SyntaxError) as cm: + compile(source, '<testcase>', 'exec') + self.assertEqual(cm.exception.offset, offset) + self.assertEqual(cm.exception.end_offset, end_offset) + def testSyntaxErrorOffset(self): check = self.check check('def fact(x):\n\treturn x!\n', 2, 10) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 7e02191db86be5a..c471f857660ec90 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -2566,7 +2566,7 @@ def test_tolerant_incompatible_prefix_position_after_non_ascii(self): self._get_tokens('bé )tf"2 ', extra_tokens=True) self.assertEqual( caught.exception.args, - ("'f' and 't' prefixes are incompatible", (1, 6)), + ("'f' and 't' prefixes are incompatible", (1, 5)), ) def test_tolerant_fstring_closer_at_expression_entry_depth(self): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-03-15-20-00.gh-issue-156894.Rt9Bx4.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-03-15-20-00.gh-issue-156894.Rt9Bx4.rst new file mode 100644 index 000000000000000..9b279ab27947867 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-03-15-20-00.gh-issue-156894.Rt9Bx4.rst @@ -0,0 +1,2 @@ +Fix the position of syntax errors which cover a range if the line contains +non-ASCII characters before the error. diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index bbd64760a18a664..e99be70ea5e9801 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -9,6 +9,20 @@ /* ############## ERRORS ############## */ +/* Convert a 1-based column in bytes into a 1-based column in characters. + The line is UTF-8 encoded, so it is enough to skip continuation bytes. */ +static int +byte_col_to_char_col(const char *line, int byte_col) +{ + int char_col = 1; + for (int i = 0; i < byte_col - 1; i++) { + if ((line[i] & 0xC0) != 0x80) { + char_col++; + } + } + return char_col; +} + static int _syntaxerror_range(struct tok_state *tok, const char *format, int col_offset, int end_col_offset, @@ -35,9 +49,15 @@ _syntaxerror_range(struct tok_state *tok, const char *format, if (col_offset == -1) { col_offset = (int)PyUnicode_GET_LENGTH(errtext); } + else if (col_offset > 0) { + col_offset = byte_col_to_char_col(tok->line_start, col_offset); + } if (end_col_offset == -1) { end_col_offset = col_offset; } + else if (end_col_offset > 0) { + end_col_offset = byte_col_to_char_col(tok->line_start, end_col_offset); + } Py_ssize_t line_len = strcspn(tok->line_start, "\n"); if (line_len != tok->cur - tok->line_start) { _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
