https://github.com/python/cpython/commit/113ad29d192c683c77c8a3a0cf3881f0628c6862 commit: 113ad29d192c683c77c8a3a0cf3881f0628c6862 branch: 3.14 author: Serhiy Storchaka <[email protected]> committer: serhiy-storchaka <[email protected]> date: 2026-09-04T11:44:46Z summary:
[3.14] gh-156894: Fix the position of syntax errors which cover a range (GH-156901) (GH-156927) The callers of _PyTokenizer_syntaxerror_known_range() pass columns in bytes, but SyntaxError.offset and end_offset are columns in characters. (cherry picked from commit 59a691361cb7b1d4fde3b92f98a61490381c62c9) Co-authored-by: Claude Opus 5 (1M context) <[email protected]> files: A Misc/NEWS.d/next/Core_and_Builtins/2026-09-03-15-20-00.gh-issue-156894.Rt9Bx4.rst M Lib/test/test_exceptions.py M Parser/tokenizer/helpers.c diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index d8f52332005f4ca..49179d9f1eb4ffd 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -232,6 +232,23 @@ def test_error_offset_continuation_characters(self): check = self.check check('"\\\n"(1 for c in I,\\\n\\', 2, 2) + def testSyntaxErrorRange(self): + # gh-156894: the position was reported in bytes, not in characters, + # for the errors which cover a range + for source, offset, end_offset in [ + ('abcd = 00010', 8, 11), + ('\u03b1\u03b2\u03b3\u03b4 = 00010', 8, 11), + ('a\u0301b\u0308c\u20d7d\u1ab0 = 00010', 12, 15), + ("abcd = ub'a'", 8, 10), + ("\u03b1\u03b2\u03b3\u03b4 = ub'a'", 8, 10), + ("a\u0301b\u0308c\u20d7d\u1ab0 = ub'a'", 12, 14), + ]: + with self.subTest(source=source): + with self.assertRaises(SyntaxError) as cm: + compile(source, '<testcase>', 'exec') + self.assertEqual(cm.exception.offset, offset) + self.assertEqual(cm.exception.end_offset, end_offset) + def testSyntaxErrorOffset(self): check = self.check check('def fact(x):\n\treturn x!\n', 2, 10) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-03-15-20-00.gh-issue-156894.Rt9Bx4.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-03-15-20-00.gh-issue-156894.Rt9Bx4.rst new file mode 100644 index 000000000000000..9b279ab27947867 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-03-15-20-00.gh-issue-156894.Rt9Bx4.rst @@ -0,0 +1,2 @@ +Fix the position of syntax errors which cover a range if the line contains +non-ASCII characters before the error. diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index 92d439e90363c8e..8d72dd96839e26d 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -8,6 +8,20 @@ /* ############## ERRORS ############## */ +/* Convert a 1-based column in bytes into a 1-based column in characters. + The line is UTF-8 encoded, so it is enough to skip continuation bytes. */ +static int +byte_col_to_char_col(const char *line, int byte_col) +{ + int char_col = 1; + for (int i = 0; i < byte_col - 1; i++) { + if ((line[i] & 0xC0) != 0x80) { + char_col++; + } + } + return char_col; +} + static int _syntaxerror_range(struct tok_state *tok, const char *format, int col_offset, int end_col_offset, @@ -34,9 +48,15 @@ _syntaxerror_range(struct tok_state *tok, const char *format, if (col_offset == -1) { col_offset = (int)PyUnicode_GET_LENGTH(errtext); } + else if (col_offset > 0) { + col_offset = byte_col_to_char_col(tok->line_start, col_offset); + } if (end_col_offset == -1) { end_col_offset = col_offset; } + else if (end_col_offset > 0) { + end_col_offset = byte_col_to_char_col(tok->line_start, end_col_offset); + } Py_ssize_t line_len = strcspn(tok->line_start, "\n"); if (line_len != tok->cur - tok->line_start) { _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
