https://github.com/python/cpython/commit/e22d4a0ba7b4cc199fccf259d7e1b9b37a5cc971 commit: e22d4a0ba7b4cc199fccf259d7e1b9b37a5cc971 branch: 3.13 author: Serhiy Storchaka <[email protected]> committer: serhiy-storchaka <[email protected]> date: 2026-09-04T11:43:14Z summary:
[3.13] gh-156894: Fix the position of syntax errors which cover a range (GH-156901) (GH-156928) The callers of _PyTokenizer_syntaxerror_known_range() pass columns in bytes, but SyntaxError.offset and end_offset are columns in characters. (cherry picked from commit 59a691361cb7b1d4fde3b92f98a61490381c62c9) Co-authored-by: Claude Opus 5 (1M context) <[email protected]> files: A Misc/NEWS.d/next/Core_and_Builtins/2026-09-03-15-20-00.gh-issue-156894.Rt9Bx4.rst M Lib/test/test_exceptions.py M Parser/tokenizer/helpers.c diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index e34bd7b83a19b03..f8d06b469b651e2 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -230,6 +230,23 @@ def test_error_offset_continuation_characters(self): check = self.check check('"\\\n"(1 for c in I,\\\n\\', 2, 2) + def testSyntaxErrorRange(self): + # gh-156894: the position was reported in bytes, not in characters, + # for the errors which cover a range + for source, offset, end_offset in [ + ('abcd = 00010', 8, 11), + ('\u03b1\u03b2\u03b3\u03b4 = 00010', 8, 11), + ('a\u0301b\u0308c\u20d7d\u1ab0 = 00010', 12, 15), + ("abcd = ub'a'", 10, 13), + ("\u03b1\u03b2\u03b3\u03b4 = ub'a'", 10, 13), + ("a\u0301b\u0308c\u20d7d\u1ab0 = ub'a'", 14, 17), + ]: + with self.subTest(source=source): + with self.assertRaises(SyntaxError) as cm: + compile(source, '<testcase>', 'exec') + self.assertEqual(cm.exception.offset, offset) + self.assertEqual(cm.exception.end_offset, end_offset) + def testSyntaxErrorOffset(self): check = self.check check('def fact(x):\n\treturn x!\n', 2, 10) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-03-15-20-00.gh-issue-156894.Rt9Bx4.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-03-15-20-00.gh-issue-156894.Rt9Bx4.rst new file mode 100644 index 000000000000000..9b279ab27947867 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-03-15-20-00.gh-issue-156894.Rt9Bx4.rst @@ -0,0 +1,2 @@ +Fix the position of syntax errors which cover a range if the line contains +non-ASCII characters before the error. diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index 976d039f571554d..aad30b90cbf879c 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -7,6 +7,20 @@ /* ############## ERRORS ############## */ +/* Convert a 1-based column in bytes into a 1-based column in characters. + The line is UTF-8 encoded, so it is enough to skip continuation bytes. */ +static int +byte_col_to_char_col(const char *line, int byte_col) +{ + int char_col = 1; + for (int i = 0; i < byte_col - 1; i++) { + if ((line[i] & 0xC0) != 0x80) { + char_col++; + } + } + return char_col; +} + static int _syntaxerror_range(struct tok_state *tok, const char *format, int col_offset, int end_col_offset, @@ -33,9 +47,15 @@ _syntaxerror_range(struct tok_state *tok, const char *format, if (col_offset == -1) { col_offset = (int)PyUnicode_GET_LENGTH(errtext); } + else if (col_offset > 0) { + col_offset = byte_col_to_char_col(tok->line_start, col_offset); + } if (end_col_offset == -1) { end_col_offset = col_offset; } + else if (end_col_offset > 0) { + end_col_offset = byte_col_to_char_col(tok->line_start, end_col_offset); + } Py_ssize_t line_len = strcspn(tok->line_start, "\n"); if (line_len != tok->cur - tok->line_start) { _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
