https://github.com/python/cpython/commit/02d4c3b4347da57ba086f2ab7cdf6c0f56c9deeb commit: 02d4c3b4347da57ba086f2ab7cdf6c0f56c9deeb branch: 3.14 author: Miss Islington (bot) <[email protected]> committer: pablogsal <[email protected]> date: 2026-09-14T11:09:26-04:00 summary:
[3.14] gh-157378: Fix SyntaxError.offset for "Non-UTF-8 code" error (GH-157412) (#157489) gh-157378: Fix SyntaxError.offset for "Non-UTF-8 code" error (GH-157412) (cherry picked from commit e66bec09b722b8955d34f51f2c4b63793d59f9cf) Co-authored-by: Shamil <[email protected]> files: A Misc/NEWS.d/next/Core_and_Builtins/2026-09-13-12-46-10.gh-issue-157378.Rg7Tq2.rst M Lib/test/test_exceptions.py M Parser/tokenizer/helpers.c diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 49179d9f1eb4ffd..5a5cc6b3e7ddce4 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -249,6 +249,16 @@ def testSyntaxErrorRange(self): self.assertEqual(cm.exception.offset, offset) self.assertEqual(cm.exception.end_offset, end_offset) + def testSyntaxErrorNonUTF8Offset(self): + # gh-157378: the position was reported one column short for each + # multi-byte character preceding the invalid byte on the same line + check = self.check + check(b'X\x80', 1, 2, 1, 2) + check(b'\xc3\xa9X\x80', 1, 3, 1, 3) + check(b'\t\xc3\xa9X\x80', 1, 4, 1, 4) + check(b'a\xc3\xa9b\x80c', 1, 4, 1, 4) + check(b'a\n\xc3\xa9X\x80', 2, 3, 2, 3) + def testSyntaxErrorOffset(self): check = self.check check('def fact(x):\n\treturn x!\n', 2, 10) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-13-12-46-10.gh-issue-157378.Rg7Tq2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-13-12-46-10.gh-issue-157378.Rg7Tq2.rst new file mode 100644 index 000000000000000..569642492992d3a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-13-12-46-10.gh-issue-157378.Rg7Tq2.rst @@ -0,0 +1,3 @@ +Fix ``SyntaxError.offset`` and ``SyntaxError.end_offset`` for the +"Non-UTF-8 code starting with ..." error when a non-ASCII character precedes +the invalid byte on the same line. Patch by Shamil Abdulaev. diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index 8d72dd96839e26d..e046f179d3817b7 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -578,17 +578,14 @@ _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno) const char *badchar = NULL; const char *c; int length; - int col_offset = 0; const char *line_start = line; for (c = line; *c; c += length) { if (!(length = valid_utf8((const unsigned char *)c))) { badchar = c; break; } - col_offset++; if (*c == '\n') { lineno++; - col_offset = 0; line_start = c + 1; } } @@ -597,7 +594,8 @@ _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno) tok->line_start = line_start; tok->cur = (char *)badchar; _PyTokenizer_syntaxerror_known_range(tok, - col_offset + 1, col_offset + 1, + (int)(badchar - line_start) + 1, + (int)(badchar - line_start) + 1, "Non-UTF-8 code starting with '\\x%.2x'" "%s%V on line %i, " "but no encoding declared; " _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
