https://github.com/python/cpython/commit/e66bec09b722b8955d34f51f2c4b63793d59f9cf
commit: e66bec09b722b8955d34f51f2c4b63793d59f9cf
branch: main
author: Shamil <[email protected]>
committer: pablogsal <[email protected]>
date: 2026-09-14T12:13:40Z
summary:
gh-157378: Fix SyntaxError.offset for "Non-UTF-8 code" error (#157412)
files:
A
Misc/NEWS.d/next/Core_and_Builtins/2026-09-13-12-46-10.gh-issue-157378.Rg7Tq2.rst
M Lib/test/test_exceptions.py
M Parser/tokenizer/helpers.c
diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py
index 0c02b38dd3c0a89..750e45bb1644686 100644
--- a/Lib/test/test_exceptions.py
+++ b/Lib/test/test_exceptions.py
@@ -250,6 +250,16 @@ def testSyntaxErrorRange(self):
self.assertEqual(cm.exception.offset, offset)
self.assertEqual(cm.exception.end_offset, end_offset)
+ def testSyntaxErrorNonUTF8Offset(self):
+ # gh-157378: the position was reported one column short for each
+ # multi-byte character preceding the invalid byte on the same line
+ check = self.check
+ check(b'X\x80', 1, 2, 1, 2)
+ check(b'\xc3\xa9X\x80', 1, 3, 1, 3)
+ check(b'\t\xc3\xa9X\x80', 1, 4, 1, 4)
+ check(b'a\xc3\xa9b\x80c', 1, 4, 1, 4)
+ check(b'a\n\xc3\xa9X\x80', 2, 3, 2, 3)
+
def testSyntaxErrorOffset(self):
check = self.check
check('def fact(x):\n\treturn x!\n', 2, 10)
diff --git
a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-13-12-46-10.gh-issue-157378.Rg7Tq2.rst
b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-13-12-46-10.gh-issue-157378.Rg7Tq2.rst
new file mode 100644
index 000000000000000..569642492992d3a
--- /dev/null
+++
b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-13-12-46-10.gh-issue-157378.Rg7Tq2.rst
@@ -0,0 +1,3 @@
+Fix ``SyntaxError.offset`` and ``SyntaxError.end_offset`` for the
+"Non-UTF-8 code starting with ..." error when a non-ASCII character precedes
+the invalid byte on the same line. Patch by Shamil Abdulaev.
diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c
index c803b787d9dae68..d0ada5ac1131a71 100644
--- a/Parser/tokenizer/helpers.c
+++ b/Parser/tokenizer/helpers.c
@@ -329,17 +329,14 @@ _PyTokenizer_ensure_utf8(const char *line, struct
tok_state *tok, int lineno)
const char *badchar = NULL;
const char *c;
int length;
- int col_offset = 0;
const char *line_start = line;
for (c = line; *c; c += length) {
if (!(length = valid_utf8((const unsigned char *)c))) {
badchar = c;
break;
}
- col_offset++;
if (*c == '\n') {
lineno++;
- col_offset = 0;
line_start = c + 1;
}
}
@@ -348,7 +345,8 @@ _PyTokenizer_ensure_utf8(const char *line, struct tok_state
*tok, int lineno)
tok->line_start = _PyLexer_BufferOffset(tok, line_start);
tok->cur = _PyLexer_BufferOffset(tok, badchar);
_PyTokenizer_syntaxerror_known_range(tok,
- col_offset + 1, col_offset + 1,
+ (int)(badchar - line_start) + 1,
+ (int)(badchar - line_start) + 1,
"Non-UTF-8 code starting with '\\x%.2x'"
"%s%V on line %i, "
"but no encoding declared; "
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
