https://github.com/python/cpython/commit/f1a5f68e3761e010ccd4dda1342500c5ae40bbc4
commit: f1a5f68e3761e010ccd4dda1342500c5ae40bbc4
branch: main
author: Bartosz Sławecki <[email protected]>
committer: serhiy-storchaka <[email protected]>
date: 2026-06-14T12:33:03Z
summary:
gh-151461: Fix encoding-related exception handling in file tokenizer (GH-151462)
files:
A
Misc/NEWS.d/next/Core_and_Builtins/2026-06-14-05-05-15.gh-issue-151461.5q0s88.rst
M Lib/test/test_source_encoding.py
M Parser/pegen.c
M Parser/pegen.h
M Parser/pegen_errors.c
M Parser/tokenizer/helpers.c
M Parser/tokenizer/helpers.h
diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py
index 8ac64b3105708f7..53fffe7cfb56d09 100644
--- a/Lib/test/test_source_encoding.py
+++ b/Lib/test/test_source_encoding.py
@@ -387,8 +387,7 @@ def test_utf8_non_utf8_third_line_error(self):
b'#third\xa4\n'
b'raise RuntimeError\n')
self.check_script_error(src,
- br"'utf-8' codec can't decode byte|"
- br"encoding problem: utf8")
+ br"'utf-8' codec can't decode byte")
def test_crlf(self):
src = (b'print(ascii("""\r\n"""))\n')
@@ -540,6 +539,20 @@ def check_script_error(self, src, expected, lineno=...):
line = line.removeprefix('\ufeff')
self.assertIn(line.encode(), err)
+ def test_coding_spec_unknown_encoding(self):
+ src = (b'# coding: c1252\n'
+ b'print("Hi!")\n')
+ self.check_script_error(src, br"unknown encoding: c1252")
+
+ def test_coding_spec_decode_error(self):
+ src = (b'# coding: shift-jis\n'
+ b'print("\xc4\x85")\n')
+ self.check_script_error(src, br"'shift_jis' codec can't decode byte")
+
+ def test_coding_spec_non_text_encoding(self):
+ src = (b'# coding: hex_codec\n'
+ b'print("eggs")\n')
+ self.check_script_error(src, br"'hex_codec' is not a text encoding")
if __name__ == "__main__":
diff --git
a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-14-05-05-15.gh-issue-151461.5q0s88.rst
b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-14-05-05-15.gh-issue-151461.5q0s88.rst
new file mode 100644
index 000000000000000..d76a9bc95278bcb
--- /dev/null
+++
b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-14-05-05-15.gh-issue-151461.5q0s88.rst
@@ -0,0 +1,3 @@
+Fix direct execution of files with invalid source encodings to report the
+underlying codec lookup or decoding error instead of the generic
+``SyntaxError: encoding problem`` message. Patch by Bartosz Sławecki.
diff --git a/Parser/pegen.c b/Parser/pegen.c
index 569f5afb3120085..bb222b50fc095f2 100644
--- a/Parser/pegen.c
+++ b/Parser/pegen.c
@@ -9,6 +9,7 @@
#include "lexer/lexer.h"
#include "tokenizer/tokenizer.h"
+#include "tokenizer/helpers.h"
#include "pegen.h"
// Internal parser functions
@@ -993,7 +994,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int
start_rule, PyObject *filena
struct tok_state *tok = _PyTokenizer_FromFile(fp, enc, ps1, ps2);
if (tok == NULL) {
if (PyErr_Occurred()) {
- _PyPegen_raise_tokenizer_init_error(filename_ob);
+ _PyTokenizer_raise_init_error(filename_ob);
return NULL;
}
return NULL;
@@ -1051,7 +1052,7 @@ _PyPegen_run_parser_from_string(const char *str, int
start_rule, PyObject *filen
}
if (tok == NULL) {
if (PyErr_Occurred()) {
- _PyPegen_raise_tokenizer_init_error(filename_ob);
+ _PyTokenizer_raise_init_error(filename_ob);
}
return NULL;
}
diff --git a/Parser/pegen.h b/Parser/pegen.h
index 85c9ada765d9bd4..5c461e82a7f0fa7 100644
--- a/Parser/pegen.h
+++ b/Parser/pegen.h
@@ -174,7 +174,6 @@ typedef enum {
} TARGETS_TYPE;
int _Pypegen_raise_decode_error(Parser *p);
-void _PyPegen_raise_tokenizer_init_error(PyObject *filename);
int _Pypegen_tokenizer_error(Parser *p);
void *_PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const
char *errmsg, ...);
void *_PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c
index 312699415efd9af..b13e1c079220a92 100644
--- a/Parser/pegen_errors.c
+++ b/Parser/pegen_errors.c
@@ -10,53 +10,6 @@
// TOKENIZER ERRORS
-void
-_PyPegen_raise_tokenizer_init_error(PyObject *filename)
-{
- if (!(PyErr_ExceptionMatches(PyExc_LookupError)
- || PyErr_ExceptionMatches(PyExc_SyntaxError)
- || PyErr_ExceptionMatches(PyExc_ValueError)
- || PyErr_ExceptionMatches(PyExc_UnicodeDecodeError))) {
- return;
- }
- PyObject *errstr = NULL;
- PyObject *tuple = NULL;
- PyObject *type;
- PyObject *value;
- PyObject *tback;
- PyErr_Fetch(&type, &value, &tback);
- if (PyErr_GivenExceptionMatches(value, PyExc_SyntaxError)) {
- if (PyObject_SetAttr(value, &_Py_ID(filename), filename)) {
- goto error;
- }
- PyErr_Restore(type, value, tback);
- return;
- }
- errstr = PyObject_Str(value);
- if (!errstr) {
- goto error;
- }
-
- PyObject *tmp = Py_BuildValue("(OiiO)", filename, 0, -1, Py_None);
- if (!tmp) {
- goto error;
- }
-
- tuple = _PyTuple_FromPair(errstr, tmp);
- Py_DECREF(tmp);
- if (!tuple) {
- goto error;
- }
- PyErr_SetObject(PyExc_SyntaxError, tuple);
-
-error:
- Py_XDECREF(type);
- Py_XDECREF(value);
- Py_XDECREF(tback);
- Py_XDECREF(errstr);
- Py_XDECREF(tuple);
-}
-
static inline void
raise_unclosed_parentheses_error(Parser *p) {
int error_lineno = p->tok->parenlinenostack[p->tok->level-1];
diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c
index c69e66d0ab9b7a8..62b0971d418c396 100644
--- a/Parser/tokenizer/helpers.c
+++ b/Parser/tokenizer/helpers.c
@@ -1,6 +1,8 @@
#include "Python.h"
#include "errcode.h"
+#include "pycore_runtime.h" // _Py_ID()
#include "pycore_token.h"
+#include "pycore_tuple.h" // _PyTuple_FromPair
#include "../lexer/state.h"
@@ -149,6 +151,53 @@ _PyTokenizer_warn_invalid_escape_sequence(struct tok_state
*tok, int first_inval
return 0;
}
+void
+_PyTokenizer_raise_init_error(PyObject *filename)
+{
+ if (!(PyErr_ExceptionMatches(PyExc_LookupError)
+ || PyErr_ExceptionMatches(PyExc_SyntaxError)
+ || PyErr_ExceptionMatches(PyExc_ValueError)
+ || PyErr_ExceptionMatches(PyExc_UnicodeDecodeError))) {
+ return;
+ }
+ PyObject *errstr = NULL;
+ PyObject *tuple = NULL;
+ PyObject *type;
+ PyObject *value;
+ PyObject *tback;
+ PyErr_Fetch(&type, &value, &tback);
+ if (PyErr_GivenExceptionMatches(value, PyExc_SyntaxError)) {
+ if (PyObject_SetAttr(value, &_Py_ID(filename), filename)) {
+ goto error;
+ }
+ PyErr_Restore(type, value, tback);
+ return;
+ }
+ errstr = PyObject_Str(value);
+ if (!errstr) {
+ goto error;
+ }
+
+ PyObject *tmp = Py_BuildValue("(OiiO)", filename, 0, -1, Py_None);
+ if (!tmp) {
+ goto error;
+ }
+
+ tuple = _PyTuple_FromPair(errstr, tmp);
+ Py_DECREF(tmp);
+ if (!tuple) {
+ goto error;
+ }
+ PyErr_SetObject(PyExc_SyntaxError, tuple);
+
+error:
+ Py_XDECREF(type);
+ Py_XDECREF(value);
+ Py_XDECREF(tback);
+ Py_XDECREF(errstr);
+ Py_XDECREF(tuple);
+}
+
int
_PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char
*format, ...)
{
@@ -418,8 +467,8 @@ _PyTokenizer_check_coding_spec(const char* line, Py_ssize_t
size, struct tok_sta
if (tok->encoding == NULL) {
assert(tok->decoding_readline == NULL);
if (strcmp(cs, "utf-8") != 0 && !set_readline(tok, cs)) {
+ _PyTokenizer_raise_init_error(tok->filename);
_PyTokenizer_error_ret(tok);
- PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs);
PyMem_Free(cs);
return 0;
}
diff --git a/Parser/tokenizer/helpers.h b/Parser/tokenizer/helpers.h
index 98f6445d5a3b40e..34303999a60aff7 100644
--- a/Parser/tokenizer/helpers.h
+++ b/Parser/tokenizer/helpers.h
@@ -15,6 +15,7 @@ int _PyTokenizer_indenterror(struct tok_state *tok);
int _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int
first_invalid_escape_char);
int _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const
char *format, ...);
char *_PyTokenizer_error_ret(struct tok_state *tok);
+void _PyTokenizer_raise_init_error(PyObject *filename);
char *_PyTokenizer_new_string(const char *s, Py_ssize_t len, struct tok_state
*tok);
char *_PyTokenizer_translate_newlines(const char *s, int exec_input, int
preserve_crlf, struct tok_state *tok);
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
