https://github.com/python/cpython/commit/60e105c1c11ecca1680d03c38aa06bcc77a28714
commit: 60e105c1c11ecca1680d03c38aa06bcc77a28714
branch: main
author: Sam Gross <[email protected]>
committer: colesbury <[email protected]>
date: 2024-03-19T14:40:20-04:00
summary:
gh-113964: Don't prevent new threads until all non-daemon threads exit (#116677)
Starting in Python 3.12, we prevented calling fork() and starting new threads
during interpreter finalization (shutdown). This has led to a number of
regressions and flaky tests. We should not prevent starting new threads
(or `fork()`) until all non-daemon threads exit and finalization starts in
earnest.
This changes the checks to use `_PyInterpreterState_GetFinalizing(interp)`,
which is set immediately before terminating non-daemon threads.
files:
A Misc/NEWS.d/next/Core and
Builtins/2024-03-12-20-31-57.gh-issue-113964.bJppzg.rst
M Lib/test/test_os.py
M Lib/test/test_subprocess.py
M Lib/test/test_threading.py
M Modules/_posixsubprocess.c
M Modules/_threadmodule.c
M Modules/posixmodule.c
M Objects/unicodeobject.c
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
index 4c157842d95523..4bf158247fa2ec 100644
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -5357,20 +5357,21 @@ def test_fork_warns_when_non_python_thread_exists(self):
self.assertEqual(err.decode("utf-8"), "")
self.assertEqual(out.decode("utf-8"), "")
- def test_fork_at_exit(self):
+ def test_fork_at_finalization(self):
code = """if 1:
import atexit
import os
- def exit_handler():
- pid = os.fork()
- if pid != 0:
- print("shouldn't be printed")
-
- atexit.register(exit_handler)
+ class AtFinalization:
+ def __del__(self):
+ print("OK")
+ pid = os.fork()
+ if pid != 0:
+ print("shouldn't be printed")
+ at_finalization = AtFinalization()
"""
_, out, err = assert_python_ok("-c", code)
- self.assertEqual(b"", out)
+ self.assertEqual(b"OK\n", out)
self.assertIn(b"can't fork at interpreter shutdown", err)
diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py
index d20b987961ea6f..70452ca94a6a8a 100644
--- a/Lib/test/test_subprocess.py
+++ b/Lib/test/test_subprocess.py
@@ -3398,14 +3398,15 @@ def test_preexec_at_exit(self):
def dummy():
pass
- def exit_handler():
- subprocess.Popen({ZERO_RETURN_CMD}, preexec_fn=dummy)
- print("shouldn't be printed")
-
- atexit.register(exit_handler)
+ class AtFinalization:
+ def __del__(self):
+ print("OK")
+ subprocess.Popen({ZERO_RETURN_CMD}, preexec_fn=dummy)
+ print("shouldn't be printed")
+ at_finalization = AtFinalization()
"""
_, out, err = assert_python_ok("-c", code)
- self.assertEqual(out, b'')
+ self.assertEqual(out.strip(), b"OK")
self.assertIn(b"preexec_fn not supported at interpreter shutdown", err)
@unittest.skipIf(not sysconfig.get_config_var("HAVE_VFORK"),
diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py
index f1dc12944cb6cc..4414d2bb9cdb59 100644
--- a/Lib/test/test_threading.py
+++ b/Lib/test/test_threading.py
@@ -1154,21 +1154,21 @@ def import_threading():
self.assertEqual(out, b'')
self.assertEqual(err, b'')
- def test_start_new_thread_at_exit(self):
+ def test_start_new_thread_at_finalization(self):
code = """if 1:
- import atexit
import _thread
def f():
print("shouldn't be printed")
- def exit_handler():
- _thread.start_new_thread(f, ())
-
- atexit.register(exit_handler)
+ class AtFinalization:
+ def __del__(self):
+ print("OK")
+ _thread.start_new_thread(f, ())
+ at_finalization = AtFinalization()
"""
_, out, err = assert_python_ok("-c", code)
- self.assertEqual(out, b'')
+ self.assertEqual(out.strip(), b"OK")
self.assertIn(b"can't create new thread at interpreter shutdown", err)
class ThreadJoinOnShutdown(BaseTestCase):
@@ -1297,6 +1297,30 @@ def main():
rc, out, err = assert_python_ok('-c', script)
self.assertFalse(err)
+ def test_thread_from_thread(self):
+ script = """if True:
+ import threading
+ import time
+
+ def thread2():
+ time.sleep(0.05)
+ print("OK")
+
+ def thread1():
+ time.sleep(0.05)
+ t2 = threading.Thread(target=thread2)
+ t2.start()
+
+ t = threading.Thread(target=thread1)
+ t.start()
+ # do not join() -- the interpreter waits for non-daemon threads to
+ # finish.
+ """
+ rc, out, err = assert_python_ok('-c', script)
+ self.assertEqual(err, b"")
+ self.assertEqual(out.strip(), b"OK")
+ self.assertEqual(rc, 0)
+
@skip_unless_reliable_fork
def test_reinit_tls_after_fork(self):
# Issue #13817: fork() would deadlock in a multithreaded program with
diff --git a/Misc/NEWS.d/next/Core and
Builtins/2024-03-12-20-31-57.gh-issue-113964.bJppzg.rst b/Misc/NEWS.d/next/Core
and Builtins/2024-03-12-20-31-57.gh-issue-113964.bJppzg.rst
new file mode 100644
index 00000000000000..ab370d4aa1baee
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and
Builtins/2024-03-12-20-31-57.gh-issue-113964.bJppzg.rst
@@ -0,0 +1,2 @@
+Starting new threads and process creation through :func:`os.fork` are now
+only prevented once all non-daemon threads exit.
diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c
index bcbbe70680b8e7..b160cd78177a17 100644
--- a/Modules/_posixsubprocess.c
+++ b/Modules/_posixsubprocess.c
@@ -1031,7 +1031,9 @@ subprocess_fork_exec_impl(PyObject *module, PyObject
*process_args,
Py_ssize_t fds_to_keep_len = PyTuple_GET_SIZE(py_fds_to_keep);
PyInterpreterState *interp = _PyInterpreterState_GET();
- if ((preexec_fn != Py_None) && interp->finalizing) {
+ if ((preexec_fn != Py_None) &&
+ _PyInterpreterState_GetFinalizing(interp) != NULL)
+ {
PyErr_SetString(PyExc_PythonFinalizationError,
"preexec_fn not supported at interpreter shutdown");
return NULL;
diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c
index 6889e8f6e12126..4912cd776ef5ae 100644
--- a/Modules/_threadmodule.c
+++ b/Modules/_threadmodule.c
@@ -1729,7 +1729,7 @@ do_start_new_thread(thread_module_state *state, PyObject
*func, PyObject *args,
"thread is not supported for isolated
subinterpreters");
return -1;
}
- if (interp->finalizing) {
+ if (_PyInterpreterState_GetFinalizing(interp) != NULL) {
PyErr_SetString(PyExc_PythonFinalizationError,
"can't create new thread at interpreter shutdown");
return -1;
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index 7b2d3661ee5546..2498b61d6412d5 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -7842,7 +7842,7 @@ os_fork1_impl(PyObject *module)
pid_t pid;
PyInterpreterState *interp = _PyInterpreterState_GET();
- if (interp->finalizing) {
+ if (_PyInterpreterState_GetFinalizing(interp) != NULL) {
PyErr_SetString(PyExc_PythonFinalizationError,
"can't fork at interpreter shutdown");
return NULL;
@@ -7886,7 +7886,7 @@ os_fork_impl(PyObject *module)
{
pid_t pid;
PyInterpreterState *interp = _PyInterpreterState_GET();
- if (interp->finalizing) {
+ if (_PyInterpreterState_GetFinalizing(interp) != NULL) {
PyErr_SetString(PyExc_PythonFinalizationError,
"can't fork at interpreter shutdown");
return NULL;
@@ -8719,7 +8719,7 @@ os_forkpty_impl(PyObject *module)
pid_t pid;
PyInterpreterState *interp = _PyInterpreterState_GET();
- if (interp->finalizing) {
+ if (_PyInterpreterState_GetFinalizing(interp) != NULL) {
PyErr_SetString(PyExc_PythonFinalizationError,
"can't fork at interpreter shutdown");
return NULL;
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index c8f647a7a71135..e412af5f797e7a 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -505,7 +505,7 @@ unicode_check_encoding_errors(const char *encoding, const
char *errors)
/* Disable checks during Python finalization. For example, it allows to
call _PyObject_Dump() during finalization for debugging purpose. */
- if (interp->finalizing) {
+ if (_PyInterpreterState_GetFinalizing(interp) != NULL) {
return 0;
}
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]
