[Python-checkins] gh-117431: Argument Clinic: copy forced text signature when cloning (#117591)

2024-04-10 Thread erlend-aasland
https://github.com/python/cpython/commit/0d42ac9474f857633d00b414c0715f4efa73f1ca
commit: 0d42ac9474f857633d00b414c0715f4efa73f1ca
branch: main
author: Erlend E. Aasland 
committer: erlend-aasland 
date: 2024-04-10T10:12:05+02:00
summary:

gh-117431: Argument Clinic: copy forced text signature when cloning (#117591)

files:
M Lib/test/test_clinic.py
M Objects/clinic/unicodeobject.c.h
M Tools/clinic/libclinic/dsl_parser.py
M Tools/clinic/libclinic/function.py

diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py
index a5887cdb56e3ca..e3ba3d943216de 100644
--- a/Lib/test/test_clinic.py
+++ b/Lib/test/test_clinic.py
@@ -5,7 +5,7 @@
 from functools import partial
 from test import support, test_tools
 from test.support import os_helper
-from test.support.os_helper import TESTFN, unlink
+from test.support.os_helper import TESTFN, unlink, rmtree
 from textwrap import dedent
 from unittest import TestCase
 import inspect
@@ -662,6 +662,61 @@ class C "void *" ""
 err = "Illegal C basename: '.illegal.'"
 self.expect_failure(block, err, lineno=7)
 
+def test_cloned_forced_text_signature(self):
+block = dedent("""
+/*[clinic input]
+@text_signature "($module, a[, b])"
+src
+a: object
+param a
+b: object = NULL
+/
+
+docstring
+[clinic start generated code]*/
+
+/*[clinic input]
+dst = src
+[clinic start generated code]*/
+""")
+self.clinic.parse(block)
+self.addCleanup(rmtree, "clinic")
+funcs = self.clinic.functions
+self.assertEqual(len(funcs), 2)
+
+src_docstring_lines = funcs[0].docstring.split("\n")
+dst_docstring_lines = funcs[1].docstring.split("\n")
+
+# Signatures are copied.
+self.assertEqual(src_docstring_lines[0], "src($module, a[, b])")
+self.assertEqual(dst_docstring_lines[0], "dst($module, a[, b])")
+
+# Param docstrings are copied.
+self.assertIn("param a", src_docstring_lines)
+self.assertIn("param a", dst_docstring_lines)
+
+# Docstrings are not copied.
+self.assertIn("docstring", src_docstring_lines)
+self.assertNotIn("docstring", dst_docstring_lines)
+
+def test_cloned_forced_text_signature_illegal(self):
+block = """
+/*[clinic input]
+@text_signature "($module, a[, b])"
+src
+a: object
+b: object = NULL
+/
+[clinic start generated code]*/
+
+/*[clinic input]
+@text_signature "($module, a_override[, b])"
+dst = src
+[clinic start generated code]*/
+"""
+err = "Cannot use @text_signature when cloning a function"
+self.expect_failure(block, err, lineno=11)
+
 
 class ParseFileUnitTest(TestCase):
 def expect_parsing_failure(
diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h
index 01c40b90d9b4b8..78e14b0021d006 100644
--- a/Objects/clinic/unicodeobject.c.h
+++ b/Objects/clinic/unicodeobject.c.h
@@ -357,7 +357,7 @@ unicode_expandtabs(PyObject *self, PyObject *const *args, 
Py_ssize_t nargs, PyOb
 }
 
 PyDoc_STRVAR(unicode_find__doc__,
-"find($self, sub, start=None, end=None, /)\n"
+"find($self, sub[, start[, end]], /)\n"
 "--\n"
 "\n"
 "Return the lowest index in S where substring sub is found, such that sub is 
contained within S[start:end].\n"
@@ -413,7 +413,7 @@ unicode_find(PyObject *str, PyObject *const *args, 
Py_ssize_t nargs)
 }
 
 PyDoc_STRVAR(unicode_index__doc__,
-"index($self, sub, start=None, end=None, /)\n"
+"index($self, sub[, start[, end]], /)\n"
 "--\n"
 "\n"
 "Return the lowest index in S where substring sub is found, such that sub is 
contained within S[start:end].\n"
@@ -1060,7 +1060,7 @@ unicode_removesuffix(PyObject *self, PyObject *arg)
 }
 
 PyDoc_STRVAR(unicode_rfind__doc__,
-"rfind($self, sub, start=None, end=None, /)\n"
+"rfind($self, sub[, start[, end]], /)\n"
 "--\n"
 "\n"
 "Return the highest index in S where substring sub is found, such that sub is 
contained within S[start:end].\n"
@@ -1116,7 +1116,7 @@ unicode_rfind(PyObject *str, PyObject *const *args, 
Py_ssize_t nargs)
 }
 
 PyDoc_STRVAR(unicode_rindex__doc__,
-"rindex($self, sub, start=None, end=None, /)\n"
+"rindex($self, sub[, start[, end]], /)\n"
 "--\n"
 "\n"
 "Return the highest index in S where substring sub is found, such that sub is 
contained within S[start:end].\n"
@@ -1888,4 +1888,4 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject 
*kwargs)
 exit:
 return return_value;
 }
-/*[clinic end generated code: output=3aa49013ffa3fa93 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=9fee62bd337f809b input=a9049054013a1b77]*/
diff --git a/Tools/clinic/libclinic/dsl_parser.py 
b/Tools/clinic/libclinic/dsl_parser.py
index 4c739efe1066e4..9e22d8

[Python-checkins] gh-117686: Improve the performance of ntpath.expanduser() (#117690)

2024-04-10 Thread erlend-aasland
https://github.com/python/cpython/commit/f90ff0367271ea474b4ce3c8e2643cb51d188c18
commit: f90ff0367271ea474b4ce3c8e2643cb51d188c18
branch: main
author: Nice Zombies 
committer: erlend-aasland 
date: 2024-04-10T10:28:48+02:00
summary:

gh-117686: Improve the performance of ntpath.expanduser() (#117690)

Refactor out _get_bothseps() call from the loop.

files:
M Lib/ntpath.py
M Misc/NEWS.d/3.13.0a6.rst

diff --git a/Lib/ntpath.py b/Lib/ntpath.py
index da5231ff2c0931..f5d1a2195dd633 100644
--- a/Lib/ntpath.py
+++ b/Lib/ntpath.py
@@ -368,13 +368,15 @@ def expanduser(path):
 If user or $HOME is unknown, do nothing."""
 path = os.fspath(path)
 if isinstance(path, bytes):
+seps = b'\\/'
 tilde = b'~'
 else:
+seps = '\\/'
 tilde = '~'
 if not path.startswith(tilde):
 return path
 i, n = 1, len(path)
-while i < n and path[i] not in _get_bothseps(path):
+while i < n and path[i] not in seps:
 i += 1
 
 if 'USERPROFILE' in os.environ:
diff --git a/Misc/NEWS.d/3.13.0a6.rst b/Misc/NEWS.d/3.13.0a6.rst
index 52735dba3578b5..06807b396ed5da 100644
--- a/Misc/NEWS.d/3.13.0a6.rst
+++ b/Misc/NEWS.d/3.13.0a6.rst
@@ -4,7 +4,7 @@
 .. release date: 2024-04-09
 .. section: Core and Builtins
 
-Improve performance of :func:`os.path.join`.
+Improve performance of :func:`os.path.join` and :func:`os.path.expanduser`.
 
 ..
 

___
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]


[Python-checkins] gh-117692: Fix `AttributeError` in `DocTestFinder` on wrapped `builtin_or_method` (#117699)

2024-04-10 Thread AlexWaygood
https://github.com/python/cpython/commit/4bb7d121bc0a3fd00a3c72cd915b5dd8fac5616e
commit: 4bb7d121bc0a3fd00a3c72cd915b5dd8fac5616e
branch: main
author: Nikita Sobolev 
committer: AlexWaygood 
date: 2024-04-10T10:52:47+01:00
summary:

gh-117692: Fix `AttributeError` in `DocTestFinder` on wrapped 
`builtin_or_method` (#117699)

Co-authored-by: Alex Waygood 

files:
A Misc/NEWS.d/next/Library/2024-04-09-23-22-21.gh-issue-117692.EciInD.rst
M Lib/doctest.py
M Lib/test/test_doctest/test_doctest.py

diff --git a/Lib/doctest.py b/Lib/doctest.py
index fc0da590018b40..4e362cbb9c9d6b 100644
--- a/Lib/doctest.py
+++ b/Lib/doctest.py
@@ -1140,7 +1140,14 @@ def _find_lineno(self, obj, source_lines):
 obj = obj.fget
 if inspect.isfunction(obj) and getattr(obj, '__doc__', None):
 # We don't use `docstring` var here, because `obj` can be changed.
-obj = inspect.unwrap(obj).__code__
+obj = inspect.unwrap(obj)
+try:
+obj = obj.__code__
+except AttributeError:
+# Functions implemented in C don't necessarily
+# have a __code__ attribute.
+# If there's no code, there's no lineno
+return None
 if inspect.istraceback(obj): obj = obj.tb_frame
 if inspect.isframe(obj): obj = obj.f_code
 if inspect.iscode(obj):
diff --git a/Lib/test/test_doctest/test_doctest.py 
b/Lib/test/test_doctest/test_doctest.py
index 0a2a016fff13e5..f71d62cc174d6b 100644
--- a/Lib/test/test_doctest/test_doctest.py
+++ b/Lib/test/test_doctest/test_doctest.py
@@ -2553,6 +2553,20 @@ def test_look_in_unwrapped():
 'one other test'
 """
 
+@doctest_skip_if(support.check_impl_detail(cpython=False))
+def test_wrapped_c_func():
+"""
+# https://github.com/python/cpython/issues/117692
+>>> import binascii
+>>> from test.test_doctest.decorator_mod import decorator
+
+>>> c_func_wrapped = decorator(binascii.b2a_hex)
+>>> tests = doctest.DocTestFinder(exclude_empty=False).find(c_func_wrapped)
+>>> for test in tests:
+...print(test.lineno, test.name)
+None b2a_hex
+"""
+
 def test_unittest_reportflags():
 """Default unittest reporting flags can be set to control reporting
 
diff --git 
a/Misc/NEWS.d/next/Library/2024-04-09-23-22-21.gh-issue-117692.EciInD.rst 
b/Misc/NEWS.d/next/Library/2024-04-09-23-22-21.gh-issue-117692.EciInD.rst
new file mode 100644
index 00..98a6e125c440ef
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-04-09-23-22-21.gh-issue-117692.EciInD.rst
@@ -0,0 +1,2 @@
+Fixes a bug when :class:`doctest.DocTestFinder` was failing on wrapped
+``builtin_function_or_method``.

___
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]


[Python-checkins] gh-117142: Port _ctypes to multi-phase init (GH-117181)

2024-04-10 Thread encukou
https://github.com/python/cpython/commit/ef4118222b6d5f4532a2f1e234ba03955348d2a1
commit: ef4118222b6d5f4532a2f1e234ba03955348d2a1
branch: main
author: neonene <[email protected]>
committer: encukou 
date: 2024-04-10T11:00:01Z
summary:

gh-117142: Port _ctypes to multi-phase init (GH-117181)

files:
A Misc/NEWS.d/next/Library/2024-03-29-12-21-40.gh-issue-117142.U0agfh.rst
A Modules/_ctypes/clinic/_ctypes.c.h
M Lib/test/test_ctypes/test_refcounts.py
M Modules/_ctypes/_ctypes.c
M Modules/_ctypes/callbacks.c
M Modules/_ctypes/callproc.c
M Modules/_ctypes/cfield.c
M Modules/_ctypes/ctypes.h
M Modules/_ctypes/stgdict.c
M Tools/c-analyzer/cpython/ignored.tsv

diff --git a/Lib/test/test_ctypes/test_refcounts.py 
b/Lib/test/test_ctypes/test_refcounts.py
index e6427d4a295b15..012722d8486218 100644
--- a/Lib/test/test_ctypes/test_refcounts.py
+++ b/Lib/test/test_ctypes/test_refcounts.py
@@ -4,6 +4,7 @@
 import unittest
 from test import support
 from test.support import import_helper
+from test.support import script_helper
 _ctypes_test = import_helper.import_module("_ctypes_test")
 
 
@@ -110,5 +111,18 @@ def func():
 func()
 
 
+class ModuleIsolationTest(unittest.TestCase):
+def test_finalize(self):
+# check if gc_decref() succeeds
+script = (
+"import ctypes;"
+"import sys;"
+"del sys.modules['_ctypes'];"
+"import _ctypes;"
+"exit()"
+)
+script_helper.assert_python_ok("-c", script)
+
+
 if __name__ == '__main__':
 unittest.main()
diff --git 
a/Misc/NEWS.d/next/Library/2024-03-29-12-21-40.gh-issue-117142.U0agfh.rst 
b/Misc/NEWS.d/next/Library/2024-03-29-12-21-40.gh-issue-117142.U0agfh.rst
new file mode 100644
index 00..36810bd815c502
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-03-29-12-21-40.gh-issue-117142.U0agfh.rst
@@ -0,0 +1 @@
+Convert :mod:`!_ctypes` to multi-phase initialisation (:pep:`489`).
diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c
index 631f82879311bf..3cb0b24668eb2a 100644
--- a/Modules/_ctypes/_ctypes.c
+++ b/Modules/_ctypes/_ctypes.c
@@ -126,8 +126,16 @@ bytes(cdata)
 
 #include "pycore_long.h"  // _PyLong_GetZero()
 
-ctypes_state global_state = {0};
+/*[clinic input]
+module _ctypes
+[clinic start generated code]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=476a19c49b31a75c]*/
 
+#define clinic_state() (get_module_state_by_class(cls))
+#define clinic_state_sub() (get_module_state_by_class(cls->tp_base))
+#include "clinic/_ctypes.c.h"
+#undef clinic_state
+#undef clinic_state_sub
 
 //
 
@@ -438,10 +446,15 @@ static PyType_Spec structparam_spec = {
   CType_Type - a base metaclass. Its instances (classes) have a StgInfo.
   */
 
+/*[clinic input]
+class _ctypes.CType_Type "PyObject *" "clinic_state()->CType_Type"
+[clinic start generated code]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=8389fc5b74a84f2a]*/
+
 static int
 CType_Type_traverse(PyObject *self, visitproc visit, void *arg)
 {
-ctypes_state *st = GLOBAL_STATE();
+ctypes_state *st = get_module_state_by_def_final(Py_TYPE(self));
 if (st && st->PyCType_Type) {
 StgInfo *info;
 if (PyStgInfo_FromType(st, self, &info) < 0) {
@@ -475,7 +488,7 @@ ctype_clear_stginfo(StgInfo *info)
 static int
 CType_Type_clear(PyObject *self)
 {
-ctypes_state *st = GLOBAL_STATE();
+ctypes_state *st = get_module_state_by_def_final(Py_TYPE(self));
 if (st && st->PyCType_Type) {
 StgInfo *info;
 if (PyStgInfo_FromType(st, self, &info) < 0) {
@@ -491,8 +504,7 @@ CType_Type_clear(PyObject *self)
 static void
 CType_Type_dealloc(PyObject *self)
 {
-ctypes_state *st = GLOBAL_STATE();
-
+ctypes_state *st = get_module_state_by_def_final(Py_TYPE(self));
 if (st && st->PyCType_Type) {
 StgInfo *info;
 if (PyStgInfo_FromType(st, self, &info) < 0) {
@@ -508,19 +520,27 @@ CType_Type_dealloc(PyObject *self)
 ctype_clear_stginfo(info);
 }
 }
-
 PyTypeObject *tp = Py_TYPE(self);
 PyType_Type.tp_dealloc(self);
 Py_DECREF(tp);
 }
 
+/*[clinic input]
+_ctypes.CType_Type.__sizeof__
+
+cls: defining_class
+/
+Return memory consumption of the type object.
+[clinic start generated code]*/
+
 static PyObject *
-CType_Type_sizeof(PyObject *self)
+_ctypes_CType_Type___sizeof___impl(PyObject *self, PyTypeObject *cls)
+/*[clinic end generated code: output=c68c235be84d03f3 input=d064433b6110d1ce]*/
 {
 Py_ssize_t size = Py_TYPE(self)->tp_basicsize;
 size += Py_TYPE(self)->tp_itemsize * Py_SIZE(self);
 
-ctypes_state *st = GLOBAL_STATE();
+ctypes_state *st = get_module_state_by_class(cls);
 StgInfo *info;
 if (PyStgInfo_FromType(st, self, &info) < 0) {
 return NULL;
@@ -543,8 +563,7 @@ CType_Type_repeat(PyObject *self, Py_ssize_t length);
 
 
 sta

[Python-checkins] [3.12] gh-117692: Fix `AttributeError` in `DocTestFinder` on wrapped `builtin_or_method` (GH-117699) (#117708)

2024-04-10 Thread AlexWaygood
https://github.com/python/cpython/commit/653ed76442d2988e587f4da1fc1cf1bd2bb51fbb
commit: 653ed76442d2988e587f4da1fc1cf1bd2bb51fbb
branch: 3.12
author: Miss Islington (bot) <[email protected]>
committer: AlexWaygood 
date: 2024-04-10T14:17:15Z
summary:

[3.12] gh-117692: Fix `AttributeError` in `DocTestFinder` on wrapped 
`builtin_or_method` (GH-117699) (#117708)

* gh-117692: Fix `AttributeError` in `DocTestFinder` on wrapped 
`builtin_or_method` (GH-117699)
(cherry picked from commit 4bb7d121bc0a3fd00a3c72cd915b5dd8fac5616e)

Co-authored-by: Nikita Sobolev 
Co-authored-by: Alex Waygood 

files:
A Misc/NEWS.d/next/Library/2024-04-09-23-22-21.gh-issue-117692.EciInD.rst
M Lib/doctest.py
M Lib/test/test_doctest/test_doctest.py

diff --git a/Lib/doctest.py b/Lib/doctest.py
index 696bb966549255..d617d96a35a85a 100644
--- a/Lib/doctest.py
+++ b/Lib/doctest.py
@@ -1124,7 +1124,14 @@ def _find_lineno(self, obj, source_lines):
 obj = obj.fget
 if inspect.isfunction(obj) and getattr(obj, '__doc__', None):
 # We don't use `docstring` var here, because `obj` can be changed.
-obj = inspect.unwrap(obj).__code__
+obj = inspect.unwrap(obj)
+try:
+obj = obj.__code__
+except AttributeError:
+# Functions implemented in C don't necessarily
+# have a __code__ attribute.
+# If there's no code, there's no lineno
+return None
 if inspect.istraceback(obj): obj = obj.tb_frame
 if inspect.isframe(obj): obj = obj.f_code
 if inspect.iscode(obj):
diff --git a/Lib/test/test_doctest/test_doctest.py 
b/Lib/test/test_doctest/test_doctest.py
index 9c8a8ba690d557..2722661a2363fe 100644
--- a/Lib/test/test_doctest/test_doctest.py
+++ b/Lib/test/test_doctest/test_doctest.py
@@ -2496,6 +2496,20 @@ def test_look_in_unwrapped():
 'one other test'
 """
 
+if support.check_impl_detail(cpython=True):
+def test_wrapped_c_func():
+"""
+# https://github.com/python/cpython/issues/117692
+>>> import binascii
+>>> from test.test_doctest.decorator_mod import decorator
+
+>>> c_func_wrapped = decorator(binascii.b2a_hex)
+>>> tests = 
doctest.DocTestFinder(exclude_empty=False).find(c_func_wrapped)
+>>> for test in tests:
+...print(test.lineno, test.name)
+None b2a_hex
+"""
+
 def test_unittest_reportflags():
 """Default unittest reporting flags can be set to control reporting
 
diff --git 
a/Misc/NEWS.d/next/Library/2024-04-09-23-22-21.gh-issue-117692.EciInD.rst 
b/Misc/NEWS.d/next/Library/2024-04-09-23-22-21.gh-issue-117692.EciInD.rst
new file mode 100644
index 00..98a6e125c440ef
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-04-09-23-22-21.gh-issue-117692.EciInD.rst
@@ -0,0 +1,2 @@
+Fixes a bug when :class:`doctest.DocTestFinder` was failing on wrapped
+``builtin_function_or_method``.

___
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]


[Python-checkins] gh-112536: Define `_Py_THREAD_SANITIZER` on GCC when TSan is enabled (#117702)

2024-04-10 Thread colesbury
https://github.com/python/cpython/commit/79eec66e3dc277ea6ebad8c0b33756eea6a7ab3b
commit: 79eec66e3dc277ea6ebad8c0b33756eea6a7ab3b
branch: main
author: Sam Gross 
committer: colesbury 
date: 2024-04-10T10:20:05-04:00
summary:

gh-112536: Define `_Py_THREAD_SANITIZER` on GCC when TSan is enabled (#117702)

The `__has_feature(thread_sanitizer)` is a Clang-ism. Although new
versions of GCC implement `__has_feature`, the `defined(__has_feature)`
check still fails on GCC so we don't use that code path.

files:
M Include/pyport.h

diff --git a/Include/pyport.h b/Include/pyport.h
index 9d7ef0061806ad..2ba81a4be42822 100644
--- a/Include/pyport.h
+++ b/Include/pyport.h
@@ -572,6 +572,9 @@ extern "C" {
 #  if defined(__SANITIZE_ADDRESS__)
 #define _Py_ADDRESS_SANITIZER
 #  endif
+#  if defined(__SANITIZE_THREAD__)
+#define _Py_THREAD_SANITIZER
+#  endif
 #endif
 
 

___
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]


[Python-checkins] gh-115142: Skip ``test_capi.test_dict.py`` if ``_testcapi`` and ``_testlimitedcapi`` are not available (GH-117588)

2024-04-10 Thread encukou
https://github.com/python/cpython/commit/dfcae4379f2cc4d352a180f9fef2381570aa9bcb
commit: dfcae4379f2cc4d352a180f9fef2381570aa9bcb
branch: main
author: Kirill Podoprigora 
committer: encukou 
date: 2024-04-10T16:32:57+02:00
summary:

gh-115142: Skip ``test_capi.test_dict.py`` if ``_testcapi`` and 
``_testlimitedcapi`` are not available (GH-117588)

gh-115142: Skip test_dict if _testcapi and _testlimitedcapi is not available

files:
M Lib/test/test_capi/test_dict.py

diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py
index bcc978d224a583..e726e3d813d888 100644
--- a/Lib/test/test_capi/test_dict.py
+++ b/Lib/test/test_capi/test_dict.py
@@ -2,8 +2,11 @@
 from collections import OrderedDict, UserDict
 from types import MappingProxyType
 from test import support
-import _testcapi
-import _testlimitedcapi
+from test.support import import_helper
+
+
+_testcapi = import_helper.import_module("_testcapi")
+_testlimitedcapi = import_helper.import_module("_testlimitedcapi")
 
 
 NULL = None

___
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]


[Python-checkins] [3.12] gh-112536: Define `_Py_THREAD_SANITIZER` on GCC when TSan is enabled (GH-117702) (#117713)

2024-04-10 Thread colesbury
https://github.com/python/cpython/commit/04d07964f2ccb3169ab389e7cc36c7cf59deaf2c
commit: 04d07964f2ccb3169ab389e7cc36c7cf59deaf2c
branch: 3.12
author: Miss Islington (bot) <[email protected]>
committer: colesbury 
date: 2024-04-10T14:38:10Z
summary:

[3.12] gh-112536: Define `_Py_THREAD_SANITIZER` on GCC when TSan is enabled 
(GH-117702) (#117713)

gh-112536: Define `_Py_THREAD_SANITIZER` on GCC when TSan is enabled (GH-117702)

The `__has_feature(thread_sanitizer)` is a Clang-ism. Although new
versions of GCC implement `__has_feature`, the `defined(__has_feature)`
check still fails on GCC so we don't use that code path.
(cherry picked from commit 79eec66e3dc277ea6ebad8c0b33756eea6a7ab3b)

Co-authored-by: Sam Gross 

files:
M Include/pyport.h

diff --git a/Include/pyport.h b/Include/pyport.h
index 30b9c8ebc409f0..e2bac3bf504261 100644
--- a/Include/pyport.h
+++ b/Include/pyport.h
@@ -757,6 +757,9 @@ extern char * _getpty(int *, int, mode_t, int);
 #  if defined(__SANITIZE_ADDRESS__)
 #define _Py_ADDRESS_SANITIZER
 #  endif
+#  if defined(__SANITIZE_THREAD__)
+#define _Py_THREAD_SANITIZER
+#  endif
 #endif
 
 

___
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]


[Python-checkins] gh-117531: Unblock getters after non-immediate queue shutdown (#117532)

2024-04-10 Thread gvanrossum
https://github.com/python/cpython/commit/6bc0b33a91713ee62fd1860d28b19cb620c45971
commit: 6bc0b33a91713ee62fd1860d28b19cb620c45971
branch: main
author: Laurie O 
committer: gvanrossum 
date: 2024-04-10T08:01:42-07:00
summary:

gh-117531: Unblock getters after non-immediate queue shutdown (#117532)

(This is a small tweak of the original gh-104750 which added shutdown.)

files:
M Doc/library/queue.rst
M Lib/queue.py
M Lib/test/test_queue.py

diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst
index f2a6dbf589fd87..fce23313c7de28 100644
--- a/Doc/library/queue.rst
+++ b/Doc/library/queue.rst
@@ -245,8 +245,10 @@ them down.
queue is empty. Set *immediate* to true to make :meth:`~Queue.get` raise
immediately instead.
 
-   All blocked callers of :meth:`~Queue.put` will be unblocked. If *immediate*
-   is true, also unblock callers of :meth:`~Queue.get` and :meth:`~Queue.join`.
+   All blocked callers of :meth:`~Queue.put` and :meth:`~Queue.get` will be
+   unblocked. If *immediate* is true, a task will be marked as done for each
+   remaining item in the queue, which may unblock callers of
+   :meth:`~Queue.join`.
 
.. versionadded:: 3.13
 
diff --git a/Lib/queue.py b/Lib/queue.py
index 387ce5425879a4..25beb46e30d6bd 100644
--- a/Lib/queue.py
+++ b/Lib/queue.py
@@ -239,8 +239,9 @@ def shutdown(self, immediate=False):
 By default, gets will only raise once the queue is empty. Set
 'immediate' to True to make gets raise immediately instead.
 
-All blocked callers of put() will be unblocked, and also get()
-and join() if 'immediate'.
+All blocked callers of put() and get() will be unblocked. If
+'immediate', a task is marked as done for each item remaining in
+the queue, which may unblock callers of join().
 '''
 with self.mutex:
 self.is_shutdown = True
@@ -249,9 +250,10 @@ def shutdown(self, immediate=False):
 self._get()
 if self.unfinished_tasks > 0:
 self.unfinished_tasks -= 1
-self.not_empty.notify_all()
 # release all blocked threads in `join()`
 self.all_tasks_done.notify_all()
+# All getters need to re-check queue-empty to raise ShutDown
+self.not_empty.notify_all()
 self.not_full.notify_all()
 
 # Override these methods to implement other queue organizations
diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py
index c4d10110132393..d5927fbf39142b 100644
--- a/Lib/test/test_queue.py
+++ b/Lib/test/test_queue.py
@@ -636,6 +636,23 @@ def test_shutdown_get_task_done_join(self):
 
 self.assertEqual(results, [True]*len(thrds))
 
+def test_shutdown_pending_get(self):
+def get():
+try:
+results.append(q.get())
+except Exception as e:
+results.append(e)
+
+q = self.type2test()
+results = []
+get_thread = threading.Thread(target=get)
+get_thread.start()
+q.shutdown(immediate=False)
+get_thread.join(timeout=10.0)
+self.assertFalse(get_thread.is_alive())
+self.assertEqual(len(results), 1)
+self.assertIsInstance(results[0], self.queue.ShutDown)
+
 
 class QueueTest(BaseQueueTestMixin):
 

___
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]


[Python-checkins] GH-117546: Fix symlink resolution in `os.path.realpath('loop/../link')` (#117568)

2024-04-10 Thread barneygale
https://github.com/python/cpython/commit/630df37116b1c5b381984c547ef9d23792ceb464
commit: 630df37116b1c5b381984c547ef9d23792ceb464
branch: main
author: Barney Gale 
committer: barneygale 
date: 2024-04-10T18:17:18+01:00
summary:

GH-117546: Fix symlink resolution in `os.path.realpath('loop/../link')` 
(#117568)

Continue resolving symlink targets after encountering a symlink loop, which
matches coreutils `realpath` behaviour.

files:
A Misc/NEWS.d/next/Library/2024-04-05-13-38-53.gh-issue-117546.lWjhHE.rst
M Doc/library/os.path.rst
M Lib/posixpath.py
M Lib/test/test_posixpath.py

diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst
index fcf4b6d68e018c..ebeb3bb50b8b1f 100644
--- a/Doc/library/os.path.rst
+++ b/Doc/library/os.path.rst
@@ -409,9 +409,8 @@ the :mod:`glob` module.)
style names such as ``C:\\PROGRA~1`` to ``C:\\Program Files``.
 
If a path doesn't exist or a symlink loop is encountered, and *strict* is
-   ``True``, :exc:`OSError` is raised. If *strict* is ``False``, the path is
-   resolved as far as possible and any remainder is appended without checking
-   whether it exists.
+   ``True``, :exc:`OSError` is raised. If *strict* is ``False`` these errors
+   are ignored, and so the result might be missing or otherwise inaccessible.
 
.. note::
   This function emulates the operating system's procedure for making a path
diff --git a/Lib/posixpath.py b/Lib/posixpath.py
index 79e65587e66282..8fd49cdc358908 100644
--- a/Lib/posixpath.py
+++ b/Lib/posixpath.py
@@ -431,11 +431,6 @@ def realpath(filename, *, strict=False):
 # the same links.
 seen = {}
 
-# Whether we're calling lstat() and readlink() to resolve symlinks. If we
-# encounter an OSError for a symlink loop in non-strict mode, this is
-# switched off.
-querying = True
-
 while rest:
 name = rest.pop()
 if name is None:
@@ -453,9 +448,6 @@ def realpath(filename, *, strict=False):
 newpath = path + name
 else:
 newpath = path + sep + name
-if not querying:
-path = newpath
-continue
 try:
 st = os.lstat(newpath)
 if not stat.S_ISLNK(st.st_mode):
@@ -477,11 +469,8 @@ def realpath(filename, *, strict=False):
 if strict:
 # Raise OSError(errno.ELOOP)
 os.stat(newpath)
-else:
-# Return already resolved part + rest of the path unchanged.
-path = newpath
-querying = False
-continue
+path = newpath
+continue
 seen[newpath] = None # not resolved symlink
 target = os.readlink(newpath)
 if target.startswith(sep):
diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py
index ff78410738022d..248fe2cc5d5ca8 100644
--- a/Lib/test/test_posixpath.py
+++ b/Lib/test/test_posixpath.py
@@ -484,7 +484,7 @@ def test_realpath_symlink_loops(self):
 self.assertEqual(realpath(ABSTFN+"1/../x"), dirname(ABSTFN) + "/x")
 os.symlink(ABSTFN+"x", ABSTFN+"y")
 self.assertEqual(realpath(ABSTFN+"1/../" + basename(ABSTFN) + "y"),
- ABSTFN + "y")
+ ABSTFN + "x")
 self.assertEqual(realpath(ABSTFN+"1/../" + basename(ABSTFN) + "1"),
  ABSTFN + "1")
 
diff --git 
a/Misc/NEWS.d/next/Library/2024-04-05-13-38-53.gh-issue-117546.lWjhHE.rst 
b/Misc/NEWS.d/next/Library/2024-04-05-13-38-53.gh-issue-117546.lWjhHE.rst
new file mode 100644
index 00..9762991e47a6a4
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-04-05-13-38-53.gh-issue-117546.lWjhHE.rst
@@ -0,0 +1,2 @@
+Fix issue where :func:`os.path.realpath` stopped resolving symlinks after
+encountering a symlink loop on POSIX.

___
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]


[Python-checkins] gh-67224: Make linecache imports relative to improve startup speed (#117501)

2024-04-10 Thread pablogsal
https://github.com/python/cpython/commit/689ada79150f28b0053fa6c1fb646b75ab2cc200
commit: 689ada79150f28b0053fa6c1fb646b75ab2cc200
branch: main
author: Pablo Galindo Salgado 
committer: pablogsal 
date: 2024-04-10T20:09:25+01:00
summary:

gh-67224: Make linecache imports relative to improve startup speed (#117501)

files:
M Lib/linecache.py

diff --git a/Lib/linecache.py b/Lib/linecache.py
index b97999fc1dc909..d1113b108dc5e4 100644
--- a/Lib/linecache.py
+++ b/Lib/linecache.py
@@ -5,9 +5,6 @@
 that name.
 """
 
-import sys
-import os
-
 __all__ = ["getline", "clearcache", "checkcache", "lazycache"]
 
 
@@ -66,6 +63,11 @@ def checkcache(filename=None):
 size, mtime, lines, fullname = entry
 if mtime is None:
 continue   # no-op for files loaded via a __loader__
+try:
+# This import can fail if the interpreter is shutting down
+import os
+except ImportError:
+return
 try:
 stat = os.stat(fullname)
 except OSError:
@@ -76,6 +78,12 @@ def checkcache(filename=None):
 
 
 def updatecache(filename, module_globals=None):
+# These imports are not at top level because linecache is in the critical
+# path of the interpreter startup and importing os and sys take a lot of 
time
+# and slow down the startup sequence.
+import os
+import sys
+
 """Update a cache entry and return its list of lines.
 If something's wrong, print a message, discard the cache entry,
 and return an empty list."""

___
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]


[Python-checkins] GH-117586: Speed up `pathlib.Path.glob()` by working with strings (#117589)

2024-04-10 Thread barneygale
https://github.com/python/cpython/commit/6258844c27e3b5a43816e7c559089a5fe0a47123
commit: 6258844c27e3b5a43816e7c559089a5fe0a47123
branch: main
author: Barney Gale 
committer: barneygale 
date: 2024-04-10T20:43:07+01:00
summary:

GH-117586: Speed up `pathlib.Path.glob()` by working with strings (#117589)

Move pathlib globbing implementation into a new private class: `glob._Globber`. 
This class implements fast string-based globbing. It's called by 
`pathlib.Path.glob()`, which then converts strings back to path objects.

In the private pathlib ABCs, add a `pathlib._abc.Globber` subclass that works 
with `PathBase` objects rather than strings, and calls user-defined path 
methods like `PathBase.stat()` rather than `os.stat()`.

This sets the stage for two more improvements:

- GH-115060: Query non-wildcard segments with `lstat()`
- GH-116380: Unify `pathlib` and `glob` implementations of globbing.

No change to the implementations of `glob.glob()` and `glob.iglob()`.

files:
A Misc/NEWS.d/next/Library/2024-04-06-20-31-09.gh-issue-117586.UgWdRK.rst
M Lib/glob.py
M Lib/pathlib/__init__.py
M Lib/pathlib/_abc.py

diff --git a/Lib/glob.py b/Lib/glob.py
index a915cf0bdf4502..62cf0394e921d7 100644
--- a/Lib/glob.py
+++ b/Lib/glob.py
@@ -4,7 +4,9 @@
 import os
 import re
 import fnmatch
+import functools
 import itertools
+import operator
 import stat
 import sys
 
@@ -256,7 +258,9 @@ def escape(pathname):
 return drive + pathname
 
 
+_special_parts = ('', '.', '..')
 _dir_open_flags = os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0)
+_no_recurse_symlinks = object()
 
 
 def translate(pat, *, recursive=False, include_hidden=False, seps=None):
@@ -312,3 +316,185 @@ def translate(pat, *, recursive=False, 
include_hidden=False, seps=None):
 results.append(any_sep)
 res = ''.join(results)
 return fr'(?s:{res})\Z'
+
+
[email protected]_cache(maxsize=512)
+def _compile_pattern(pat, sep, case_sensitive, recursive=True):
+"""Compile given glob pattern to a re.Pattern object (observing case
+sensitivity)."""
+flags = re.NOFLAG if case_sensitive else re.IGNORECASE
+regex = translate(pat, recursive=recursive, include_hidden=True, seps=sep)
+return re.compile(regex, flags=flags).match
+
+
+class _Globber:
+"""Class providing shell-style pattern matching and globbing.
+"""
+
+def __init__(self,  sep, case_sensitive, recursive=False):
+self.sep = sep
+self.case_sensitive = case_sensitive
+self.recursive = recursive
+
+# Low-level methods
+
+lstat = staticmethod(os.lstat)
+scandir = staticmethod(os.scandir)
+parse_entry = operator.attrgetter('path')
+concat_path = operator.add
+
+if os.name == 'nt':
+@staticmethod
+def add_slash(pathname):
+tail = os.path.splitroot(pathname)[2]
+if not tail or tail[-1] in '\\/':
+return pathname
+return f'{pathname}\\'
+else:
+@staticmethod
+def add_slash(pathname):
+if not pathname or pathname[-1] == '/':
+return pathname
+return f'{pathname}/'
+
+# High-level methods
+
+def compile(self, pat):
+return _compile_pattern(pat, self.sep, self.case_sensitive, 
self.recursive)
+
+def selector(self, parts):
+"""Returns a function that selects from a given path, walking and
+filtering according to the glob-style pattern parts in *parts*.
+"""
+if not parts:
+return self.select_exists
+part = parts.pop()
+if self.recursive and part == '**':
+selector = self.recursive_selector
+elif part in _special_parts:
+selector = self.special_selector
+else:
+selector = self.wildcard_selector
+return selector(part, parts)
+
+def special_selector(self, part, parts):
+"""Returns a function that selects special children of the given path.
+"""
+select_next = self.selector(parts)
+
+def select_special(path, exists=False):
+path = self.concat_path(self.add_slash(path), part)
+return select_next(path, exists)
+return select_special
+
+def wildcard_selector(self, part, parts):
+"""Returns a function that selects direct children of a given path,
+filtering by pattern.
+"""
+
+match = None if part == '*' else self.compile(part)
+dir_only = bool(parts)
+if dir_only:
+select_next = self.selector(parts)
+
+def select_wildcard(path, exists=False):
+try:
+# We must close the scandir() object before proceeding to
+# avoid exhausting file descriptors when globbing deep trees.
+with self.scandir(path) as scandir_it:
+entries = list(scandir_it)
+except OSError:
+pass
+else:
+for entry in entries:
+  

[Python-checkins] GH-117586: Speed up `pathlib.Path.walk()` by working with strings (#117726)

2024-04-10 Thread barneygale
https://github.com/python/cpython/commit/0cc71bde001950d3634c235e2b0d24cda6ce7dce
commit: 0cc71bde001950d3634c235e2b0d24cda6ce7dce
branch: main
author: Barney Gale 
committer: barneygale 
date: 2024-04-11T01:26:53+01:00
summary:

GH-117586: Speed up `pathlib.Path.walk()` by working with strings (#117726)

Move `pathlib.Path.walk()` implementation into `glob._Globber`. The new
`glob._Globber.walk()` classmethod works with strings internally, which is
a little faster than generating `Path` objects and keeping them normalized.
The `pathlib.Path.walk()` method converts the strings back to path objects.

In the private pathlib ABCs, our existing subclass of `_Globber` ensures
that `PathBase` instances are used throughout.

Follow-up to #117589.

files:
A Misc/NEWS.d/next/Library/2024-04-10-21-08-32.gh-issue-117586.UCL__1.rst
M Lib/glob.py
M Lib/pathlib/__init__.py
M Lib/pathlib/_abc.py

diff --git a/Lib/glob.py b/Lib/glob.py
index 62cf0394e921d7..b1d2681d687ff7 100644
--- a/Lib/glob.py
+++ b/Lib/glob.py
@@ -498,3 +498,40 @@ def select_exists(self, path, exists=False):
 yield path
 except OSError:
 pass
+
+@classmethod
+def walk(cls, root, top_down, on_error, follow_symlinks):
+"""Walk the directory tree from the given root, similar to os.walk().
+"""
+paths = [root]
+while paths:
+path = paths.pop()
+if isinstance(path, tuple):
+yield path
+continue
+try:
+with cls.scandir(path) as scandir_it:
+dirnames = []
+filenames = []
+if not top_down:
+paths.append((path, dirnames, filenames))
+for entry in scandir_it:
+name = entry.name
+try:
+if entry.is_dir(follow_symlinks=follow_symlinks):
+if not top_down:
+paths.append(cls.parse_entry(entry))
+dirnames.append(name)
+else:
+filenames.append(name)
+except OSError:
+filenames.append(name)
+except OSError as error:
+if on_error is not None:
+on_error(error)
+else:
+if top_down:
+yield path, dirnames, filenames
+if dirnames:
+prefix = cls.add_slash(path)
+paths += [cls.concat_path(prefix, d) for d in 
reversed(dirnames)]
diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py
index 88e3286d9b08dc..746cbcd9d83d86 100644
--- a/Lib/pathlib/__init__.py
+++ b/Lib/pathlib/__init__.py
@@ -586,18 +586,6 @@ def iterdir(self):
 """
 return (self._make_child_relpath(name) for name in os.listdir(self))
 
-def _scandir(self):
-return os.scandir(self)
-
-def _make_child_direntry(self, entry):
-# Transform an entry yielded from _scandir() into a path object.
-path_str = entry.name if str(self) == '.' else entry.path
-path = self.with_segments(path_str)
-path._str = path_str
-path._drv = self.drive
-path._root = self.root
-path._tail_cached = self._tail + [entry.name]
-return path
 
 def _make_child_relpath(self, name):
 if not name:
@@ -663,8 +651,12 @@ def rglob(self, pattern, *, case_sensitive=None, 
recurse_symlinks=False):
 def walk(self, top_down=True, on_error=None, follow_symlinks=False):
 """Walk the directory tree from this directory, similar to 
os.walk()."""
 sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks)
-return _abc.PathBase.walk(
-self, top_down=top_down, on_error=on_error, 
follow_symlinks=follow_symlinks)
+root_dir = str(self)
+results = self._globber.walk(root_dir, top_down, on_error, 
follow_symlinks)
+for path_str, dirnames, filenames in results:
+if root_dir == '.':
+path_str = path_str[2:]
+yield self._from_parsed_string(path_str), dirnames, filenames
 
 def absolute(self):
 """Return an absolute version of this path
diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py
index 553f797d75e793..b6cab0d285acd9 100644
--- a/Lib/pathlib/_abc.py
+++ b/Lib/pathlib/_abc.py
@@ -45,9 +45,15 @@ def _is_case_sensitive(parser):
 
 class Globber(glob._Globber):
 lstat = operator.methodcaller('lstat')
-scandir = operator.methodcaller('_scandir')
 add_slash = operator.methodcaller('joinpath', '')
 
+@staticmethod
+def scandir(path):
+# Emulate os.scandir(), which returns an object that can be used as a
+# context manager. This method is called by walk() and glob().
+from contextlib i

[Python-checkins] gh-76785: Add More Tests to test_interpreters.test_api (gh-117662)

2024-04-10 Thread ericsnowcurrently
https://github.com/python/cpython/commit/993c3cca16ed00a0bfe467f7f26ac4f5f6dfb24c
commit: 993c3cca16ed00a0bfe467f7f26ac4f5f6dfb24c
branch: main
author: Eric Snow 
committer: ericsnowcurrently 
date: 2024-04-10T18:37:01-06:00
summary:

gh-76785: Add More Tests to test_interpreters.test_api (gh-117662)

In addition to the increase test coverage, this is a precursor to sorting out 
how we handle interpreters created directly via the C-API.

files:
M Include/internal/pycore_crossinterp.h
M Include/internal/pycore_interp.h
M Include/internal/pycore_pystate.h
M Include/internal/pycore_runtime_init.h
M Lib/test/support/interpreters/__init__.py
M Lib/test/test__xxinterpchannels.py
M Lib/test/test__xxsubinterpreters.py
M Lib/test/test_capi/test_misc.py
M Lib/test/test_interpreters/test_api.py
M Lib/test/test_interpreters/utils.py
M Modules/_interpreters_common.h
M Modules/_testinternalcapi.c
M Modules/_xxinterpchannelsmodule.c
M Modules/_xxsubinterpretersmodule.c
M Python/crossinterp.c
M Python/crossinterp_exceptions.h
M Python/pylifecycle.c
M Python/pystate.c

diff --git a/Include/internal/pycore_crossinterp.h 
b/Include/internal/pycore_crossinterp.h
index 63abef864ff87f..64d881dbab7357 100644
--- a/Include/internal/pycore_crossinterp.h
+++ b/Include/internal/pycore_crossinterp.h
@@ -217,6 +217,11 @@ typedef struct _excinfo {
 const char *errdisplay;
 } _PyXI_excinfo;
 
+PyAPI_FUNC(int) _PyXI_InitExcInfo(_PyXI_excinfo *info, PyObject *exc);
+PyAPI_FUNC(PyObject *) _PyXI_FormatExcInfo(_PyXI_excinfo *info);
+PyAPI_FUNC(PyObject *) _PyXI_ExcInfoAsObject(_PyXI_excinfo *info);
+PyAPI_FUNC(void) _PyXI_ClearExcInfo(_PyXI_excinfo *info);
+
 
 typedef enum error_code {
 _PyXI_ERR_NO_ERROR = 0,
@@ -313,6 +318,21 @@ PyAPI_FUNC(PyObject *) 
_PyXI_ApplyCapturedException(_PyXI_session *session);
 PyAPI_FUNC(int) _PyXI_HasCapturedException(_PyXI_session *session);
 
 
+/*/
+/* other API */
+/*/
+
+// Export for _testinternalcapi shared extension
+PyAPI_FUNC(PyInterpreterState *) _PyXI_NewInterpreter(
+PyInterpreterConfig *config,
+PyThreadState **p_tstate,
+PyThreadState **p_save_tstate);
+PyAPI_FUNC(void) _PyXI_EndInterpreter(
+PyInterpreterState *interp,
+PyThreadState *tstate,
+PyThreadState **p_save_tstate);
+
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h
index 1bb123b8607edd..c96a9e40e4274a 100644
--- a/Include/internal/pycore_interp.h
+++ b/Include/internal/pycore_interp.h
@@ -103,11 +103,22 @@ struct _is {
 int requires_idref;
 PyThread_type_lock id_mutex;
 
+#define _PyInterpreterState_WHENCE_NOTSET -1
+#define _PyInterpreterState_WHENCE_UNKNOWN 0
+#define _PyInterpreterState_WHENCE_RUNTIME 1
+#define _PyInterpreterState_WHENCE_LEGACY_CAPI 2
+#define _PyInterpreterState_WHENCE_CAPI 3
+#define _PyInterpreterState_WHENCE_XI 4
+#define _PyInterpreterState_WHENCE_MAX 4
+long _whence;
+
 /* Has been initialized to a safe state.
 
In order to be effective, this must be set to 0 during or right
after allocation. */
 int _initialized;
+/* Has been fully initialized via pylifecycle.c. */
+int _ready;
 int finalizing;
 
 uintptr_t last_restart_version;
@@ -305,6 +316,11 @@ PyAPI_FUNC(int) 
_PyInterpreterState_IDInitref(PyInterpreterState *);
 PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *);
 PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *);
 
+PyAPI_FUNC(long) _PyInterpreterState_GetWhence(PyInterpreterState *interp);
+extern void _PyInterpreterState_SetWhence(
+PyInterpreterState *interp,
+long whence);
+
 extern const PyConfig* _PyInterpreterState_GetConfig(PyInterpreterState 
*interp);
 
 // Get a copy of the current interpreter configuration.
diff --git a/Include/internal/pycore_pystate.h 
b/Include/internal/pycore_pystate.h
index 35e266acd3ab60..eb5b5fee59009c 100644
--- a/Include/internal/pycore_pystate.h
+++ b/Include/internal/pycore_pystate.h
@@ -77,6 +77,9 @@ _Py_IsMainInterpreterFinalizing(PyInterpreterState *interp)
 interp == &_PyRuntime._main_interpreter);
 }
 
+// Export for _xxsubinterpreters module.
+PyAPI_FUNC(PyObject *) _PyInterpreterState_GetIDObject(PyInterpreterState *);
+
 // Export for _xxsubinterpreters module.
 PyAPI_FUNC(int) _PyInterpreterState_SetRunningMain(PyInterpreterState *);
 PyAPI_FUNC(void) _PyInterpreterState_SetNotRunningMain(PyInterpreterState *);
diff --git a/Include/internal/pycore_runtime_init.h 
b/Include/internal/pycore_runtime_init.h
index 88d888943d28b1..33c7a9dadfd2a1 100644
--- a/Include/internal/pycore_runtime_init.h
+++ b/Include/internal/pycore_runtime_init.h
@@ -162,6 +162,7 @@ extern PyTypeObject _PyExc_MemoryError;
 #define _PyInterpreterState_INIT(INTERP) \
 { \
 .id_refcount = -1, \
+._whence = _PyInterpreterState_WHENCE_NOTSET, \
 .imports = IMPORTS_INIT, \
 .ceval = { \
 .recursion_limit = Py_DEF