https://github.com/python/cpython/commit/4d87832d8790a2ceb9179ed28b71d83cb2a68ab5
commit: 4d87832d8790a2ceb9179ed28b71d83cb2a68ab5
branch: 3.12
author: Mark Shannon <[email protected]>
committer: encukou <[email protected]>
date: 2024-02-13T10:45:59+01:00
summary:
[3.12] GH-112215: Backport C recursion changes (GH-115083)
files:
A Misc/NEWS.d/next/Core and
Builtins/2024-02-04-01-26-20.gh-issue-112215.NyRPXM.rst
M Include/cpython/pystate.h
M Lib/test/support/__init__.py
M Lib/test/test_ast.py
M Lib/test/test_call.py
M Lib/test/test_compile.py
M Lib/test/test_isinstance.py
M Lib/test/test_plistlib.py
M Lib/test/test_sys_settrace.py
M Parser/asdl_c.py
M Python/Python-ast.c
M Python/ast.c
M Python/ast_opt.c
M Python/symtable.c
diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h
index 628f2e0996e469..95fad893786d8d 100644
--- a/Include/cpython/pystate.h
+++ b/Include/cpython/pystate.h
@@ -251,12 +251,24 @@ struct _ts {
/* WASI has limited call stack. Python's recursion limit depends on code
layout, optimization, and WASI runtime. Wasmtime can handle about 700
recursions, sometimes less. 500 is a more conservative limit. */
-#ifndef C_RECURSION_LIMIT
-# ifdef __wasi__
+#ifdef Py_DEBUG
+# if defined(__wasi__)
+# define C_RECURSION_LIMIT 150
+# else
+# define C_RECURSION_LIMIT 500
+# endif
+#else
+# if defined(__wasi__)
# define C_RECURSION_LIMIT 500
+# elif defined(__s390x__)
+# define C_RECURSION_LIMIT 800
+# elif defined(_WIN32)
+# define C_RECURSION_LIMIT 3000
+# elif defined(_Py_ADDRESS_SANITIZER)
+# define C_RECURSION_LIMIT 4000
# else
- // This value is duplicated in Lib/test/support/__init__.py
-# define C_RECURSION_LIMIT 1500
+ // This value is duplicated in Lib/test/support/__init__.py
+# define C_RECURSION_LIMIT 10000
# endif
#endif
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 8c4b4e023f633f..cb5a84aa74e05f 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -2112,13 +2112,13 @@ def set_recursion_limit(limit):
finally:
sys.setrecursionlimit(original_limit)
-def infinite_recursion(max_depth=100):
- """Set a lower limit for tests that interact with infinite recursions
- (e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some
- debug windows builds, due to not enough functions being inlined the
- stack size might not handle the default recursion limit (1000). See
- bpo-11105 for details."""
- if max_depth < 3:
+def infinite_recursion(max_depth=None):
+ if max_depth is None:
+ # Pick a number large enough to cause problems
+ # but not take too long for code that can handle
+ # very deep recursion.
+ max_depth = 20_000
+ elif max_depth < 3:
raise ValueError("max_depth must be at least 3, got {max_depth}")
depth = get_recursion_depth()
depth = max(depth - 1, 1) # Ignore infinite_recursion() frame.
@@ -2362,7 +2362,22 @@ def adjust_int_max_str_digits(max_digits):
EXCEEDS_RECURSION_LIMIT = 5000
# The default C recursion limit (from Include/cpython/pystate.h).
-C_RECURSION_LIMIT = 1500
+if Py_DEBUG:
+ if is_wasi:
+ C_RECURSION_LIMIT = 150
+ else:
+ C_RECURSION_LIMIT = 500
+else:
+ if is_wasi:
+ C_RECURSION_LIMIT = 500
+ elif hasattr(os, 'uname') and os.uname().machine == 's390x':
+ C_RECURSION_LIMIT = 800
+ elif sys.platform.startswith('win'):
+ C_RECURSION_LIMIT = 3000
+ elif check_sanitizer(address=True):
+ C_RECURSION_LIMIT = 4000
+ else:
+ C_RECURSION_LIMIT = 10000
#Windows doesn't have os.uname() but it doesn't support s390x.
skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine ==
's390x',
diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py
index 3ba7cf7b04266f..9736208a92fc6e 100644
--- a/Lib/test/test_ast.py
+++ b/Lib/test/test_ast.py
@@ -1087,9 +1087,9 @@ def next(self):
@unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI")
@support.cpython_only
def test_ast_recursion_limit(self):
- fail_depth = support.EXCEEDS_RECURSION_LIMIT
+ fail_depth = support.C_RECURSION_LIMIT + 1
crash_depth = 100_000
- success_depth = 1200
+ success_depth = int(support.C_RECURSION_LIMIT * 0.9)
def check_limit(prefix, repeated):
expect_ok = prefix + repeated * success_depth
diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py
index ec8dc29d36c16a..46abf40605f491 100644
--- a/Lib/test/test_call.py
+++ b/Lib/test/test_call.py
@@ -1,5 +1,5 @@
import unittest
-from test.support import cpython_only, requires_limited_api, skip_on_s390x
+from test.support import cpython_only, requires_limited_api, skip_on_s390x,
is_wasi, Py_DEBUG
try:
import _testcapi
except ImportError:
@@ -932,6 +932,7 @@ def test_multiple_values(self):
class TestRecursion(unittest.TestCase):
@skip_on_s390x
+ @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack")
def test_super_deep(self):
def recurse(n):
diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py
index 42df670fe00e0a..ed23d31c0d3ecd 100644
--- a/Lib/test/test_compile.py
+++ b/Lib/test/test_compile.py
@@ -607,9 +607,9 @@ def test_compiler_recursion_limit(self):
# Expected limit is C_RECURSION_LIMIT * 2
# Duplicating the limit here is a little ugly.
# Perhaps it should be exposed somewhere...
- fail_depth = C_RECURSION_LIMIT * 2 + 1
+ fail_depth = C_RECURSION_LIMIT + 1
crash_depth = C_RECURSION_LIMIT * 100
- success_depth = int(C_RECURSION_LIMIT * 1.8)
+ success_depth = int(C_RECURSION_LIMIT * 0.9)
def check_limit(prefix, repeated, mode="single"):
expect_ok = prefix + repeated * success_depth
diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py
index bf9332e40aeaf2..b3e317bd7990fd 100644
--- a/Lib/test/test_isinstance.py
+++ b/Lib/test/test_isinstance.py
@@ -352,7 +352,7 @@ def blowstack(fxn, arg, compare_to):
# Make sure that calling isinstance with a deeply nested tuple for its
# argument will raise RecursionError eventually.
tuple_arg = (compare_to,)
- for cnt in range(support.EXCEEDS_RECURSION_LIMIT):
+ for cnt in range(support.C_RECURSION_LIMIT * 2):
tuple_arg = (tuple_arg,)
fxn(arg, tuple_arg)
diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py
index 3f10f16d71996d..fa46050658afe0 100644
--- a/Lib/test/test_plistlib.py
+++ b/Lib/test/test_plistlib.py
@@ -908,7 +908,7 @@ def test_cycles(self):
self.assertIs(b['x'], b)
def test_deep_nesting(self):
- tests = [50, 100_000] if support.is_wasi else [50, 300, 100_000]
+ tests = [50, 100_000] if support.is_wasi else [50, 600, 100_000]
for N in tests:
chunks = [b'\xa1' + (i + 1).to_bytes(4, 'big') for i in range(N)]
try:
diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py
index 7e16e94aa110b2..196fd60d1973f6 100644
--- a/Lib/test/test_sys_settrace.py
+++ b/Lib/test/test_sys_settrace.py
@@ -2965,16 +2965,18 @@ def test_trace_unpack_long_sequence(self):
self.assertEqual(counts, {'call': 1, 'line': 301, 'return': 1})
def test_trace_lots_of_globals(self):
+ count = min(1000, int(support.C_RECURSION_LIMIT * 0.8))
+
code = """if 1:
def f():
return (
{}
)
- """.format("\n+\n".join(f"var{i}\n" for i in range(1000)))
- ns = {f"var{i}": i for i in range(1000)}
+ """.format("\n+\n".join(f"var{i}\n" for i in range(count)))
+ ns = {f"var{i}": i for i in range(count)}
exec(code, ns)
counts = self.count_traces(ns["f"])
- self.assertEqual(counts, {'call': 1, 'line': 2000, 'return': 1})
+ self.assertEqual(counts, {'call': 1, 'line': count * 2, 'return': 1})
class TestEdgeCases(unittest.TestCase):
diff --git a/Misc/NEWS.d/next/Core and
Builtins/2024-02-04-01-26-20.gh-issue-112215.NyRPXM.rst b/Misc/NEWS.d/next/Core
and Builtins/2024-02-04-01-26-20.gh-issue-112215.NyRPXM.rst
new file mode 100644
index 00000000000000..4f6c356e414928
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and
Builtins/2024-02-04-01-26-20.gh-issue-112215.NyRPXM.rst
@@ -0,0 +1,2 @@
+Change the C recursion limits to more closely reflect the underlying
+platform limits.
diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index d42c26396d5de2..2c34f5c1bc2675 100755
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -1393,15 +1393,14 @@ class PartingShots(StaticVisitor):
int starting_recursion_depth;
/* Be careful here to prevent overflow. */
- int COMPILER_STACK_FRAME_SCALE = 2;
PyThreadState *tstate = _PyThreadState_GET();
if (!tstate) {
return NULL;
}
struct validator vstate;
- vstate.recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
+ vstate.recursion_limit = C_RECURSION_LIMIT;
int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining;
- starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
+ starting_recursion_depth = recursion_depth;
vstate.recursion_depth = starting_recursion_depth;
PyObject *result = ast2obj_mod(state, &vstate, t);
diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index 6c95f07c386fd0..ecaff2041f9565 100644
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -13152,15 +13152,14 @@ PyObject* PyAST_mod2obj(mod_ty t)
int starting_recursion_depth;
/* Be careful here to prevent overflow. */
- int COMPILER_STACK_FRAME_SCALE = 2;
PyThreadState *tstate = _PyThreadState_GET();
if (!tstate) {
return NULL;
}
struct validator vstate;
- vstate.recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
+ vstate.recursion_limit = C_RECURSION_LIMIT;
int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining;
- starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
+ starting_recursion_depth = recursion_depth;
vstate.recursion_depth = starting_recursion_depth;
PyObject *result = ast2obj_mod(state, &vstate, t);
diff --git a/Python/ast.c b/Python/ast.c
index 82d7beec0ee510..76f6556ded098b 100644
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -1038,9 +1038,6 @@ validate_type_params(struct validator *state,
asdl_type_param_seq *tps)
}
-/* See comments in symtable.c. */
-#define COMPILER_STACK_FRAME_SCALE 2
-
int
_PyAST_Validate(mod_ty mod)
{
@@ -1057,9 +1054,9 @@ _PyAST_Validate(mod_ty mod)
}
/* Be careful here to prevent overflow. */
int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining;
- starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
+ starting_recursion_depth = recursion_depth;
state.recursion_depth = starting_recursion_depth;
- state.recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
+ state.recursion_limit = C_RECURSION_LIMIT;
switch (mod->kind) {
case Module_kind:
diff --git a/Python/ast_opt.c b/Python/ast_opt.c
index f8c4a9513236b9..e881b7fe2d4413 100644
--- a/Python/ast_opt.c
+++ b/Python/ast_opt.c
@@ -1102,9 +1102,6 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_,
_PyASTOptimizeState *stat
#undef CALL_OPT
#undef CALL_SEQ
-/* See comments in symtable.c. */
-#define COMPILER_STACK_FRAME_SCALE 2
-
int
_PyAST_Optimize(mod_ty mod, PyArena *arena, _PyASTOptimizeState *state)
{
@@ -1118,9 +1115,9 @@ _PyAST_Optimize(mod_ty mod, PyArena *arena,
_PyASTOptimizeState *state)
}
/* Be careful here to prevent overflow. */
int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining;
- starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
+ starting_recursion_depth = recursion_depth;
state->recursion_depth = starting_recursion_depth;
- state->recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
+ state->recursion_limit = C_RECURSION_LIMIT;
int ret = astfold_mod(mod, arena, state);
assert(ret || PyErr_Occurred());
diff --git a/Python/symtable.c b/Python/symtable.c
index 7972984a550066..cfa5bec4434b3b 100644
--- a/Python/symtable.c
+++ b/Python/symtable.c
@@ -281,11 +281,6 @@ symtable_new(void)
return NULL;
}
-/* Using a scaling factor means this should automatically adjust when
- the recursion limit is adjusted for small or large C stack allocations.
-*/
-#define COMPILER_STACK_FRAME_SCALE 2
-
struct symtable *
_PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future)
{
@@ -312,9 +307,9 @@ _PySymtable_Build(mod_ty mod, PyObject *filename,
PyFutureFeatures *future)
}
/* Be careful here to prevent overflow. */
int recursion_depth = C_RECURSION_LIMIT - tstate->c_recursion_remaining;
- starting_recursion_depth = recursion_depth * COMPILER_STACK_FRAME_SCALE;
+ starting_recursion_depth = recursion_depth;
st->recursion_depth = starting_recursion_depth;
- st->recursion_limit = C_RECURSION_LIMIT * COMPILER_STACK_FRAME_SCALE;
+ st->recursion_limit = C_RECURSION_LIMIT;
/* Make the initial symbol information gathering pass */
if (!symtable_enter_block(st, &_Py_ID(top), ModuleBlock, (void *)mod, 0,
0, 0, 0)) {
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]
