[Python-checkins] gh-69990: Make Profile.print_stats support sorting by multiple values (GH-104590)

2024-02-16 Thread serhiy-storchaka 
https://github.com/python/cpython/commit/2a7a0020c9d006d268b839320979364498a5f0e6
commit: 2a7a0020c9d006d268b839320979364498a5f0e6
branch: main
author: Furkan Onder 
committer: serhiy-storchaka 
date: 2024-02-16T12:03:46Z
summary:

gh-69990: Make Profile.print_stats support sorting by multiple values 
(GH-104590)

Co-authored-by: Chiu-Hsiang Hsu

files:
A Misc/NEWS.d/next/Library/2023-05-17-21-33-21.gh-issue-69990.Blvz9G.rst
M Doc/library/profile.rst
M Lib/cProfile.py
M Lib/profile.py
M Lib/test/test_profile.py

diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst
index cc059b66fcb84b..3ca802e024bc27 100644
--- a/Doc/library/profile.rst
+++ b/Doc/library/profile.rst
@@ -299,6 +299,13 @@ functions:
   Create a :class:`~pstats.Stats` object based on the current
   profile and print the results to stdout.
 
+  The *sort* parameter specifies the sorting order of the displayed
+  statistics. It accepts a single key or a tuple of keys to enable
+  multi-level sorting, as in :func:`Stats.sort_stats 
`.
+
+  .. versionadded:: 3.13
+ :meth:`~Profile.print_stats` now accepts a tuple of keys.
+
.. method:: dump_stats(filename)
 
   Write the results of the current profile to *filename*.
diff --git a/Lib/cProfile.py b/Lib/cProfile.py
index 135a12c3965c00..9c132372dc4ee0 100755
--- a/Lib/cProfile.py
+++ b/Lib/cProfile.py
@@ -41,7 +41,9 @@ class Profile(_lsprof.Profiler):
 
 def print_stats(self, sort=-1):
 import pstats
-pstats.Stats(self).strip_dirs().sort_stats(sort).print_stats()
+if not isinstance(sort, tuple):
+sort = (sort,)
+pstats.Stats(self).strip_dirs().sort_stats(*sort).print_stats()
 
 def dump_stats(self, file):
 import marshal
diff --git a/Lib/profile.py b/Lib/profile.py
index 4b82523b03d64b..f2f8c2f21333e0 100755
--- a/Lib/profile.py
+++ b/Lib/profile.py
@@ -387,8 +387,9 @@ def simulate_cmd_complete(self):
 
 def print_stats(self, sort=-1):
 import pstats
-pstats.Stats(self).strip_dirs().sort_stats(sort). \
-  print_stats()
+if not isinstance(sort, tuple):
+sort = (sort,)
+pstats.Stats(self).strip_dirs().sort_stats(*sort).print_stats()
 
 def dump_stats(self, file):
 with open(file, 'wb') as f:
diff --git a/Lib/test/test_profile.py b/Lib/test/test_profile.py
index a1dfc9abbb8ef7..0f16b92334999c 100644
--- a/Lib/test/test_profile.py
+++ b/Lib/test/test_profile.py
@@ -7,7 +7,7 @@
 from difflib import unified_diff
 from io import StringIO
 from test.support.os_helper import TESTFN, unlink, temp_dir, change_cwd
-from contextlib import contextmanager
+from contextlib import contextmanager, redirect_stdout
 
 import profile
 from test.profilee import testfunc, timer
@@ -92,6 +92,11 @@ def test_run(self):
 self.profilermodule.run("int('1')", filename=TESTFN)
 self.assertTrue(os.path.exists(TESTFN))
 
+def test_run_with_sort_by_values(self):
+with redirect_stdout(StringIO()) as f:
+self.profilermodule.run("int('1')", sort=('tottime', 'stdname'))
+self.assertIn("Ordered by: internal time, standard name", f.getvalue())
+
 def test_runctx(self):
 with silent():
 self.profilermodule.runctx("testfunc()", globals(), locals())
diff --git 
a/Misc/NEWS.d/next/Library/2023-05-17-21-33-21.gh-issue-69990.Blvz9G.rst 
b/Misc/NEWS.d/next/Library/2023-05-17-21-33-21.gh-issue-69990.Blvz9G.rst
new file mode 100644
index 00..b0cdf44f7b9e39
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2023-05-17-21-33-21.gh-issue-69990.Blvz9G.rst
@@ -0,0 +1 @@
+:meth:`Profile.print_stats` has been improved to accept multiple sort 
arguments. Patched by Chiu-Hsiang Hsu and Furkan Onder.

___
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]

[Python-checkins] Add `Python/tier2_redundancy_eliminator_cases.c.h` to `.gitattributes` as generated (#115551)

2024-02-16 Thread sobolevn 
https://github.com/python/cpython/commit/b178eae3a676473e1c2287a46b4941fc0bcd193f
commit: b178eae3a676473e1c2287a46b4941fc0bcd193f
branch: main
author: Nikita Sobolev 
committer: sobolevn 
date: 2024-02-16T13:10:21Z
summary:

Add `Python/tier2_redundancy_eliminator_cases.c.h` to `.gitattributes` as 
generated (#115551)

files:
M .gitattributes

diff --git a/.gitattributes b/.gitattributes
index 07d877027b09f6..c984797e1ac7c6 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -94,7 +94,7 @@ Programs/test_frozenmain.h  generated
 Python/Python-ast.c generated
 Python/executor_cases.c.h   generated
 Python/generated_cases.c.h  generated
-Python/tier2_redundancy_eliminator_bytecodes.c.hgenerated
+Python/tier2_redundancy_eliminator_cases.c.hgenerated
 Python/opcode_targets.h generated
 Python/stdlib_module_names.hgenerated
 Tools/peg_generator/pegen/grammar_parser.py generated

___
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]

[Python-checkins] gh-102013: Move PyUnstable_GC_VisitObjects() to Include/cpython/objimpl.h (#115560)

2024-02-16 Thread vstinner 
https://github.com/python/cpython/commit/144eb5605b445d22729db6c416d03cc24947ba56
commit: 144eb5605b445d22729db6c416d03cc24947ba56
branch: main
author: Victor Stinner 
committer: vstinner 
date: 2024-02-16T15:49:13+01:00
summary:

gh-102013: Move PyUnstable_GC_VisitObjects() to Include/cpython/objimpl.h 
(#115560)

Include/objimpl.h must only contain the limited C API, whereas
PyUnstable_GC_VisitObjects() is excluded from the limited C API.

files:
M Include/cpython/objimpl.h
M Include/objimpl.h

diff --git a/Include/cpython/objimpl.h b/Include/cpython/objimpl.h
index 58a30aeea6ac64..e0c2ce286f13ce 100644
--- a/Include/cpython/objimpl.h
+++ b/Include/cpython/objimpl.h
@@ -85,3 +85,20 @@ PyAPI_FUNC(PyObject **) 
PyObject_GET_WEAKREFS_LISTPTR(PyObject *op);
 
 PyAPI_FUNC(PyObject *) PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *,
  size_t);
+
+
+/* Visit all live GC-capable objects, similar to gc.get_objects(None). The
+ * supplied callback is called on every such object with the void* arg set
+ * to the supplied arg. Returning 0 from the callback ends iteration, returning
+ * 1 allows iteration to continue. Returning any other value may result in
+ * undefined behaviour.
+ *
+ * If new objects are (de)allocated by the callback it is undefined if they
+ * will be visited.
+
+ * Garbage collection is disabled during operation. Explicitly running a
+ * collection in the callback may lead to undefined behaviour e.g. visiting the
+ * same objects multiple times or not at all.
+ */
+typedef int (*gcvisitobjects_t)(PyObject*, void*);
+PyAPI_FUNC(void) PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void* 
arg);
diff --git a/Include/objimpl.h b/Include/objimpl.h
index ff5fa7a8c1d3d8..56472a72e42d34 100644
--- a/Include/objimpl.h
+++ b/Include/objimpl.h
@@ -153,25 +153,6 @@ PyAPI_FUNC(int) PyGC_Enable(void);
 PyAPI_FUNC(int) PyGC_Disable(void);
 PyAPI_FUNC(int) PyGC_IsEnabled(void);
 
-
-#if !defined(Py_LIMITED_API)
-/* Visit all live GC-capable objects, similar to gc.get_objects(None). The
- * supplied callback is called on every such object with the void* arg set
- * to the supplied arg. Returning 0 from the callback ends iteration, returning
- * 1 allows iteration to continue. Returning any other value may result in
- * undefined behaviour.
- *
- * If new objects are (de)allocated by the callback it is undefined if they
- * will be visited.
-
- * Garbage collection is disabled during operation. Explicitly running a
- * collection in the callback may lead to undefined behaviour e.g. visiting the
- * same objects multiple times or not at all.
- */
-typedef int (*gcvisitobjects_t)(PyObject*, void*);
-PyAPI_FUNC(void) PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void* 
arg);
-#endif
-
 /* Test if a type has a GC head */
 #define PyType_IS_GC(t) PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC)
 

___
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]

[Python-checkins] gh-115480: Minor fixups in int constant propagation (GH-115507)

2024-02-16 Thread Fidget-Spinner 
https://github.com/python/cpython/commit/f92857a93016aa26ba93959d2bdb690ef52e7f07
commit: f92857a93016aa26ba93959d2bdb690ef52e7f07
branch: main
author: Ken Jin 
committer: Fidget-Spinner 
date: 2024-02-16T22:59:43+08:00
summary:

gh-115480: Minor fixups in int constant propagation (GH-115507)

files:
M Python/optimizer_analysis.c
M Python/tier2_redundancy_eliminator_bytecodes.c
M Python/tier2_redundancy_eliminator_cases.c.h

diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c
index d73bc310345f41..b104d2fa7baec9 100644
--- a/Python/optimizer_analysis.c
+++ b/Python/optimizer_analysis.c
@@ -588,16 +588,17 @@ remove_globals(_PyInterpreterFrame *frame, 
_PyUOpInstruction *buffer,
 INST->oparg = ARG;\
 INST->operand = OPERAND;
 
+#define OUT_OF_SPACE_IF_NULL(EXPR) \
+do {   \
+if ((EXPR) == NULL) {  \
+goto out_of_space; \
+}  \
+} while (0);
+
 #define _LOAD_ATTR_NOT_NULL \
 do {\
-attr = sym_new_known_notnull(ctx); \
-if (attr == NULL) { \
-goto error; \
-} \
-null = sym_new_null(ctx); \
-if (null == NULL) { \
-goto error; \
-} \
+OUT_OF_SPACE_IF_NULL(attr = sym_new_known_notnull(ctx)); \
+OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); \
 } while (0);
 
 
diff --git a/Python/tier2_redundancy_eliminator_bytecodes.c 
b/Python/tier2_redundancy_eliminator_bytecodes.c
index 39ea0eef627632..6aae590a8e51e4 100644
--- a/Python/tier2_redundancy_eliminator_bytecodes.c
+++ b/Python/tier2_redundancy_eliminator_bytecodes.c
@@ -43,10 +43,8 @@ dummy_func(void) {
 
 op(_LOAD_FAST_AND_CLEAR, (-- value)) {
 value = GETLOCAL(oparg);
-_Py_UOpsSymType *temp = sym_new_null(ctx);
-if (temp == NULL) {
-goto out_of_space;
-}
+_Py_UOpsSymType *temp;
+OUT_OF_SPACE_IF_NULL(temp = sym_new_null(ctx));
 GETLOCAL(oparg) = temp;
 }
 
@@ -89,14 +87,12 @@ dummy_func(void) {
 if (temp == NULL) {
 goto error;
 }
-res = sym_new_const(ctx, temp);
-// TODO replace opcode with constant propagated one and add tests!
+OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, temp));
+// TODO gh-115506:
+// replace opcode with constant propagated one and add tests!
 }
 else {
-res = sym_new_known_type(ctx, &PyLong_Type);
-if (res == NULL) {
-goto out_of_space;
-}
+OUT_OF_SPACE_IF_NULL(res = sym_new_known_type(ctx, &PyLong_Type));
 }
 }
 
@@ -109,14 +105,12 @@ dummy_func(void) {
 if (temp == NULL) {
 goto error;
 }
-res = sym_new_const(ctx, temp);
-// TODO replace opcode with constant propagated one and add tests!
+OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, temp));
+// TODO gh-115506:
+// replace opcode with constant propagated one and add tests!
 }
 else {
-res = sym_new_known_type(ctx, &PyLong_Type);
-if (res == NULL) {
-goto out_of_space;
-}
+OUT_OF_SPACE_IF_NULL(res = sym_new_known_type(ctx, &PyLong_Type));
 }
 }
 
@@ -129,14 +123,12 @@ dummy_func(void) {
 if (temp == NULL) {
 goto error;
 }
-res = sym_new_const(ctx, temp);
-// TODO replace opcode with constant propagated one and add tests!
+OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, temp));
+// TODO gh-115506:
+// replace opcode with constant propagated one and add tests!
 }
 else {
-res = sym_new_known_type(ctx, &PyLong_Type);
-if (res == NULL) {
-goto out_of_space;
-}
+OUT_OF_SPACE_IF_NULL(res = sym_new_known_type(ctx, &PyLong_Type));
 }
 }
 
@@ -147,39 +139,21 @@ dummy_func(void) {
 }
 
 op(_LOAD_CONST_INLINE, (ptr/4 -- value)) {
-value = sym_new_const(ctx, ptr);
-if (value == NULL) {
-goto out_of_space;
-}
+OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr));
 }
 
 op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) {
-value = sym_new_const(ctx, ptr);
-if (value == NULL) {
-goto out_of_space;
-}
+OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr));
 }
 
 op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) {
-value = sym_new_const(ctx, ptr);
-if (value == NULL) {
-goto out_of_space;
-}
-null = sym_new_null(ctx);
-if (null == NULL) {
-goto out_of_space;
-}
+OUT_OF_SPACE_IF_NULL(value = sym_new_const(ctx, ptr));
+OUT_OF_SPACE_IF_NULL(null

[Python-checkins] gh-112529: Make the GC scheduling thread-safe (#114880)

2024-02-16 Thread colesbury 
https://github.com/python/cpython/commit/b24c9161a651f549ed48f4b4dba8996fe9cc4e09
commit: b24c9161a651f549ed48f4b4dba8996fe9cc4e09
branch: main
author: Sam Gross 
committer: colesbury 
date: 2024-02-16T11:22:27-05:00
summary:

gh-112529: Make the GC scheduling thread-safe (#114880)

The GC keeps track of the number of allocations (less deallocations)
since the last GC. This buffers the count in thread-local state and uses
atomic operations to modify the per-interpreter count. The thread-local
buffering avoids contention on shared state.

A consequence is that the GC scheduling is not as precise, so
"test_sneaky_frame_object" is skipped because it requires that the GC be
run exactly after allocating a frame object.

files:
M Include/internal/pycore_gc.h
M Include/internal/pycore_tstate.h
M Lib/test/test_frame.py
M Lib/test/test_gc.py
M Modules/gcmodule.c
M Objects/typeobject.c
M Python/gc_free_threading.c

diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h
index 582a16bf5218ce..a98864f7431398 100644
--- a/Include/internal/pycore_gc.h
+++ b/Include/internal/pycore_gc.h
@@ -260,6 +260,13 @@ struct _gc_runtime_state {
 Py_ssize_t long_lived_pending;
 };
 
+#ifdef Py_GIL_DISABLED
+struct _gc_thread_state {
+/* Thread-local allocation count. */
+Py_ssize_t alloc_count;
+};
+#endif
+
 
 extern void _PyGC_InitState(struct _gc_runtime_state *);
 
diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h
index 97aa85a659fa7b..7fb9ab2056704e 100644
--- a/Include/internal/pycore_tstate.h
+++ b/Include/internal/pycore_tstate.h
@@ -28,6 +28,7 @@ typedef struct _PyThreadStateImpl {
 PyThreadState base;
 
 #ifdef Py_GIL_DISABLED
+struct _gc_thread_state gc;
 struct _mimalloc_thread_state mimalloc;
 struct _Py_object_freelists freelists;
 struct _brc_thread_state brc;
diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py
index baed03d92b9e56..f88206de550da0 100644
--- a/Lib/test/test_frame.py
+++ b/Lib/test/test_frame.py
@@ -13,7 +13,7 @@
 _testcapi = None
 
 from test import support
-from test.support import threading_helper
+from test.support import threading_helper, Py_GIL_DISABLED
 from test.support.script_helper import assert_python_ok
 
 
@@ -294,6 +294,7 @@ def gen():
 assert_python_ok("-c", code)
 
 @support.cpython_only
[email protected](Py_GIL_DISABLED, "test requires precise GC scheduling")
 def test_sneaky_frame_object(self):
 
 def trace(frame, event, arg):
diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py
index b01f344cb14a1a..dd09643788d62f 100644
--- a/Lib/test/test_gc.py
+++ b/Lib/test/test_gc.py
@@ -363,6 +363,7 @@ def __del__(self):
 # To minimize variations, though, we first store the get_count() results
 # and check them at the end.
 @refcount_test
[email protected](Py_GIL_DISABLED, 'needs precise allocation counts')
 def test_get_count(self):
 gc.collect()
 a, b, c = gc.get_count()
diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c
index a2b66b9b78c169..961165e16a0fee 100644
--- a/Modules/gcmodule.c
+++ b/Modules/gcmodule.c
@@ -201,6 +201,16 @@ gc_get_count_impl(PyObject *module)
 /*[clinic end generated code: output=354012e67b16398f input=a392794a08251751]*/
 {
 GCState *gcstate = get_gc_state();
+
+#ifdef Py_GIL_DISABLED
+_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
+struct _gc_thread_state *gc = &tstate->gc;
+
+// Flush the local allocation count to the global count
+_Py_atomic_add_int(&gcstate->generations[0].count, (int)gc->alloc_count);
+gc->alloc_count = 0;
+#endif
+
 return Py_BuildValue("(iii)",
  gcstate->generations[0].count,
  gcstate->generations[1].count,
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 0118ee255ef017..2e25c207c64382 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -1835,6 +1835,8 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t 
nitems)
 if (presize) {
 ((PyObject **)alloc)[0] = NULL;
 ((PyObject **)alloc)[1] = NULL;
+}
+if (PyType_IS_GC(type)) {
 _PyObject_GC_Link(obj);
 }
 memset(obj, '\0', size);
diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c
index 3dc1dc19182eb4..a758c99285a539 100644
--- a/Python/gc_free_threading.c
+++ b/Python/gc_free_threading.c
@@ -23,6 +23,11 @@ typedef struct _gc_runtime_state GCState;
 #  define GC_DEBUG
 #endif
 
+// Each thread buffers the count of allocated objects in a thread-local
+// variable up to +/- this amount to reduce the overhead of updating
+// the global count.
+#define LOCAL_ALLOC_COUNT_THRESHOLD 512
+
 // Automatically choose the generation that needs collecting.
 #define GENERATION_AUTO (-1)
 
@@ -959,6 +964,41 @@ gc_should_collect(GCState *gcstate)
 gcstate->generations[1].threshold == 0);
 }
 
+static void
+record_allocation(PyThreadState *tstate)
+{
+

[Python-checkins] gh-113743: Give _PyTypes_AfterFork a prototype. (gh-115563)

2024-02-16 Thread benjaminp 
https://github.com/python/cpython/commit/2ac9d9f2fbeb743ae6d6b1cbf73337c230e21f3c
commit: 2ac9d9f2fbeb743ae6d6b1cbf73337c230e21f3c
branch: main
author: Benjamin Peterson 
committer: benjaminp 
date: 2024-02-16T08:49:41-08:00
summary:

gh-113743: Give _PyTypes_AfterFork a prototype. (gh-115563)

Fixes a compiler warning.

files:
M Objects/typeobject.c

diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 2e25c207c64382..fe3b7b87c8b4b6 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -4945,7 +4945,7 @@ update_cache_gil_disabled(struct type_cache_entry *entry, 
PyObject *name,
 #endif
 
 void
-_PyTypes_AfterFork()
+_PyTypes_AfterFork(void)
 {
 #ifdef Py_GIL_DISABLED
 struct type_cache *cache = get_type_cache();

___
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]

[Python-checkins] gh-115362: Add documentation to pystats output (#115365)

2024-02-16 Thread AlexWaygood 
https://github.com/python/cpython/commit/fbb016973149d983d30351bdd1aaf00df285c776
commit: fbb016973149d983d30351bdd1aaf00df285c776
branch: main
author: Michael Droettboom 
committer: AlexWaygood 
date: 2024-02-16T17:06:07Z
summary:

gh-115362: Add documentation to pystats output (#115365)

files:
M Tools/scripts/summarize_stats.py

diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py
index 7891b9cf923d33..5bc39fceb4b2a1 100644
--- a/Tools/scripts/summarize_stats.py
+++ b/Tools/scripts/summarize_stats.py
@@ -11,6 +11,7 @@
 import argparse
 import collections
 from collections.abc import KeysView
+from dataclasses import dataclass
 from datetime import date
 import enum
 import functools
@@ -21,6 +22,7 @@
 from pathlib import Path
 import re
 import sys
+import textwrap
 from typing import Any, Callable, TextIO, TypeAlias
 
 
@@ -115,6 +117,64 @@ def save_raw_data(data: RawData, json_output: TextIO):
 json.dump(data, json_output)
 
 
+@dataclass(frozen=True)
+class Doc:
+text: str
+doc: str
+
+def markdown(self) -> str:
+return textwrap.dedent(
+f"""
+{self.text}
+
+ⓘ
+
+{self.doc}
+
+"""
+)
+
+
+class Count(int):
+def markdown(self) -> str:
+return format(self, ",d")
+
+
+@dataclass(frozen=True)
+class Ratio:
+num: int
+den: int | None = None
+percentage: bool = True
+
+def __float__(self):
+if self.den == 0:
+return 0.0
+elif self.den is None:
+return self.num
+else:
+return self.num / self.den
+
+def markdown(self) -> str:
+if self.den is None:
+return ""
+elif self.den == 0:
+if self.num != 0:
+return f"{self.num:,} / 0 !!"
+return ""
+elif self.percentage:
+return f"{self.num / self.den:,.01%}"
+else:
+return f"{self.num / self.den:,.02f}"
+
+
+class DiffRatio(Ratio):
+def __init__(self, base: int | str, head: int | str):
+if isinstance(base, str) or isinstance(head, str):
+super().__init__(0, 0)
+else:
+super().__init__(head - base, base)
+
+
 class OpcodeStats:
 """
 Manages the data related to specific set of opcodes, e.g. tier1 (with 
prefix
@@ -389,17 +449,54 @@ def get_optimization_stats(self) -> dict[str, tuple[int, 
int | None]]:
 low_confidence = self._data["Optimization low confidence"]
 
 return {
-"Optimization attempts": (attempts, None),
-"Traces created": (created, attempts),
-"Trace stack overflow": (trace_stack_overflow, attempts),
-"Trace stack underflow": (trace_stack_underflow, attempts),
-"Trace too long": (trace_too_long, attempts),
-"Trace too short": (trace_too_short, attempts),
-"Inner loop found": (inner_loop, attempts),
-"Recursive call": (recursive_call, attempts),
-"Low confidence": (low_confidence, attempts),
-"Traces executed": (executed, None),
-"Uops executed": (uops, executed),
+Doc(
+"Optimization attempts",
+"The number of times a potential trace is identified.  
Specifically, this "
+"occurs in the JUMP BACKWARD instruction when the counter 
reaches a "
+"threshold.",
+): (
+attempts,
+None,
+),
+Doc(
+"Traces created", "The number of traces that were successfully 
created."
+): (created, attempts),
+Doc(
+"Trace stack overflow",
+"A trace is truncated because it would require more than 5 
stack frames.",
+): (trace_stack_overflow, attempts),
+Doc(
+"Trace stack underflow",
+"A potential trace is abandoned because it pops more frames 
than it pushes.",
+): (trace_stack_underflow, attempts),
+Doc(
+"Trace too long",
+"A trace is truncated because it is longer than the 
instruction buffer.",
+): (trace_too_long, attempts),
+Doc(
+"Trace too short",
+"A potential trace is abandoced because it it too short.",
+): (trace_too_short, attempts),
+Doc(
+"Inner loop found", "A trace is truncated because it has an 
inner loop"
+): (inner_loop, attempts),
+Doc(
+"Recursive call",
+"A trace is truncated because it has a recursive call.",
+): (recursive_call, attempts),
+Doc(
+"Low confidence",
+"A trace is abandoned because the likelihood of the jump to 
top being taken "
+"is too low.",
+): (

[Python-checkins] gh-115480: Type / constant propagation for float binary uops (GH-115550)

2024-02-16 Thread Fidget-Spinner 
https://github.com/python/cpython/commit/13addd2bbdcbf96c5ea26a0f425c049f1b71e945
commit: 13addd2bbdcbf96c5ea26a0f425c049f1b71e945
branch: main
author: Peter Lazorchak 
committer: Fidget-Spinner 
date: 2024-02-17T02:02:48+08:00
summary:

gh-115480: Type / constant propagation for float binary uops (GH-115550)

Co-authored-by: Ken Jin 

files:
M Lib/test/test_capi/test_opt.py
M Python/tier2_redundancy_eliminator_bytecodes.c
M Python/tier2_redundancy_eliminator_cases.c.h

diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py
index 1a8ed3441fa855..66860c67966859 100644
--- a/Lib/test/test_capi/test_opt.py
+++ b/Lib/test/test_capi/test_opt.py
@@ -561,6 +561,16 @@ def testfunc(n):
 
 class TestUopsOptimization(unittest.TestCase):
 
+def _run_with_optimizer(self, testfunc, arg):
+res = None
+opt = _testinternalcapi.get_uop_optimizer()
+with temporary_optimizer(opt):
+res = testfunc(arg)
+
+ex = get_first_executor(testfunc)
+return res, ex
+
+
 def test_int_type_propagation(self):
 def testfunc(loops):
 num = 0
@@ -570,12 +580,7 @@ def testfunc(loops):
 num += 1
 return a
 
-opt = _testinternalcapi.get_uop_optimizer()
-res = None
-with temporary_optimizer(opt):
-res = testfunc(32)
-
-ex = get_first_executor(testfunc)
+res, ex = self._run_with_optimizer(testfunc, 32)
 self.assertIsNotNone(ex)
 self.assertEqual(res, 63)
 binop_count = [opname for opname, _, _ in ex if opname == 
"_BINARY_OP_ADD_INT"]
@@ -642,12 +647,7 @@ def testfunc(loops):
 num += 1
 return a
 
-opt = _testinternalcapi.get_uop_optimizer()
-res = None
-with temporary_optimizer(opt):
-res = testfunc(64)
-
-ex = get_first_executor(testfunc)
+res, ex = self._run_with_optimizer(testfunc, 64)
 self.assertIsNotNone(ex)
 binop_count = [opname for opname, _, _ in ex if opname == 
"_BINARY_OP_ADD_INT"]
 self.assertGreaterEqual(len(binop_count), 3)
@@ -659,11 +659,7 @@ def dummy(x):
 for i in range(n):
 dummy(i)
 
-opt = _testinternalcapi.get_uop_optimizer()
-with temporary_optimizer(opt):
-testfunc(32)
-
-ex = get_first_executor(testfunc)
+res, ex = self._run_with_optimizer(testfunc, 32)
 self.assertIsNotNone(ex)
 uops = {opname for opname, _, _ in ex}
 self.assertIn("_PUSH_FRAME", uops)
@@ -677,11 +673,7 @@ def testfunc(n):
 x = i + i
 return x
 
-opt = _testinternalcapi.get_uop_optimizer()
-with temporary_optimizer(opt):
-res = testfunc(32)
-
-ex = get_first_executor(testfunc)
+res, ex = self._run_with_optimizer(testfunc, 32)
 self.assertEqual(res, 62)
 self.assertIsNotNone(ex)
 uops = {opname for opname, _, _ in ex}
@@ -699,11 +691,7 @@ def testfunc(n):
 res = x + z + a + b
 return res
 
-opt = _testinternalcapi.get_uop_optimizer()
-with temporary_optimizer(opt):
-res = testfunc(32)
-
-ex = get_first_executor(testfunc)
+res, ex = self._run_with_optimizer(testfunc, 32)
 self.assertEqual(res, 4)
 self.assertIsNotNone(ex)
 uops = {opname for opname, _, _ in ex}
@@ -716,11 +704,8 @@ def testfunc(n):
 for _ in range(n):
 return [i for i in range(n)]
 
-opt = _testinternalcapi.get_uop_optimizer()
-with temporary_optimizer(opt):
-testfunc(32)
-
-ex = get_first_executor(testfunc)
+res, ex = self._run_with_optimizer(testfunc, 32)
+self.assertEqual(res, list(range(32)))
 self.assertIsNotNone(ex)
 uops = {opname for opname, _, _ in ex}
 self.assertNotIn("_BINARY_OP_ADD_INT", uops)
@@ -785,6 +770,56 @@ def testfunc(n):
 """))
 self.assertEqual(result[0].rc, 0, result)
 
+def test_float_add_constant_propagation(self):
+def testfunc(n):
+a = 1.0
+for _ in range(n):
+a = a + 0.1
+return a
+
+res, ex = self._run_with_optimizer(testfunc, 32)
+self.assertAlmostEqual(res, 4.2)
+self.assertIsNotNone(ex)
+uops = {opname for opname, _, _ in ex}
+guard_both_float_count = [opname for opname, _, _ in ex if opname == 
"_GUARD_BOTH_FLOAT"]
+self.assertLessEqual(len(guard_both_float_count), 1)
+# TODO gh-115506: this assertion may change after propagating 
constants.
+# We'll also need to verify that propagation actually occurs.
+self.assertIn("_BINARY_OP_ADD_FLOAT", uops)
+
+def test_float_subtract_constant_propagation(self):
+def testfunc(n):
+a = 1.0
+for _ in range(n):
+a = a - 0.1
+

[Python-checkins] gh-114271: Make `thread._rlock` thread-safe in free-threaded builds (#115102)

2024-02-16 Thread colesbury 
https://github.com/python/cpython/commit/f366e215044e348659df814c27bf70e78907df21
commit: f366e215044e348659df814c27bf70e78907df21
branch: main
author: mpage 
committer: colesbury 
date: 2024-02-16T13:29:25-05:00
summary:

gh-114271: Make `thread._rlock` thread-safe in free-threaded builds (#115102)

The ID of the owning thread (`rlock_owner`) may be accessed by
multiple threads without holding the underlying lock; relaxed
atomics are used in place of the previous loads/stores.

The number of times that the lock has been acquired (`rlock_count`)
is only ever accessed by the thread that holds the lock; we do not
need to use atomics to access it.

files:
M Include/cpython/pyatomic.h
M Include/cpython/pyatomic_gcc.h
M Include/cpython/pyatomic_msc.h
M Include/cpython/pyatomic_std.h
M Modules/_threadmodule.c

diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h
index e10d48285367cf..9b5774190ee99e 100644
--- a/Include/cpython/pyatomic.h
+++ b/Include/cpython/pyatomic.h
@@ -360,6 +360,8 @@ _Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj);
 static inline void *
 _Py_atomic_load_ptr_relaxed(const void *obj);
 
+static inline unsigned long long
+_Py_atomic_load_ullong_relaxed(const unsigned long long *obj);
 
 // --- _Py_atomic_store --
 // Atomically performs `*obj = value` (sequential consistency)
@@ -452,6 +454,10 @@ _Py_atomic_store_ptr_relaxed(void *obj, void *value);
 static inline void
 _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value);
 
+static inline void
+_Py_atomic_store_ullong_relaxed(unsigned long long *obj,
+unsigned long long value);
+
 
 // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release 
 
diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h
index 4095e1873d8b07..bc74149f73e83c 100644
--- a/Include/cpython/pyatomic_gcc.h
+++ b/Include/cpython/pyatomic_gcc.h
@@ -358,6 +358,10 @@ static inline void *
 _Py_atomic_load_ptr_relaxed(const void *obj)
 { return (void *)__atomic_load_n((const void **)obj, __ATOMIC_RELAXED); }
 
+static inline unsigned long long
+_Py_atomic_load_ullong_relaxed(const unsigned long long *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
 
 // --- _Py_atomic_store --
 
@@ -476,6 +480,11 @@ static inline void
 _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value)
 { __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
 
+static inline void
+_Py_atomic_store_ullong_relaxed(unsigned long long *obj,
+unsigned long long value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
 
 // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release 
 
diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h
index b5c1ec94112562..6ab6401cf81e8a 100644
--- a/Include/cpython/pyatomic_msc.h
+++ b/Include/cpython/pyatomic_msc.h
@@ -712,6 +712,12 @@ _Py_atomic_load_ptr_relaxed(const void *obj)
 return *(void * volatile *)obj;
 }
 
+static inline unsigned long long
+_Py_atomic_load_ullong_relaxed(const unsigned long long *obj)
+{
+return *(volatile unsigned long long *)obj;
+}
+
 
 // --- _Py_atomic_store --
 
@@ -886,6 +892,14 @@ _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t 
value)
 *(volatile Py_ssize_t *)obj = value;
 }
 
+static inline void
+_Py_atomic_store_ullong_relaxed(unsigned long long *obj,
+unsigned long long value)
+{
+*(volatile unsigned long long *)obj = value;
+}
+
+
 // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release 
 
 static inline void *
diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h
index 6c934a2c5e7b64..d3004dbd24ed09 100644
--- a/Include/cpython/pyatomic_std.h
+++ b/Include/cpython/pyatomic_std.h
@@ -619,6 +619,14 @@ _Py_atomic_load_ptr_relaxed(const void *obj)
 memory_order_relaxed);
 }
 
+static inline unsigned long long
+_Py_atomic_load_ullong_relaxed(const unsigned long long *obj)
+{
+_Py_USING_STD;
+return atomic_load_explicit((const _Atomic(unsigned long long)*)obj,
+memory_order_relaxed);
+}
+
 
 // --- _Py_atomic_store --
 
@@ -835,6 +843,15 @@ _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t 
value)
   memory_order_relaxed);
 }
 
+static inline void
+_Py_atomic_store_ullong_relaxed(unsigned long long *obj,
+unsigned long long value)
+{
+_Py_USING_STD;
+atomic_store_explicit((_Atomic(unsigned long long)*)obj, value,
+  memory_order_relaxed);
+}
+
 
 // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release 
 
diff --git a/Modules/_threadmodule.c b/Modul

[Python-checkins] gh-112720: make it easier to subclass and modify dis.ArgResolver's jump arg resolution (#115564)

2024-02-16 Thread iritkatriel 
https://github.com/python/cpython/commit/74e6f4b32fceea8e8ffb0424d9bdc6589faf7ee4
commit: 74e6f4b32fceea8e8ffb0424d9bdc6589faf7ee4
branch: main
author: Irit Katriel <[email protected]>
committer: iritkatriel <[email protected]>
date: 2024-02-16T19:25:19Z
summary:

gh-112720: make it easier to subclass and modify dis.ArgResolver's jump arg 
resolution (#115564)

files:
A Misc/NEWS.d/next/Library/2024-02-16-16-40-10.gh-issue-112720.io6_Ac.rst
M Lib/dis.py
M Lib/test/test_dis.py

diff --git a/Lib/dis.py b/Lib/dis.py
index f05ea1a24f45a7..d146bcbb5097ef 100644
--- a/Lib/dis.py
+++ b/Lib/dis.py
@@ -505,6 +505,20 @@ def __init__(self, co_consts=None, names=None, 
varname_from_oparg=None, labels_m
 self.varname_from_oparg = varname_from_oparg
 self.labels_map = labels_map or {}
 
+def offset_from_jump_arg(self, op, arg, offset):
+deop = _deoptop(op)
+if deop in hasjabs:
+return arg * 2
+elif deop in hasjrel:
+signed_arg = -arg if _is_backward_jump(deop) else arg
+argval = offset + 2 + signed_arg*2
+caches = _get_cache_size(_all_opname[deop])
+argval += 2 * caches
+if deop == ENTER_EXECUTOR:
+argval += 2
+return argval
+return None
+
 def get_label_for_offset(self, offset):
 return self.labels_map.get(offset, None)
 
@@ -536,17 +550,11 @@ def get_argval_argrepr(self, op, arg, offset):
 argrepr = f"{argrepr} + NULL|self"
 else:
 argval, argrepr = _get_name_info(arg, get_name)
-elif deop in hasjabs:
-argval = arg*2
-argrepr = f"to L{self.labels_map[argval]}"
-elif deop in hasjrel:
-signed_arg = -arg if _is_backward_jump(deop) else arg
-argval = offset + 2 + signed_arg*2
-caches = _get_cache_size(_all_opname[deop])
-argval += 2 * caches
-if deop == ENTER_EXECUTOR:
-argval += 2
-argrepr = f"to L{self.labels_map[argval]}"
+elif deop in hasjump or deop in hasexc:
+argval = self.offset_from_jump_arg(op, arg, offset)
+lbl = self.get_label_for_offset(argval)
+assert lbl is not None
+argrepr = f"to L{lbl}"
 elif deop in (LOAD_FAST_LOAD_FAST, STORE_FAST_LOAD_FAST, 
STORE_FAST_STORE_FAST):
 arg1 = arg >> 4
 arg2 = arg & 15
diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py
index a5917da346dded..a93cb509b651c5 100644
--- a/Lib/test/test_dis.py
+++ b/Lib/test/test_dis.py
@@ -1986,6 +1986,22 @@ def f(opcode, oparg, offset, *init_args):
 self.assertEqual(f(opcode.opmap["BINARY_OP"], 3, *args), (3, '<<'))
 self.assertEqual(f(opcode.opmap["CALL_INTRINSIC_1"], 2, *args), (2, 
'INTRINSIC_IMPORT_STAR'))
 
+def test_custom_arg_resolver(self):
+class MyArgResolver(dis.ArgResolver):
+def offset_from_jump_arg(self, op, arg, offset):
+return arg + 1
+
+def get_label_for_offset(self, offset):
+return 2 * offset
+
+def f(opcode, oparg, offset, *init_args):
+arg_resolver = MyArgResolver(*init_args)
+return arg_resolver.get_argval_argrepr(opcode, oparg, offset)
+offset = 42
+self.assertEqual(f(opcode.opmap["JUMP_BACKWARD"], 1, offset), (2, 'to 
L4'))
+self.assertEqual(f(opcode.opmap["SETUP_FINALLY"], 2, offset), (3, 'to 
L6'))
+
+
 def get_instructions(self, code):
 return dis._get_instructions_bytes(code)
 
diff --git 
a/Misc/NEWS.d/next/Library/2024-02-16-16-40-10.gh-issue-112720.io6_Ac.rst 
b/Misc/NEWS.d/next/Library/2024-02-16-16-40-10.gh-issue-112720.io6_Ac.rst
new file mode 100644
index 00..32916ede4dee35
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-02-16-16-40-10.gh-issue-112720.io6_Ac.rst
@@ -0,0 +1,2 @@
+Refactor :class:`dis.ArgResolver` to make it possible to subclass and change
+the way jump args are interpreted.

___
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]

[Python-checkins] [3.11] gh-115570: Fix DeprecationWarnings in test_typing (#115571)

2024-02-16 Thread JelleZijlstra 
https://github.com/python/cpython/commit/c01a4808d3d212bf9833306293e4d5806d941651
commit: c01a4808d3d212bf9833306293e4d5806d941651
branch: 3.11
author: Jelle Zijlstra 
committer: JelleZijlstra 
date: 2024-02-16T19:37:42Z
summary:

[3.11] gh-115570: Fix DeprecationWarnings in test_typing (#115571)

Co-authored-by: Alex Waygood 

files:
A Misc/NEWS.d/next/Library/2024-02-16-10-18-25.gh-issue-115570.bI6uu3.rst
M Lib/test/test_typing.py
M Lib/typing.py

diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py
index 8d6d693ed08437..287e566a9b67d6 100644
--- a/Lib/test/test_typing.py
+++ b/Lib/test/test_typing.py
@@ -7319,6 +7319,17 @@ def test_re_submodule(self):
 self.assertEqual(__name__, 'typing.re')
 self.assertEqual(len(w), 1)
 
+def test_re_submodule_access_basics(self):
+with warnings.catch_warnings():
+warnings.filterwarnings("error", category=DeprecationWarning)
+from typing import re
+self.assertIsInstance(re.__doc__, str)
+self.assertEqual(re.__name__, "typing.re")
+self.assertIsInstance(re.__dict__, types.MappingProxyType)
+
+with self.assertWarns(DeprecationWarning):
+re.Match
+
 def test_cannot_subclass(self):
 with self.assertRaises(TypeError) as ex:
 
diff --git a/Lib/typing.py b/Lib/typing.py
index c62993dbd515f8..b04ab854e105a4 100644
--- a/Lib/typing.py
+++ b/Lib/typing.py
@@ -3382,7 +3382,7 @@ def __enter__(self) -> 'TextIO':
 
 class _DeprecatedType(type):
 def __getattribute__(cls, name):
-if name not in ("__dict__", "__module__") and name in cls.__dict__:
+if name not in {"__dict__", "__module__", "__doc__"} and name in 
cls.__dict__:
 warnings.warn(
 f"{cls.__name__} is deprecated, import directly "
 f"from typing instead. {cls.__name__} will be removed "
diff --git 
a/Misc/NEWS.d/next/Library/2024-02-16-10-18-25.gh-issue-115570.bI6uu3.rst 
b/Misc/NEWS.d/next/Library/2024-02-16-10-18-25.gh-issue-115570.bI6uu3.rst
new file mode 100644
index 00..f3c8a11380a283
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-02-16-10-18-25.gh-issue-115570.bI6uu3.rst
@@ -0,0 +1,3 @@
+A :exc:`DeprecationWarning` is no longer omitted on access to the
+``__doc__`` attributes of the deprecated ``typing.io`` and ``typing.re``
+pseudo-modules.

___
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]

[Python-checkins] gh-115556: Remove quotes from command-line arguments in test.bat and rt.bat (#115557)

2024-02-16 Thread ambv 
https://github.com/python/cpython/commit/711f42de2e3749208cfa7effa0d45b04e4e1fdd4
commit: 711f42de2e3749208cfa7effa0d45b04e4e1fdd4
branch: main
author: Łukasz Langa 
committer: ambv 
date: 2024-02-16T21:24:56+01:00
summary:

gh-115556: Remove quotes from command-line arguments in test.bat and rt.bat 
(#115557)

This change essentially replaces usage of `%1` with `%~1`, which removes
quotes, if any. Without this change, the if statements fail due to
the quotes mangling the syntax.

Additionally, this change works around comma being treated as a parameter
delimiter in test.bat by escaping commas at time of parsing. Tested
combinations of rt and regrtest arguments, all seems to work as before
but now you can specify commas in arguments like "-uall,extralargefile".

files:
A Misc/NEWS.d/next/Tests/2024-02-16-13-04-28.gh-issue-115556.rjaQ9w.rst
M PCbuild/rt.bat
M Tools/buildbot/test.bat

diff --git 
a/Misc/NEWS.d/next/Tests/2024-02-16-13-04-28.gh-issue-115556.rjaQ9w.rst 
b/Misc/NEWS.d/next/Tests/2024-02-16-13-04-28.gh-issue-115556.rjaQ9w.rst
new file mode 100644
index 00..c2811b133d9314
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2024-02-16-13-04-28.gh-issue-115556.rjaQ9w.rst
@@ -0,0 +1,2 @@
+On Windows, commas passed in arguments to ``Tools\buildbot\test.bat`` and
+``PCbuild\\rt.bat`` are now properly handled.
diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat
index 293f99ae135faa..ac530a5206271f 100644
--- a/PCbuild/rt.bat
+++ b/PCbuild/rt.bat
@@ -38,18 +38,18 @@ set regrtestargs=--fast-ci
 set exe=
 
 :CheckOpts
-if "%1"=="-O" (set dashO=-O) & shift & goto CheckOpts
-if "%1"=="-q" (set qmode=yes)& shift & goto CheckOpts
-if "%1"=="-d" (set suffix=_d)& shift & goto CheckOpts
+if "%~1"=="-O" (set dashO=-O) & shift & goto CheckOpts
+if "%~1"=="-q" (set qmode=yes)& shift & goto CheckOpts
+if "%~1"=="-d" (set suffix=_d)& shift & goto CheckOpts
 rem HACK: Need some way to infer the version number in this script
-if "%1"=="--disable-gil" (set pyname=python3.13t) & shift & goto CheckOpts
-if "%1"=="-win32" (set prefix=%pcbuild%win32) & shift & goto CheckOpts
-if "%1"=="-x64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts
-if "%1"=="-amd64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts
-if "%1"=="-arm64" (set prefix=%pcbuild%arm64) & shift & goto CheckOpts
-if "%1"=="-arm32" (set prefix=%pcbuild%arm32) & shift & goto CheckOpts
-if "%1"=="-p" (call :SetPlatform %~2) & shift & shift & goto CheckOpts
-if NOT "%1"=="" (set regrtestargs=%regrtestargs% %1) & shift & goto CheckOpts
+if "%~1"=="--disable-gil" (set pyname=python3.13t) & shift & goto CheckOpts
+if "%~1"=="-win32" (set prefix=%pcbuild%win32) & shift & goto CheckOpts
+if "%~1"=="-x64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts
+if "%~1"=="-amd64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts
+if "%~1"=="-arm64" (set prefix=%pcbuild%arm64) & shift & goto CheckOpts
+if "%~1"=="-arm32" (set prefix=%pcbuild%arm32) & shift & goto CheckOpts
+if "%~1"=="-p" (call :SetPlatform %~2) & shift & shift & goto CheckOpts
+if NOT "%~1"=="" (set regrtestargs=%regrtestargs% %~1) & shift & goto CheckOpts
 
 if not defined prefix set prefix=%pcbuild%amd64
 set exe=%prefix%\%pyname%%suffix%.exe
diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat
index 781f9a4c8206c8..0c47470a0ecb7a 100644
--- a/Tools/buildbot/test.bat
+++ b/Tools/buildbot/test.bat
@@ -7,17 +7,10 @@ set here=%~dp0
 set rt_opts=-q -d
 set regrtest_args=
 set arm32_ssh=
+set cmdline_args=%*
+set cmdline_args=%cmdline_args:,=#COMMA#%
 
-:CheckOpts
-if "%1"=="-x64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts
-if "%1"=="-arm64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts
-if "%1"=="-arm32" (set rt_opts=%rt_opts% %1) & (set arm32_ssh=true) & shift & 
goto CheckOpts
-if "%1"=="-d" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts
-if "%1"=="-O" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts
-if "%1"=="-q" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts
-if "%1"=="+d" (set rt_opts=%rt_opts:-d=%) & shift & goto CheckOpts
-if "%1"=="+q" (set rt_opts=%rt_opts:-q=%) & shift & goto CheckOpts
-if NOT "%1"=="" (set regrtest_args=%regrtest_args% %1) & shift & goto CheckOpts
+call:CheckOpts %cmdline_args%
 
 if "%PROCESSOR_ARCHITECTURE%"=="ARM" if "%arm32_ssh%"=="true" goto 
NativeExecution
 if "%arm32_ssh%"=="true" goto :Arm32Ssh
@@ -49,3 +42,16 @@ echo The test worker should have the SSH agent running.
 echo Also a key must be created with ssh-keygen and added to both the buildbot 
worker machine
 echo and the ARM32 worker device: see 
https://docs.microsoft.com/en-us/windows/iot-core/connect-your-device/ssh
 exit /b 127
+
+:CheckOpts
+set arg="%~1"
+if %arg%=="-x64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts
+if %arg%=="-arm64" (set rt_opts=%rt_opts% %1) & shift & goto CheckOpts
+if %arg%=="-arm32" (set rt_opts=%rt_opts% %1) & (set arm32_ssh=true) & shift & 
goto CheckOpts
+if %arg%=="-d" (set rt_opts=%rt_opts% %1) & shift & g

[Python-checkins] gh-115103: Implement delayed memory reclamation (QSBR) (#115180)

2024-02-16 Thread colesbury 
https://github.com/python/cpython/commit/590319072773bd6cdcca655c420d3adb84838e96
commit: 590319072773bd6cdcca655c420d3adb84838e96
branch: main
author: Sam Gross 
committer: colesbury 
date: 2024-02-16T15:25:19-05:00
summary:

gh-115103: Implement delayed memory reclamation (QSBR) (#115180)

This adds a safe memory reclamation scheme based on FreeBSD's "GUS" and
quiescent state based reclamation (QSBR). The API provides a mechanism
for callers to detect when it is safe to free memory that may be
concurrently accessed by readers.

files:
A Include/internal/pycore_qsbr.h
A Python/qsbr.c
M Doc/license.rst
M Include/cpython/pyatomic.h
M Include/cpython/pyatomic_gcc.h
M Include/cpython/pyatomic_msc.h
M Include/cpython/pyatomic_std.h
M Include/internal/pycore_interp.h
M Include/internal/pycore_runtime_init.h
M Include/internal/pycore_tstate.h
M Makefile.pre.in
M Modules/posixmodule.c
M PCbuild/_freeze_module.vcxproj
M PCbuild/_freeze_module.vcxproj.filters
M PCbuild/pythoncore.vcxproj
M PCbuild/pythoncore.vcxproj.filters
M Python/ceval_macros.h
M Python/pystate.c

diff --git a/Doc/license.rst b/Doc/license.rst
index 9fc0ff7161a591..cbe918bd1acfe3 100644
--- a/Doc/license.rst
+++ b/Doc/license.rst
@@ -1095,3 +1095,35 @@ which is distributed under the MIT license::
   LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
   OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
   WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+Global Unbounded Sequences (GUS)
+
+
+The file :file:`Python/qsbr.c` is adapted from FreeBSD's "Global Unbounded
+Sequences" safe memory reclamation scheme in
+`subr_smr.c 
`_.
+The file is distributed under the 2-Clause BSD License::
+
+  Copyright (c) 2019,2020 Jeffrey Roberson 
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions
+  are met:
+  1. Redistributions of source code must retain the above copyright
+ notice unmodified, this list of conditions, and the following
+ disclaimer.
+  2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h
index 9b5774190ee99e..737eed8b12dd81 100644
--- a/Include/cpython/pyatomic.h
+++ b/Include/cpython/pyatomic.h
@@ -475,6 +475,12 @@ _Py_atomic_store_int_release(int *obj, int value);
 static inline int
 _Py_atomic_load_int_acquire(const int *obj);
 
+static inline void
+_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value);
+
+static inline uint64_t
+_Py_atomic_load_uint64_acquire(const uint64_t *obj);
+
 static inline uint32_t
 _Py_atomic_load_uint32_acquire(const uint32_t *obj);
 
diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h
index bc74149f73e83c..de23edfc6877d2 100644
--- a/Include/cpython/pyatomic_gcc.h
+++ b/Include/cpython/pyatomic_gcc.h
@@ -504,6 +504,14 @@ static inline int
 _Py_atomic_load_int_acquire(const int *obj)
 { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); }
 
+static inline void
+_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); }
+
+static inline uint64_t
+_Py_atomic_load_uint64_acquire(const uint64_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); }
+
 static inline uint32_t
 _Py_atomic_load_uint32_acquire(const uint32_t *obj)
 { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); }
diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h
index 6ab6401cf81e8a..9809d9806d7b57 100644
--- a/Include/cpython/pyatomic_msc.h
+++ b/Include/cpython/pyatomic_msc.h
@@ -952,13 +952,39 @@ _Py_atomic_load_int_acquire(const int *obj)
 #endif
 }
 
+static inline void
+_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+*(uint64_t volatile *)obj = value;
+#elif defined(_M_ARM64)
+_Py_atomic_ASSERT_ARG_TYPE(unsigned __int64);
+__stlr64((unsigned __int64 volat

[Python-checkins] gh-85294: Handle missing arguments to @singledispatchmethod gracefully (GH-21471)

2024-02-16 Thread serhiy-storchaka 
https://github.com/python/cpython/commit/8b776e0f41d7711f3e2be2435bf85f2d5fa6e009
commit: 8b776e0f41d7711f3e2be2435bf85f2d5fa6e009
branch: main
author: Ammar Askar 
committer: serhiy-storchaka 
date: 2024-02-16T23:17:30+02:00
summary:

gh-85294: Handle missing arguments to @singledispatchmethod gracefully 
(GH-21471)

Co-authored-by: Serhiy Storchaka 

files:
A Misc/NEWS.d/next/Library/2020-07-13-23-59-42.bpo-41122.8P_Brh.rst
M Lib/functools.py
M Lib/test/test_functools.py

diff --git a/Lib/functools.py b/Lib/functools.py
index ee4197b386178d..7045be551c8c49 100644
--- a/Lib/functools.py
+++ b/Lib/functools.py
@@ -918,7 +918,6 @@ def wrapper(*args, **kw):
 if not args:
 raise TypeError(f'{funcname} requires at least '
 '1 positional argument')
-
 return dispatch(args[0].__class__)(*args, **kw)
 
 funcname = getattr(func, '__name__', 'singledispatch function')
@@ -968,7 +967,11 @@ def __get__(self, obj, cls=None):
 return _method
 
 dispatch = self.dispatcher.dispatch
+funcname = getattr(self.func, '__name__', 'singledispatchmethod 
method')
 def _method(*args, **kwargs):
+if not args:
+raise TypeError(f'{funcname} requires at least '
+'1 positional argument')
 return dispatch(args[0].__class__).__get__(obj, cls)(*args, 
**kwargs)
 
 _method.__isabstractmethod__ = self.__isabstractmethod__
diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py
index 7c66b906d308ba..2c814d5e40 100644
--- a/Lib/test/test_functools.py
+++ b/Lib/test/test_functools.py
@@ -2867,11 +2867,26 @@ def _(arg: typing.Union[int, typing.Iterable[str]]):
 
 def test_invalid_positional_argument(self):
 @functools.singledispatch
-def f(*args):
+def f(*args, **kwargs):
 pass
 msg = 'f requires at least 1 positional argument'
 with self.assertRaisesRegex(TypeError, msg):
 f()
+msg = 'f requires at least 1 positional argument'
+with self.assertRaisesRegex(TypeError, msg):
+f(a=1)
+
+def test_invalid_positional_argument_singledispatchmethod(self):
+class A:
[email protected]
+def t(self, *args, **kwargs):
+pass
+msg = 't requires at least 1 positional argument'
+with self.assertRaisesRegex(TypeError, msg):
+A().t()
+msg = 't requires at least 1 positional argument'
+with self.assertRaisesRegex(TypeError, msg):
+A().t(a=1)
 
 def test_union(self):
 @functools.singledispatch
diff --git a/Misc/NEWS.d/next/Library/2020-07-13-23-59-42.bpo-41122.8P_Brh.rst 
b/Misc/NEWS.d/next/Library/2020-07-13-23-59-42.bpo-41122.8P_Brh.rst
new file mode 100644
index 00..76568d407449f5
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2020-07-13-23-59-42.bpo-41122.8P_Brh.rst
@@ -0,0 +1,3 @@
+Failing to pass arguments properly to :func:`functools.singledispatchmethod`
+now throws a TypeError instead of hitting an index out of bounds
+internally.

___
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]

[Python-checkins] [3.12] gh-115570: Fix DeprecationWarnings in test_typing (#115571) (#115574)

2024-02-16 Thread JelleZijlstra 
https://github.com/python/cpython/commit/83c7dd5200bae16d2f2f942fb0a826723d3023cc
commit: 83c7dd5200bae16d2f2f942fb0a826723d3023cc
branch: 3.12
author: Jelle Zijlstra 
committer: JelleZijlstra 
date: 2024-02-16T13:46:48-08:00
summary:

[3.12] gh-115570: Fix DeprecationWarnings in test_typing (#115571) (#115574)

Co-authored-by: Alex Waygood 

files:
A Misc/NEWS.d/next/Library/2024-02-16-10-18-25.gh-issue-115570.bI6uu3.rst
M Lib/test/test_typing.py
M Lib/typing.py

diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py
index ab9d8a0a20ce7d..7f9c10dd2a54e6 100644
--- a/Lib/test/test_typing.py
+++ b/Lib/test/test_typing.py
@@ -8123,6 +8123,17 @@ def test_re_submodule(self):
 self.assertEqual(__name__, 'typing.re')
 self.assertEqual(len(w), 1)
 
+def test_re_submodule_access_basics(self):
+with warnings.catch_warnings():
+warnings.filterwarnings("error", category=DeprecationWarning)
+from typing import re
+self.assertIsInstance(re.__doc__, str)
+self.assertEqual(re.__name__, "typing.re")
+self.assertIsInstance(re.__dict__, types.MappingProxyType)
+
+with self.assertWarns(DeprecationWarning):
+re.Match
+
 def test_cannot_subclass(self):
 with self.assertRaisesRegex(
 TypeError,
diff --git a/Lib/typing.py b/Lib/typing.py
index cbd4ce8bfaa6bb..1e4c725be473b7 100644
--- a/Lib/typing.py
+++ b/Lib/typing.py
@@ -3256,11 +3256,11 @@ def __enter__(self) -> 'TextIO':
 
 class _DeprecatedType(type):
 def __getattribute__(cls, name):
-if name not in ("__dict__", "__module__") and name in cls.__dict__:
+if name not in {"__dict__", "__module__", "__doc__"} and name in 
cls.__dict__:
 warnings.warn(
 f"{cls.__name__} is deprecated, import directly "
 f"from typing instead. {cls.__name__} will be removed "
-"in Python 3.12.",
+"in Python 3.13.",
 DeprecationWarning,
 stacklevel=2,
 )
diff --git 
a/Misc/NEWS.d/next/Library/2024-02-16-10-18-25.gh-issue-115570.bI6uu3.rst 
b/Misc/NEWS.d/next/Library/2024-02-16-10-18-25.gh-issue-115570.bI6uu3.rst
new file mode 100644
index 00..f3c8a11380a283
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-02-16-10-18-25.gh-issue-115570.bI6uu3.rst
@@ -0,0 +1,3 @@
+A :exc:`DeprecationWarning` is no longer omitted on access to the
+``__doc__`` attributes of the deprecated ``typing.io`` and ``typing.re``
+pseudo-modules.

___
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]

[Python-checkins] docs: Add glossary term references to shutil docs (#115559)

2024-02-16 Thread CAM-Gerlach 
https://github.com/python/cpython/commit/318f2190bc93796008b0a4241243b0851b418436
commit: 318f2190bc93796008b0a4241243b0851b418436
branch: main
author: Brian Schubert 
committer: CAM-Gerlach 
date: 2024-02-16T16:04:17-06:00
summary:

docs: Add glossary term references to shutil docs (#115559)

Add glossary term references to shutil docs

files:
M Doc/library/shutil.rst

diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst
index ff8c9a189ab3de..f388375045c912 100644
--- a/Doc/library/shutil.rst
+++ b/Doc/library/shutil.rst
@@ -39,7 +39,7 @@ Directory and files operations
 
 .. function:: copyfileobj(fsrc, fdst[, length])
 
-   Copy the contents of the file-like object *fsrc* to the file-like object 
*fdst*.
+   Copy the contents of the :term:`file-like object ` *fsrc* to 
the file-like object *fdst*.
The integer *length*, if given, is the buffer size. In particular, a 
negative
*length* value means to copy the data without looping over the source data 
in
chunks; by default the data is read in chunks to avoid uncontrolled memory
@@ -52,7 +52,7 @@ Directory and files operations
 
Copy the contents (no metadata) of the file named *src* to a file named
*dst* and return *dst* in the most efficient way possible.
-   *src* and *dst* are path-like objects or path names given as strings.
+   *src* and *dst* are :term:`path-like objects ` or path 
names given as strings.
 
*dst* must be the complete target file name; look at :func:`~shutil.copy`
for a copy that accepts a target directory path.  If *src* and *dst*
@@ -94,7 +94,7 @@ Directory and files operations
 .. function:: copymode(src, dst, *, follow_symlinks=True)
 
Copy the permission bits from *src* to *dst*.  The file contents, owner, and
-   group are unaffected.  *src* and *dst* are path-like objects or path names
+   group are unaffected.  *src* and *dst* are :term:`path-like objects 
` or path names
given as strings.
If *follow_symlinks* is false, and both *src* and *dst* are symbolic links,
:func:`copymode` will attempt to modify the mode of *dst* itself (rather
@@ -113,7 +113,7 @@ Directory and files operations
Copy the permission bits, last access time, last modification time, and
flags from *src* to *dst*.  On Linux, :func:`copystat` also copies the
"extended attributes" where possible.  The file contents, owner, and
-   group are unaffected.  *src* and *dst* are path-like objects or path
+   group are unaffected.  *src* and *dst* are :term:`path-like objects 
` or path
names given as strings.
 
If *follow_symlinks* is false, and *src* and *dst* both

___
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]

[Python-checkins] [3.12] docs: Add glossary term references to shutil docs (GH-115559) (#115578)

2024-02-16 Thread CAM-Gerlach 
https://github.com/python/cpython/commit/d08d5b62517c71a7f8781a6b2d3520a0e4bf7b19
commit: d08d5b62517c71a7f8781a6b2d3520a0e4bf7b19
branch: 3.12
author: Miss Islington (bot) <[email protected]>
committer: CAM-Gerlach 
date: 2024-02-16T22:12:49Z
summary:

[3.12] docs: Add glossary term references to shutil docs (GH-115559) (#115578)

docs: Add glossary term references to shutil docs (GH-115559)

Add glossary term references to shutil docs
(cherry picked from commit 318f2190bc93796008b0a4241243b0851b418436)

Co-authored-by: Brian Schubert 

files:
M Doc/library/shutil.rst

diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst
index 3b98a196a8c78d..7922696c982914 100644
--- a/Doc/library/shutil.rst
+++ b/Doc/library/shutil.rst
@@ -39,7 +39,7 @@ Directory and files operations
 
 .. function:: copyfileobj(fsrc, fdst[, length])
 
-   Copy the contents of the file-like object *fsrc* to the file-like object 
*fdst*.
+   Copy the contents of the :term:`file-like object ` *fsrc* to 
the file-like object *fdst*.
The integer *length*, if given, is the buffer size. In particular, a 
negative
*length* value means to copy the data without looping over the source data 
in
chunks; by default the data is read in chunks to avoid uncontrolled memory
@@ -52,7 +52,7 @@ Directory and files operations
 
Copy the contents (no metadata) of the file named *src* to a file named
*dst* and return *dst* in the most efficient way possible.
-   *src* and *dst* are path-like objects or path names given as strings.
+   *src* and *dst* are :term:`path-like objects ` or path 
names given as strings.
 
*dst* must be the complete target file name; look at :func:`~shutil.copy`
for a copy that accepts a target directory path.  If *src* and *dst*
@@ -94,7 +94,7 @@ Directory and files operations
 .. function:: copymode(src, dst, *, follow_symlinks=True)
 
Copy the permission bits from *src* to *dst*.  The file contents, owner, and
-   group are unaffected.  *src* and *dst* are path-like objects or path names
+   group are unaffected.  *src* and *dst* are :term:`path-like objects 
` or path names
given as strings.
If *follow_symlinks* is false, and both *src* and *dst* are symbolic links,
:func:`copymode` will attempt to modify the mode of *dst* itself (rather
@@ -113,7 +113,7 @@ Directory and files operations
Copy the permission bits, last access time, last modification time, and
flags from *src* to *dst*.  On Linux, :func:`copystat` also copies the
"extended attributes" where possible.  The file contents, owner, and
-   group are unaffected.  *src* and *dst* are path-like objects or path
+   group are unaffected.  *src* and *dst* are :term:`path-like objects 
` or path
names given as strings.
 
If *follow_symlinks* is false, and *src* and *dst* both

___
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]

[Python-checkins] [3.11] docs: Add glossary term references to shutil docs (GH-115559) (#115579)

2024-02-16 Thread CAM-Gerlach 
https://github.com/python/cpython/commit/7de9caec0da04d6c9b8786dc02754532e22bf432
commit: 7de9caec0da04d6c9b8786dc02754532e22bf432
branch: 3.11
author: Miss Islington (bot) <[email protected]>
committer: CAM-Gerlach 
date: 2024-02-16T22:12:45Z
summary:

[3.11] docs: Add glossary term references to shutil docs (GH-115559) (#115579)

docs: Add glossary term references to shutil docs (GH-115559)

Add glossary term references to shutil docs
(cherry picked from commit 318f2190bc93796008b0a4241243b0851b418436)

Co-authored-by: Brian Schubert 

files:
M Doc/library/shutil.rst

diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst
index 82259949670994..6a6e42da8c7367 100644
--- a/Doc/library/shutil.rst
+++ b/Doc/library/shutil.rst
@@ -39,7 +39,7 @@ Directory and files operations
 
 .. function:: copyfileobj(fsrc, fdst[, length])
 
-   Copy the contents of the file-like object *fsrc* to the file-like object 
*fdst*.
+   Copy the contents of the :term:`file-like object ` *fsrc* to 
the file-like object *fdst*.
The integer *length*, if given, is the buffer size. In particular, a 
negative
*length* value means to copy the data without looping over the source data 
in
chunks; by default the data is read in chunks to avoid uncontrolled memory
@@ -52,7 +52,7 @@ Directory and files operations
 
Copy the contents (no metadata) of the file named *src* to a file named
*dst* and return *dst* in the most efficient way possible.
-   *src* and *dst* are path-like objects or path names given as strings.
+   *src* and *dst* are :term:`path-like objects ` or path 
names given as strings.
 
*dst* must be the complete target file name; look at :func:`~shutil.copy`
for a copy that accepts a target directory path.  If *src* and *dst*
@@ -94,7 +94,7 @@ Directory and files operations
 .. function:: copymode(src, dst, *, follow_symlinks=True)
 
Copy the permission bits from *src* to *dst*.  The file contents, owner, and
-   group are unaffected.  *src* and *dst* are path-like objects or path names
+   group are unaffected.  *src* and *dst* are :term:`path-like objects 
` or path names
given as strings.
If *follow_symlinks* is false, and both *src* and *dst* are symbolic links,
:func:`copymode` will attempt to modify the mode of *dst* itself (rather
@@ -113,7 +113,7 @@ Directory and files operations
Copy the permission bits, last access time, last modification time, and
flags from *src* to *dst*.  On Linux, :func:`copystat` also copies the
"extended attributes" where possible.  The file contents, owner, and
-   group are unaffected.  *src* and *dst* are path-like objects or path
+   group are unaffected.  *src* and *dst* are :term:`path-like objects 
` or path
names given as strings.
 
If *follow_symlinks* is false, and *src* and *dst* both

___
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]

[Python-checkins] gh-111968: Split _Py_async_gen_asend_freelist out of _Py_async_gen_fr… (gh-115546)

2024-02-16 Thread corona10 
https://github.com/python/cpython/commit/8db8d7118e1ef22bc7cdc3d8b657bc10c22c2fd6
commit: 8db8d7118e1ef22bc7cdc3d8b657bc10c22c2fd6
branch: main
author: Donghee Na 
committer: corona10 
date: 2024-02-17T10:03:10+09:00
summary:

gh-111968: Split _Py_async_gen_asend_freelist out of _Py_async_gen_fr… 
(gh-115546)

files:
M Include/internal/pycore_freelist.h
M Objects/genobject.c

diff --git a/Include/internal/pycore_freelist.h 
b/Include/internal/pycore_freelist.h
index 9900ce98037060..e684e084b8bef8 100644
--- a/Include/internal/pycore_freelist.h
+++ b/Include/internal/pycore_freelist.h
@@ -105,11 +105,15 @@ struct _Py_async_gen_freelist {
fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend
are short-living objects that are instantiated for every
__anext__() call. */
-struct _PyAsyncGenWrappedValue* value_freelist[_PyAsyncGen_MAXFREELIST];
-int value_numfree;
+struct _PyAsyncGenWrappedValue* items[_PyAsyncGen_MAXFREELIST];
+int numfree;
+#endif
+};
 
-struct PyAsyncGenASend* asend_freelist[_PyAsyncGen_MAXFREELIST];
-int asend_numfree;
+struct _Py_async_gen_asend_freelist {
+#ifdef WITH_FREELISTS
+struct PyAsyncGenASend* items[_PyAsyncGen_MAXFREELIST];
+int numfree;
 #endif
 };
 
@@ -129,6 +133,7 @@ struct _Py_object_freelists {
 struct _Py_slice_freelist slices;
 struct _Py_context_freelist contexts;
 struct _Py_async_gen_freelist async_gens;
+struct _Py_async_gen_asend_freelist async_gen_asends;
 struct _Py_object_stack_freelist object_stacks;
 };
 
diff --git a/Objects/genobject.c b/Objects/genobject.c
index a1b6db1b5889d3..8d1dbb72ba9ec2 100644
--- a/Objects/genobject.c
+++ b/Objects/genobject.c
@@ -1635,6 +1635,13 @@ get_async_gen_freelist(void)
 struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
 return &freelists->async_gens;
 }
+
+static struct _Py_async_gen_asend_freelist *
+get_async_gen_asend_freelist(void)
+{
+struct _Py_object_freelists *freelists = _Py_object_freelists_GET();
+return &freelists->async_gen_asends;
+}
 #endif
 
 
@@ -1659,25 +1666,27 @@ void
 _PyAsyncGen_ClearFreeLists(struct _Py_object_freelists *freelist_state, int 
is_finalization)
 {
 #ifdef WITH_FREELISTS
-struct _Py_async_gen_freelist *state = &freelist_state->async_gens;
+struct _Py_async_gen_freelist *freelist = &freelist_state->async_gens;
 
-while (state->value_numfree > 0) {
+while (freelist->numfree > 0) {
 _PyAsyncGenWrappedValue *o;
-o = state->value_freelist[--state->value_numfree];
+o = freelist->items[--freelist->numfree];
 assert(_PyAsyncGenWrappedValue_CheckExact(o));
 PyObject_GC_Del(o);
 }
 
-while (state->asend_numfree > 0) {
+struct _Py_async_gen_asend_freelist *asend_freelist = 
&freelist_state->async_gen_asends;
+
+while (asend_freelist->numfree > 0) {
 PyAsyncGenASend *o;
-o = state->asend_freelist[--state->asend_numfree];
+o = asend_freelist->items[--asend_freelist->numfree];
 assert(Py_IS_TYPE(o, &_PyAsyncGenASend_Type));
 PyObject_GC_Del(o);
 }
 
 if (is_finalization) {
-state->value_numfree = -1;
-state->asend_numfree = -1;
+freelist->numfree = -1;
+asend_freelist->numfree = -1;
 }
 #endif
 }
@@ -1726,11 +1735,11 @@ async_gen_asend_dealloc(PyAsyncGenASend *o)
 Py_CLEAR(o->ags_gen);
 Py_CLEAR(o->ags_sendval);
 #ifdef WITH_FREELISTS
-struct _Py_async_gen_freelist *async_gen_freelist = 
get_async_gen_freelist();
-if (async_gen_freelist->asend_numfree >= 0 && 
async_gen_freelist->asend_numfree < _PyAsyncGen_MAXFREELIST) {
+struct _Py_async_gen_asend_freelist *freelist = 
get_async_gen_asend_freelist();
+if (freelist->numfree >= 0 && freelist->numfree < _PyAsyncGen_MAXFREELIST) 
{
 assert(PyAsyncGenASend_CheckExact(o));
 _PyGC_CLEAR_FINALIZED((PyObject *)o);
-
async_gen_freelist->asend_freelist[async_gen_freelist->asend_numfree++] = o;
+freelist->items[freelist->numfree++] = o;
 }
 else
 #endif
@@ -1896,10 +1905,10 @@ async_gen_asend_new(PyAsyncGenObject *gen, PyObject 
*sendval)
 {
 PyAsyncGenASend *o;
 #ifdef WITH_FREELISTS
-struct _Py_async_gen_freelist *async_gen_freelist = 
get_async_gen_freelist();
-if (async_gen_freelist->asend_numfree > 0) {
-async_gen_freelist->asend_numfree--;
-o = 
async_gen_freelist->asend_freelist[async_gen_freelist->asend_numfree];
+struct _Py_async_gen_asend_freelist *freelist = 
get_async_gen_asend_freelist();
+if (freelist->numfree > 0) {
+freelist->numfree--;
+o = freelist->items[freelist->numfree];
 _Py_NewReference((PyObject *)o);
 }
 else
@@ -1931,10 +1940,10 @@ async_gen_wrapped_val_dealloc(_PyAsyncGenWrappedValue 
*o)
 _PyObject_GC_UNTRACK((PyObject *)o);
 Py_CLEAR(o->agw_val);
 #ifdef WITH_FREELISTS
-struct _Py_async_gen_freelist *async_gen_freelist = 
ge

[Python-checkins] gh-113812: Allow DatagramTransport.sendto to send empty data (#115199)

2024-02-16 Thread gvanrossum 
https://github.com/python/cpython/commit/73e8637002639e565938d3f205bf46e7f1dbd6a8
commit: 73e8637002639e565938d3f205bf46e7f1dbd6a8
branch: main
author: Jamie Phan 
committer: gvanrossum 
date: 2024-02-16T18:38:07-08:00
summary:

gh-113812: Allow DatagramTransport.sendto to send empty data (#115199)

Also include the UDP packet header sizes (8 bytes per packet)
in the buffer size reported to the flow control subsystem.

files:
A Misc/NEWS.d/next/Library/2024-02-09-12-22-47.gh-issue-113812.wOraaG.rst
M Doc/library/asyncio-protocol.rst
M Doc/whatsnew/3.13.rst
M Lib/asyncio/proactor_events.py
M Lib/asyncio/selector_events.py
M Lib/asyncio/transports.py
M Lib/test/test_asyncio/test_proactor_events.py
M Lib/test/test_asyncio/test_selector_events.py

diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst
index ecd8cdc709af7d..7c08d65f26bc27 100644
--- a/Doc/library/asyncio-protocol.rst
+++ b/Doc/library/asyncio-protocol.rst
@@ -362,6 +362,11 @@ Datagram Transports
This method does not block; it buffers the data and arranges
for it to be sent out asynchronously.
 
+   .. versionchanged:: 3.13
+  This method can be called with an empty bytes object to send a
+  zero-length datagram. The buffer size calculation used for flow
+  control is also updated to account for the datagram header.
+
 .. method:: DatagramTransport.abort()
 
Close the transport immediately, without waiting for pending
diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst
index 1e0764144a2855..7c6a2af28758be 100644
--- a/Doc/whatsnew/3.13.rst
+++ b/Doc/whatsnew/3.13.rst
@@ -218,6 +218,12 @@ asyncio
   the Unix socket when the server is closed.
   (Contributed by Pierre Ossman in :gh:`111246`.)
 
+* :meth:`asyncio.DatagramTransport.sendto` will now send zero-length
+  datagrams if called with an empty bytes object. The transport flow
+  control also now accounts for the datagram header when calculating
+  the buffer size.
+  (Contributed by Jamie Phan in :gh:`115199`.)
+
 copy
 
 
diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py
index 1e2a730cf368a9..a512db6367b20a 100644
--- a/Lib/asyncio/proactor_events.py
+++ b/Lib/asyncio/proactor_events.py
@@ -487,9 +487,6 @@ def sendto(self, data, addr=None):
 raise TypeError('data argument must be bytes-like object (%r)',
 type(data))
 
-if not data:
-return
-
 if self._address is not None and addr not in (None, self._address):
 raise ValueError(
 f'Invalid address: must be None or {self._address}')
@@ -502,7 +499,7 @@ def sendto(self, data, addr=None):
 
 # Ensure that what we buffer is immutable.
 self._buffer.append((bytes(data), addr))
-self._buffer_size += len(data)
+self._buffer_size += len(data) + 8  # include header bytes
 
 if self._write_fut is None:
 # No current write operations are active, kick one off
diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py
index 10fbdd76e93f79..8e888d26ea0737 100644
--- a/Lib/asyncio/selector_events.py
+++ b/Lib/asyncio/selector_events.py
@@ -1241,8 +1241,6 @@ def sendto(self, data, addr=None):
 if not isinstance(data, (bytes, bytearray, memoryview)):
 raise TypeError(f'data argument must be a bytes-like object, '
 f'not {type(data).__name__!r}')
-if not data:
-return
 
 if self._address:
 if addr not in (None, self._address):
@@ -1278,7 +1276,7 @@ def sendto(self, data, addr=None):
 
 # Ensure that what we buffer is immutable.
 self._buffer.append((bytes(data), addr))
-self._buffer_size += len(data)
+self._buffer_size += len(data) + 8  # include header bytes
 self._maybe_pause_protocol()
 
 def _sendto_ready(self):
diff --git a/Lib/asyncio/transports.py b/Lib/asyncio/transports.py
index 30fd41d49af71f..34c7ad44ffd8ab 100644
--- a/Lib/asyncio/transports.py
+++ b/Lib/asyncio/transports.py
@@ -181,6 +181,8 @@ def sendto(self, data, addr=None):
 to be sent out asynchronously.
 addr is target socket address.
 If addr is None use target address pointed on transport creation.
+If data is an empty bytes object a zero-length datagram will be
+sent.
 """
 raise NotImplementedError
 
diff --git a/Lib/test/test_asyncio/test_proactor_events.py 
b/Lib/test/test_asyncio/test_proactor_events.py
index c42856e578b8cc..fcaa2f6ade2b76 100644
--- a/Lib/test/test_asyncio/test_proactor_events.py
+++ b/Lib/test/test_asyncio/test_proactor_events.py
@@ -585,11 +585,10 @@ def test_sendto_memoryview(self):
 
 def test_sendto_no_data(self):
 transport = self.datagram_transport()
-transport._buffer.append((b'data', ('0.0.0.0', 12345)))
-transport.sendto(b'', ())
-self.assertFalse(self.sock.sendto.called)
-self