[Python-checkins] gh-124295: Add translation tests for argparse (GH-124803)
https://github.com/python/cpython/commit/0922a4ae0d2803e3a4e9f3d2ccd217364cfd700e
commit: 0922a4ae0d2803e3a4e9f3d2ccd217364cfd700e
branch: main
author: Tomas R.
committer: serhiy-storchaka
date: 2024-10-27T18:52:31+02:00
summary:
gh-124295: Add translation tests for argparse (GH-124803)
files:
A Lib/test/translationdata/argparse/msgids.txt
A Misc/NEWS.d/next/Tests/2024-09-30-22-52-44.gh-issue-124295.VZy5kx.rst
M Lib/test/test_argparse.py
M Makefile.pre.in
diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py
index ed1c5c34e526aa..06b616069c68ff 100644
--- a/Lib/test/test_argparse.py
+++ b/Lib/test/test_argparse.py
@@ -7,8 +7,10 @@
import operator
import os
import py_compile
+import re
import shutil
import stat
+import subprocess
import sys
import textwrap
import tempfile
@@ -17,9 +19,13 @@
import warnings
from enum import StrEnum
+from pathlib import Path
+from test.support import REPO_ROOT
+from test.support import TEST_HOME_DIR
from test.support import captured_stderr
from test.support import import_helper
from test.support import os_helper
+from test.support import requires_subprocess
from test.support import script_helper
from unittest import mock
@@ -7014,6 +7020,54 @@ def test_directory_in_zipfile(self, compiled=False):
def test_directory_in_zipfile_compiled(self):
self.test_directory_in_zipfile(compiled=True)
+# =
+# Translation tests
+# =
+
+pygettext = Path(REPO_ROOT) / 'Tools' / 'i18n' / 'pygettext.py'
+snapshot_path = Path(TEST_HOME_DIR) / 'translationdata' / 'argparse' /
'msgids.txt'
+
+msgid_pattern = re.compile(r'msgid(.*?)(?:msgid_plural|msgctxt|msgstr)',
re.DOTALL)
+msgid_string_pattern = re.compile(r'"((?:\\"|[^"])*)"')
+
+
+@requires_subprocess()
+class TestTranslations(unittest.TestCase):
+
+def test_translations(self):
+# Test messages extracted from the argparse module against a snapshot
+res = generate_po_file(stdout_only=False)
+self.assertEqual(res.returncode, 0)
+self.assertEqual(res.stderr, '')
+msgids = extract_msgids(res.stdout)
+snapshot = snapshot_path.read_text().splitlines()
+self.assertListEqual(msgids, snapshot)
+
+
+def generate_po_file(*, stdout_only=True):
+res = subprocess.run([sys.executable, pygettext,
+ '--no-location', '-o', '-', argparse.__file__],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,
text=True)
+if stdout_only:
+return res.stdout
+return res
+
+
+def extract_msgids(po):
+msgids = []
+for msgid in msgid_pattern.findall(po):
+msgid_string = ''.join(msgid_string_pattern.findall(msgid))
+msgid_string = msgid_string.replace(r'\"', '"')
+if msgid_string:
+msgids.append(msgid_string)
+return sorted(msgids)
+
+
+def update_translation_snapshots():
+contents = generate_po_file()
+msgids = extract_msgids(contents)
+snapshot_path.write_text('\n'.join(msgids))
+
def tearDownModule():
# Remove global references to avoid looking like we have refleaks.
@@ -7022,4 +7076,8 @@ def tearDownModule():
if __name__ == '__main__':
+# To regenerate translation snapshots
+if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update':
+update_translation_snapshots()
+sys.exit(0)
unittest.main()
diff --git a/Lib/test/translationdata/argparse/msgids.txt
b/Lib/test/translationdata/argparse/msgids.txt
new file mode 100644
index 00..f4d0a05635cfc1
--- /dev/null
+++ b/Lib/test/translationdata/argparse/msgids.txt
@@ -0,0 +1,41 @@
+ (default: %(default)s)
+%(heading)s:
+%(prog)s: error: %(message)s\n
+%(prog)s: warning: %(message)s\n
+%r is not callable
+'required' is an invalid argument for positionals
+.__call__() not defined
+ambiguous option: %(option)s could match %(matches)s
+argument "-" with mode %r
+argument %(argument_name)s: %(message)s
+argument '%(argument_name)s' is deprecated
+can't open '%(filename)s': %(error)s
+cannot have multiple subparser arguments
+cannot merge actions - two groups are named %r
+command '%(parser_name)s' is deprecated
+conflicting subparser alias: %s
+conflicting subparser: %s
+dest= is required for options like %r
+expected at least one argument
+expected at most one argument
+expected one argument
+ignored explicit argument %r
+invalid %(type)s value: %(value)r
+invalid choice: %(value)r (choose from %(choices)s)
+invalid choice: %(value)r, maybe you meant %(closest)r? (choose from
%(choices)s)
+invalid conflict_resolution value: %r
+invalid option string %(option)r: must start with a character %(prefix_chars)r
+mutually exclusive arguments must be optional
+not allowed with argument %s
+one of the arguments %s is required
+option '%(option)s' is deprecated
+options
+positional arguments
+show program's version number and exit
+show this help message and exit
+subcommands
+the following arguments are required: %s
+unexp
[Python-checkins] gh-125966: fix use-after-free on `fut->fut_callback0` due to an evil callback's `__eq__` in asyncio (#125967)
https://github.com/python/cpython/commit/ed5059eeb1aa50b481957307db5a34b937497382 commit: ed5059eeb1aa50b481957307db5a34b937497382 branch: main author: Bénédikt Tran <[email protected]> committer: kumaraditya303 date: 2024-10-27T22:40:10+05:30 summary: gh-125966: fix use-after-free on `fut->fut_callback0` due to an evil callback's `__eq__` in asyncio (#125967) files: A Misc/NEWS.d/next/Library/2024-10-25-10-53-56.gh-issue-125966.eOCYU_.rst M Lib/test/test_asyncio/test_futures.py M Modules/_asynciomodule.c diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index bac76b074981df..3a4291e3a68ca6 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -999,6 +999,24 @@ def evil_call_soon(*args, **kwargs): # returns an empty list but the C implementation returns None. self.assertIn(fut._callbacks, (None, [])) +def test_use_after_free_on_fut_callback_0_with_evil__eq__(self): +# Special thanks to Nico-Posada for the original PoC. +# See https://github.com/python/cpython/issues/125966. + +fut = self._new_future() + +class cb_pad: +def __eq__(self, other): +return True + +class evil(cb_pad): +def __eq__(self, other): +fut.remove_done_callback(None) +return NotImplemented + +fut.add_done_callback(cb_pad()) +fut.remove_done_callback(evil()) + def test_use_after_free_on_fut_callback_0_with_evil__getattribute__(self): # see: https://github.com/python/cpython/issues/125984 diff --git a/Misc/NEWS.d/next/Library/2024-10-25-10-53-56.gh-issue-125966.eOCYU_.rst b/Misc/NEWS.d/next/Library/2024-10-25-10-53-56.gh-issue-125966.eOCYU_.rst new file mode 100644 index 00..9fe8795de18003 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-25-10-53-56.gh-issue-125966.eOCYU_.rst @@ -0,0 +1,2 @@ +Fix a use-after-free crash in :meth:`asyncio.Future.remove_done_callback`. +Patch by Bénédikt Tran. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index a4dab29ee9428d..d4135f04e56575 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -1019,7 +1019,12 @@ _asyncio_Future_remove_done_callback_impl(FutureObj *self, PyTypeObject *cls, ENSURE_FUTURE_ALIVE(state, self) if (self->fut_callback0 != NULL) { -int cmp = PyObject_RichCompareBool(self->fut_callback0, fn, Py_EQ); +// Beware: An evil PyObject_RichCompareBool could free fut_callback0 +// before a recursive call is made with that same arg. For details, see +// https://github.com/python/cpython/pull/125967#discussion_r1816593340. +PyObject *fut_callback0 = Py_NewRef(self->fut_callback0); +int cmp = PyObject_RichCompareBool(fut_callback0, fn, Py_EQ); +Py_DECREF(fut_callback0); if (cmp == -1) { return NULL; } ___ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3/lists/python-checkins.python.org/ Member address: [email protected]
[Python-checkins] [3.13] gh-125984: fix use-after-free on `fut->fut_{callback,context}0` due to an evil `loop.__getattribute__` (GH-126003) (#126043)
https://github.com/python/cpython/commit/25421c715d735162dc8fac1eb38de30eca3a2552 commit: 25421c715d735162dc8fac1eb38de30eca3a2552 branch: 3.13 author: Miss Islington (bot) <[email protected]> committer: kumaraditya303 date: 2024-10-27T15:28:02Z summary: [3.13] gh-125984: fix use-after-free on `fut->fut_{callback,context}0` due to an evil `loop.__getattribute__` (GH-126003) (#126043) gh-125984: fix use-after-free on `fut->fut_{callback,context}0` due to an evil `loop.__getattribute__` (GH-126003) (cherry picked from commit f819d4301d7c75f02be1187fda017f0e7b608816) Co-authored-by: Bénédikt Tran <[email protected]> files: A Misc/NEWS.d/next/Library/2024-10-26-12-50-48.gh-issue-125984.d4vp5_.rst M Lib/test/test_asyncio/test_futures.py M Modules/_asynciomodule.c diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index e9d5d97db0388e..ccdaff620bd602 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -32,6 +32,25 @@ def last_cb(): pass +class ReachableCode(Exception): +"""Exception to raise to indicate that some code was reached. + +Use this exception if using mocks is not a good alternative. +""" + + +class SimpleEvilEventLoop(asyncio.base_events.BaseEventLoop): +"""Base class for UAF and other evil stuff requiring an evil event loop.""" + +def get_debug(self): # to suppress tracebacks +return False + +def __del__(self): +# Automatically close the evil event loop to avoid warnings. +if not self.is_closed() and not self.is_running(): +self.close() + + class DuckFuture: # Class that does not inherit from Future but aims to be duck-type # compatible with it. @@ -956,6 +975,7 @@ def __eq__(self, other): fut.remove_done_callback(evil()) def test_evil_call_soon_list_mutation(self): +# see: https://github.com/python/cpython/issues/125969 called_on_fut_callback0 = False pad = lambda: ... @@ -970,9 +990,8 @@ def evil_call_soon(*args, **kwargs): else: called_on_fut_callback0 = True -fake_event_loop = lambda: ... +fake_event_loop = SimpleEvilEventLoop() fake_event_loop.call_soon = evil_call_soon -fake_event_loop.get_debug = lambda: False # suppress traceback with mock.patch.object(self, 'loop', fake_event_loop): fut = self._new_future() @@ -988,6 +1007,56 @@ def evil_call_soon(*args, **kwargs): # returns an empty list but the C implementation returns None. self.assertIn(fut._callbacks, (None, [])) +def test_use_after_free_on_fut_callback_0_with_evil__getattribute__(self): +# see: https://github.com/python/cpython/issues/125984 + +class EvilEventLoop(SimpleEvilEventLoop): +def call_soon(self, *args, **kwargs): +super().call_soon(*args, **kwargs) +raise ReachableCode + +def __getattribute__(self, name): +nonlocal fut_callback_0 +if name == 'call_soon': +fut.remove_done_callback(fut_callback_0) +del fut_callback_0 +return object.__getattribute__(self, name) + +evil_loop = EvilEventLoop() +with mock.patch.object(self, 'loop', evil_loop): +fut = self._new_future() +self.assertIs(fut.get_loop(), evil_loop) + +fut_callback_0 = lambda: ... +fut.add_done_callback(fut_callback_0) +self.assertRaises(ReachableCode, fut.set_result, "boom") + +def test_use_after_free_on_fut_context_0_with_evil__getattribute__(self): +# see: https://github.com/python/cpython/issues/125984 + +class EvilEventLoop(SimpleEvilEventLoop): +def call_soon(self, *args, **kwargs): +super().call_soon(*args, **kwargs) +raise ReachableCode + +def __getattribute__(self, name): +if name == 'call_soon': +# resets the future's event loop +fut.__init__(loop=SimpleEvilEventLoop()) +return object.__getattribute__(self, name) + +evil_loop = EvilEventLoop() +with mock.patch.object(self, 'loop', evil_loop): +fut = self._new_future() +self.assertIs(fut.get_loop(), evil_loop) + +fut_callback_0 = mock.Mock() +fut_context_0 = mock.Mock() +fut.add_done_callback(fut_callback_0, context=fut_context_0) +del fut_context_0 +del fut_callback_0 +self.assertRaises(ReachableCode, fut.set_result, "boom") + @unittest.skipUnless(hasattr(futures, '_CFuture'), 'requires the C _asyncio module') diff --git a/Misc/NEWS.d/next/Library/2024-10-26-12-50-48.gh-issue-125984.d4vp5_.rst b/Misc/NEWS.d/
[Python-checkins] [3.12] gh-125984: fix use-after-free on `fut->fut_{callback,context}0` due to an evil `loop.__getattribute__` (GH-126003) (#126044)
https://github.com/python/cpython/commit/fdedb2618a7ed12977ec8c06b82d7296c18bd84c commit: fdedb2618a7ed12977ec8c06b82d7296c18bd84c branch: 3.12 author: Miss Islington (bot) <[email protected]> committer: kumaraditya303 date: 2024-10-27T15:23:07Z summary: [3.12] gh-125984: fix use-after-free on `fut->fut_{callback,context}0` due to an evil `loop.__getattribute__` (GH-126003) (#126044) gh-125984: fix use-after-free on `fut->fut_{callback,context}0` due to an evil `loop.__getattribute__` (GH-126003) (cherry picked from commit f819d4301d7c75f02be1187fda017f0e7b608816) Co-authored-by: Bénédikt Tran <[email protected]> files: A Misc/NEWS.d/next/Library/2024-10-26-12-50-48.gh-issue-125984.d4vp5_.rst M Lib/test/test_asyncio/test_futures.py M Modules/_asynciomodule.c diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 396ca75c7d3fab..63804256ec7726 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -31,6 +31,25 @@ def last_cb(): pass +class ReachableCode(Exception): +"""Exception to raise to indicate that some code was reached. + +Use this exception if using mocks is not a good alternative. +""" + + +class SimpleEvilEventLoop(asyncio.base_events.BaseEventLoop): +"""Base class for UAF and other evil stuff requiring an evil event loop.""" + +def get_debug(self): # to suppress tracebacks +return False + +def __del__(self): +# Automatically close the evil event loop to avoid warnings. +if not self.is_closed() and not self.is_running(): +self.close() + + class DuckFuture: # Class that does not inherit from Future but aims to be duck-type # compatible with it. @@ -937,6 +956,7 @@ def __eq__(self, other): fut.remove_done_callback(evil()) def test_evil_call_soon_list_mutation(self): +# see: https://github.com/python/cpython/issues/125969 called_on_fut_callback0 = False pad = lambda: ... @@ -951,9 +971,8 @@ def evil_call_soon(*args, **kwargs): else: called_on_fut_callback0 = True -fake_event_loop = lambda: ... +fake_event_loop = SimpleEvilEventLoop() fake_event_loop.call_soon = evil_call_soon -fake_event_loop.get_debug = lambda: False # suppress traceback with mock.patch.object(self, 'loop', fake_event_loop): fut = self._new_future() @@ -969,6 +988,56 @@ def evil_call_soon(*args, **kwargs): # returns an empty list but the C implementation returns None. self.assertIn(fut._callbacks, (None, [])) +def test_use_after_free_on_fut_callback_0_with_evil__getattribute__(self): +# see: https://github.com/python/cpython/issues/125984 + +class EvilEventLoop(SimpleEvilEventLoop): +def call_soon(self, *args, **kwargs): +super().call_soon(*args, **kwargs) +raise ReachableCode + +def __getattribute__(self, name): +nonlocal fut_callback_0 +if name == 'call_soon': +fut.remove_done_callback(fut_callback_0) +del fut_callback_0 +return object.__getattribute__(self, name) + +evil_loop = EvilEventLoop() +with mock.patch.object(self, 'loop', evil_loop): +fut = self._new_future() +self.assertIs(fut.get_loop(), evil_loop) + +fut_callback_0 = lambda: ... +fut.add_done_callback(fut_callback_0) +self.assertRaises(ReachableCode, fut.set_result, "boom") + +def test_use_after_free_on_fut_context_0_with_evil__getattribute__(self): +# see: https://github.com/python/cpython/issues/125984 + +class EvilEventLoop(SimpleEvilEventLoop): +def call_soon(self, *args, **kwargs): +super().call_soon(*args, **kwargs) +raise ReachableCode + +def __getattribute__(self, name): +if name == 'call_soon': +# resets the future's event loop +fut.__init__(loop=SimpleEvilEventLoop()) +return object.__getattribute__(self, name) + +evil_loop = EvilEventLoop() +with mock.patch.object(self, 'loop', evil_loop): +fut = self._new_future() +self.assertIs(fut.get_loop(), evil_loop) + +fut_callback_0 = mock.Mock() +fut_context_0 = mock.Mock() +fut.add_done_callback(fut_callback_0, context=fut_context_0) +del fut_context_0 +del fut_callback_0 +self.assertRaises(ReachableCode, fut.set_result, "boom") + @unittest.skipUnless(hasattr(futures, '_CFuture'), 'requires the C _asyncio module') diff --git a/Misc/NEWS.d/next/Library/2024-10-26-12-50-48.gh-issue-125984.d4vp5_.rst b/Misc/NEWS.d/n
[Python-checkins] gh-126018: Avoid aborting due to unnecessary assert in `sys.audit` (#126020)
https://github.com/python/cpython/commit/80eec52fc813bc7d20478da3114ec6ffd73e7c31 commit: 80eec52fc813bc7d20478da3114ec6ffd73e7c31 branch: main author: devdanzin <[email protected]> committer: JelleZijlstra date: 2024-10-27T07:41:42-07:00 summary: gh-126018: Avoid aborting due to unnecessary assert in `sys.audit` (#126020) Co-authored-by: Bénédikt Tran <[email protected]> Co-authored-by: Jelle Zijlstra files: A Misc/NEWS.d/next/Core_and_Builtins/2024-10-26-23-50-03.gh-issue-126018.Hq-qcM.rst M Lib/test/audit-tests.py M Lib/test/test_audit.py M Python/sysmodule.c diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index b9021467817f27..6df09d891433ea 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -567,6 +567,17 @@ def hook(event, args): _winapi.CreateNamedPipe(pipe_name, _winapi.PIPE_ACCESS_DUPLEX, 8, 2, 0, 0, 0, 0) +def test_assert_unicode(): +import sys +sys.addaudithook(lambda *args: None) +try: +sys.audit(9) +except TypeError: +pass +else: +raise RuntimeError("Expected sys.audit(9) to fail.") + + if __name__ == "__main__": from test.support import suppress_msvcrt_asserts diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index 7206307d8b0664..ddd9f951143df7 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -307,5 +307,12 @@ def test_winapi_createnamedpipe(self): self.assertEqual(actual, expected) +def test_assert_unicode(self): +# See gh-126018 +returncode, _, stderr = self.run_python("test_assert_unicode") +if returncode: +self.fail(stderr) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-26-23-50-03.gh-issue-126018.Hq-qcM.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-26-23-50-03.gh-issue-126018.Hq-qcM.rst new file mode 100644 index 00..e019408638997b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-26-23-50-03.gh-issue-126018.Hq-qcM.rst @@ -0,0 +1,2 @@ +Fix a crash in :func:`sys.audit` when passing a non-string as first argument +and Python was compiled in debug mode. diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 8b9209324002ce..24af4798eeac3b 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -519,7 +519,6 @@ sys_audit(PyObject *self, PyObject *const *args, Py_ssize_t argc) } assert(args[0] != NULL); -assert(PyUnicode_Check(args[0])); if (!should_audit(tstate->interp)) { Py_RETURN_NONE; ___ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3/lists/python-checkins.python.org/ Member address: [email protected]
[Python-checkins] gh-125984: fix use-after-free on `fut->fut_{callback,context}0` due to an evil `loop.__getattribute__` (#126003)
https://github.com/python/cpython/commit/f819d4301d7c75f02be1187fda017f0e7b608816 commit: f819d4301d7c75f02be1187fda017f0e7b608816 branch: main author: Bénédikt Tran <[email protected]> committer: kumaraditya303 date: 2024-10-27T20:34:43+05:30 summary: gh-125984: fix use-after-free on `fut->fut_{callback,context}0` due to an evil `loop.__getattribute__` (#126003) files: A Misc/NEWS.d/next/Library/2024-10-26-12-50-48.gh-issue-125984.d4vp5_.rst M Lib/test/test_asyncio/test_futures.py M Modules/_asynciomodule.c diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 4e3efa7ad6abc0..bac76b074981df 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -32,6 +32,25 @@ def last_cb(): pass +class ReachableCode(Exception): +"""Exception to raise to indicate that some code was reached. + +Use this exception if using mocks is not a good alternative. +""" + + +class SimpleEvilEventLoop(asyncio.base_events.BaseEventLoop): +"""Base class for UAF and other evil stuff requiring an evil event loop.""" + +def get_debug(self): # to suppress tracebacks +return False + +def __del__(self): +# Automatically close the evil event loop to avoid warnings. +if not self.is_closed() and not self.is_running(): +self.close() + + class DuckFuture: # Class that does not inherit from Future but aims to be duck-type # compatible with it. @@ -948,6 +967,7 @@ def __eq__(self, other): fut.remove_done_callback(evil()) def test_evil_call_soon_list_mutation(self): +# see: https://github.com/python/cpython/issues/125969 called_on_fut_callback0 = False pad = lambda: ... @@ -962,9 +982,8 @@ def evil_call_soon(*args, **kwargs): else: called_on_fut_callback0 = True -fake_event_loop = lambda: ... +fake_event_loop = SimpleEvilEventLoop() fake_event_loop.call_soon = evil_call_soon -fake_event_loop.get_debug = lambda: False # suppress traceback with mock.patch.object(self, 'loop', fake_event_loop): fut = self._new_future() @@ -980,6 +999,56 @@ def evil_call_soon(*args, **kwargs): # returns an empty list but the C implementation returns None. self.assertIn(fut._callbacks, (None, [])) +def test_use_after_free_on_fut_callback_0_with_evil__getattribute__(self): +# see: https://github.com/python/cpython/issues/125984 + +class EvilEventLoop(SimpleEvilEventLoop): +def call_soon(self, *args, **kwargs): +super().call_soon(*args, **kwargs) +raise ReachableCode + +def __getattribute__(self, name): +nonlocal fut_callback_0 +if name == 'call_soon': +fut.remove_done_callback(fut_callback_0) +del fut_callback_0 +return object.__getattribute__(self, name) + +evil_loop = EvilEventLoop() +with mock.patch.object(self, 'loop', evil_loop): +fut = self._new_future() +self.assertIs(fut.get_loop(), evil_loop) + +fut_callback_0 = lambda: ... +fut.add_done_callback(fut_callback_0) +self.assertRaises(ReachableCode, fut.set_result, "boom") + +def test_use_after_free_on_fut_context_0_with_evil__getattribute__(self): +# see: https://github.com/python/cpython/issues/125984 + +class EvilEventLoop(SimpleEvilEventLoop): +def call_soon(self, *args, **kwargs): +super().call_soon(*args, **kwargs) +raise ReachableCode + +def __getattribute__(self, name): +if name == 'call_soon': +# resets the future's event loop +fut.__init__(loop=SimpleEvilEventLoop()) +return object.__getattribute__(self, name) + +evil_loop = EvilEventLoop() +with mock.patch.object(self, 'loop', evil_loop): +fut = self._new_future() +self.assertIs(fut.get_loop(), evil_loop) + +fut_callback_0 = mock.Mock() +fut_context_0 = mock.Mock() +fut.add_done_callback(fut_callback_0, context=fut_context_0) +del fut_context_0 +del fut_callback_0 +self.assertRaises(ReachableCode, fut.set_result, "boom") + @unittest.skipUnless(hasattr(futures, '_CFuture'), 'requires the C _asyncio module') diff --git a/Misc/NEWS.d/next/Library/2024-10-26-12-50-48.gh-issue-125984.d4vp5_.rst b/Misc/NEWS.d/next/Library/2024-10-26-12-50-48.gh-issue-125984.d4vp5_.rst new file mode 100644 index 00..7a1d7b53b11301 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-26-12-50-48.gh-issue-125984.d4vp5_.rst @@ -0,0 +1,3 @@ +Fix use-after-free crashes on :class:`asyncio.Future` objects f
[Python-checkins] [3.13] gh-126018: Avoid aborting due to unnecessary assert in `sys.audit` (GH-126020) (#126042)
https://github.com/python/cpython/commit/e4157f0fd71676c5dcee687ad17871dc226a5560 commit: e4157f0fd71676c5dcee687ad17871dc226a5560 branch: 3.13 author: Miss Islington (bot) <[email protected]> committer: JelleZijlstra date: 2024-10-27T15:07:33Z summary: [3.13] gh-126018: Avoid aborting due to unnecessary assert in `sys.audit` (GH-126020) (#126042) (cherry picked from commit 80eec52fc813bc7d20478da3114ec6ffd73e7c31) Co-authored-by: devdanzin <[email protected]> Co-authored-by: Bénédikt Tran <[email protected]> Co-authored-by: Jelle Zijlstra files: A Misc/NEWS.d/next/Core_and_Builtins/2024-10-26-23-50-03.gh-issue-126018.Hq-qcM.rst M Lib/test/audit-tests.py M Lib/test/test_audit.py M Python/sysmodule.c diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index b9021467817f27..6df09d891433ea 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -567,6 +567,17 @@ def hook(event, args): _winapi.CreateNamedPipe(pipe_name, _winapi.PIPE_ACCESS_DUPLEX, 8, 2, 0, 0, 0, 0) +def test_assert_unicode(): +import sys +sys.addaudithook(lambda *args: None) +try: +sys.audit(9) +except TypeError: +pass +else: +raise RuntimeError("Expected sys.audit(9) to fail.") + + if __name__ == "__main__": from test.support import suppress_msvcrt_asserts diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index 7206307d8b0664..ddd9f951143df7 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -307,5 +307,12 @@ def test_winapi_createnamedpipe(self): self.assertEqual(actual, expected) +def test_assert_unicode(self): +# See gh-126018 +returncode, _, stderr = self.run_python("test_assert_unicode") +if returncode: +self.fail(stderr) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-26-23-50-03.gh-issue-126018.Hq-qcM.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-26-23-50-03.gh-issue-126018.Hq-qcM.rst new file mode 100644 index 00..e019408638997b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-26-23-50-03.gh-issue-126018.Hq-qcM.rst @@ -0,0 +1,2 @@ +Fix a crash in :func:`sys.audit` when passing a non-string as first argument +and Python was compiled in debug mode. diff --git a/Python/sysmodule.c b/Python/sysmodule.c index a521bcc3531f23..3f170fff156fcd 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -518,7 +518,6 @@ sys_audit(PyObject *self, PyObject *const *args, Py_ssize_t argc) } assert(args[0] != NULL); -assert(PyUnicode_Check(args[0])); if (!should_audit(tstate->interp)) { Py_RETURN_NONE; ___ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3/lists/python-checkins.python.org/ Member address: [email protected]
[Python-checkins] gh-126012: Add `__class_getitem__` to `memoryview` (#126013)
https://github.com/python/cpython/commit/dc76a4ad3c46af3fb70361be10b8a791cd126931 commit: dc76a4ad3c46af3fb70361be10b8a791cd126931 branch: main author: Brian Schubert committer: sobolevn date: 2024-10-27T13:36:03+03:00 summary: gh-126012: Add `__class_getitem__` to `memoryview` (#126013) Co-authored-by: Bénédikt Tran <[email protected]> Co-authored-by: Peter Bierma files: A Misc/NEWS.d/next/Core and Builtins/2024-10-26-13-32-48.gh-issue-126012.2KalhG.rst M Doc/library/stdtypes.rst M Doc/whatsnew/3.14.rst M Lib/test/test_genericalias.py M Objects/memoryobject.c diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index a6e2e3b8928ebe..704637d675ead3 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -3889,6 +3889,9 @@ copying. .. versionchanged:: 3.5 memoryviews can now be indexed with tuple of integers. + .. versionchanged:: next + memoryview is now a :term:`generic type`. + :class:`memoryview` has several methods: .. method:: __eq__(exporter) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 1ccfa329d5546a..a6f595ccf08bf4 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -194,6 +194,10 @@ Other language changes :mod:`copyable `. (Contributed by Serhiy Storchaka in :gh:`125767`.) +* The :class:`memoryview` type now supports subscription, + making it a :term:`generic type`. + (Contributed by Brian Schubert in :gh:`126012`.) + New modules === diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 12564b423493aa..3048d038c782d4 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -100,7 +100,7 @@ class BaseTest(unittest.TestCase): """Test basics.""" -generic_types = [type, tuple, list, dict, set, frozenset, enumerate, +generic_types = [type, tuple, list, dict, set, frozenset, enumerate, memoryview, defaultdict, deque, SequenceMatcher, dircmp, diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-10-26-13-32-48.gh-issue-126012.2KalhG.rst b/Misc/NEWS.d/next/Core and Builtins/2024-10-26-13-32-48.gh-issue-126012.2KalhG.rst new file mode 100644 index 00..5307920ddcf200 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-10-26-13-32-48.gh-issue-126012.2KalhG.rst @@ -0,0 +1,2 @@ +The :class:`memoryview` type now supports subscription, making it a +:term:`generic type`. diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index a2472d4873641d..d4672e8198cb24 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -3286,6 +3286,7 @@ static PyMethodDef memory_methods[] = { MEMORYVIEW__FROM_FLAGS_METHODDEF {"__enter__", memory_enter, METH_NOARGS, NULL}, {"__exit__",memory_exit, METH_VARARGS, memory_exit_doc}, +{"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} }; ___ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3/lists/python-checkins.python.org/ Member address: [email protected]
[Python-checkins] [3.13] gh-125966: fix use-after-free on `fut->fut_callback0` due to an evil callback's `__eq__` in asyncio (GH-125967) (#126047)
https://github.com/python/cpython/commit/ac31a2607fdc48986b7f3e46b113b1484d266ef2 commit: ac31a2607fdc48986b7f3e46b113b1484d266ef2 branch: 3.13 author: Miss Islington (bot) <[email protected]> committer: kumaraditya303 date: 2024-10-27T17:36:10Z summary: [3.13] gh-125966: fix use-after-free on `fut->fut_callback0` due to an evil callback's `__eq__` in asyncio (GH-125967) (#126047) gh-125966: fix use-after-free on `fut->fut_callback0` due to an evil callback's `__eq__` in asyncio (GH-125967) (cherry picked from commit ed5059eeb1aa50b481957307db5a34b937497382) Co-authored-by: Bénédikt Tran <[email protected]> files: A Misc/NEWS.d/next/Library/2024-10-25-10-53-56.gh-issue-125966.eOCYU_.rst M Lib/test/test_asyncio/test_futures.py M Modules/_asynciomodule.c diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index ccdaff620bd602..ec3029983b73e8 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -1007,6 +1007,24 @@ def evil_call_soon(*args, **kwargs): # returns an empty list but the C implementation returns None. self.assertIn(fut._callbacks, (None, [])) +def test_use_after_free_on_fut_callback_0_with_evil__eq__(self): +# Special thanks to Nico-Posada for the original PoC. +# See https://github.com/python/cpython/issues/125966. + +fut = self._new_future() + +class cb_pad: +def __eq__(self, other): +return True + +class evil(cb_pad): +def __eq__(self, other): +fut.remove_done_callback(None) +return NotImplemented + +fut.add_done_callback(cb_pad()) +fut.remove_done_callback(evil()) + def test_use_after_free_on_fut_callback_0_with_evil__getattribute__(self): # see: https://github.com/python/cpython/issues/125984 diff --git a/Misc/NEWS.d/next/Library/2024-10-25-10-53-56.gh-issue-125966.eOCYU_.rst b/Misc/NEWS.d/next/Library/2024-10-25-10-53-56.gh-issue-125966.eOCYU_.rst new file mode 100644 index 00..9fe8795de18003 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-25-10-53-56.gh-issue-125966.eOCYU_.rst @@ -0,0 +1,2 @@ +Fix a use-after-free crash in :meth:`asyncio.Future.remove_done_callback`. +Patch by Bénédikt Tran. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 79b3b709bc272d..c47b921d72ae3a 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -965,7 +965,12 @@ _asyncio_Future_remove_done_callback_impl(FutureObj *self, PyTypeObject *cls, ENSURE_FUTURE_ALIVE(state, self) if (self->fut_callback0 != NULL) { -int cmp = PyObject_RichCompareBool(self->fut_callback0, fn, Py_EQ); +// Beware: An evil PyObject_RichCompareBool could free fut_callback0 +// before a recursive call is made with that same arg. For details, see +// https://github.com/python/cpython/pull/125967#discussion_r1816593340. +PyObject *fut_callback0 = Py_NewRef(self->fut_callback0); +int cmp = PyObject_RichCompareBool(fut_callback0, fn, Py_EQ); +Py_DECREF(fut_callback0); if (cmp == -1) { return NULL; } ___ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3/lists/python-checkins.python.org/ Member address: [email protected]
[Python-checkins] [3.12] gh-125966: fix use-after-free on `fut->fut_callback0` due to an evil callback's `__eq__` in asyncio (GH-125967) (#126048)
https://github.com/python/cpython/commit/4fc1da1b2757c49f9e6eda9cbc7513cc2ed5180c commit: 4fc1da1b2757c49f9e6eda9cbc7513cc2ed5180c branch: 3.12 author: Miss Islington (bot) <[email protected]> committer: kumaraditya303 date: 2024-10-27T17:32:11Z summary: [3.12] gh-125966: fix use-after-free on `fut->fut_callback0` due to an evil callback's `__eq__` in asyncio (GH-125967) (#126048) gh-125966: fix use-after-free on `fut->fut_callback0` due to an evil callback's `__eq__` in asyncio (GH-125967) (cherry picked from commit ed5059eeb1aa50b481957307db5a34b937497382) Co-authored-by: Bénédikt Tran <[email protected]> files: A Misc/NEWS.d/next/Library/2024-10-25-10-53-56.gh-issue-125966.eOCYU_.rst M Lib/test/test_asyncio/test_futures.py M Modules/_asynciomodule.c diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 63804256ec7726..8be6dcac548ec0 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -988,6 +988,24 @@ def evil_call_soon(*args, **kwargs): # returns an empty list but the C implementation returns None. self.assertIn(fut._callbacks, (None, [])) +def test_use_after_free_on_fut_callback_0_with_evil__eq__(self): +# Special thanks to Nico-Posada for the original PoC. +# See https://github.com/python/cpython/issues/125966. + +fut = self._new_future() + +class cb_pad: +def __eq__(self, other): +return True + +class evil(cb_pad): +def __eq__(self, other): +fut.remove_done_callback(None) +return NotImplemented + +fut.add_done_callback(cb_pad()) +fut.remove_done_callback(evil()) + def test_use_after_free_on_fut_callback_0_with_evil__getattribute__(self): # see: https://github.com/python/cpython/issues/125984 diff --git a/Misc/NEWS.d/next/Library/2024-10-25-10-53-56.gh-issue-125966.eOCYU_.rst b/Misc/NEWS.d/next/Library/2024-10-25-10-53-56.gh-issue-125966.eOCYU_.rst new file mode 100644 index 00..9fe8795de18003 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-25-10-53-56.gh-issue-125966.eOCYU_.rst @@ -0,0 +1,2 @@ +Fix a use-after-free crash in :meth:`asyncio.Future.remove_done_callback`. +Patch by Bénédikt Tran. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 8787fe4ac1cc51..9016c077e0d657 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -1044,7 +1044,12 @@ _asyncio_Future_remove_done_callback_impl(FutureObj *self, PyTypeObject *cls, ENSURE_FUTURE_ALIVE(state, self) if (self->fut_callback0 != NULL) { -int cmp = PyObject_RichCompareBool(self->fut_callback0, fn, Py_EQ); +// Beware: An evil PyObject_RichCompareBool could free fut_callback0 +// before a recursive call is made with that same arg. For details, see +// https://github.com/python/cpython/pull/125967#discussion_r1816593340. +PyObject *fut_callback0 = Py_NewRef(self->fut_callback0); +int cmp = PyObject_RichCompareBool(fut_callback0, fn, Py_EQ); +Py_DECREF(fut_callback0); if (cmp == -1) { return NULL; } ___ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3/lists/python-checkins.python.org/ Member address: [email protected]
[Python-checkins] gh-124295: Skip translation tests when pygettext is missing (GH-126051)
https://github.com/python/cpython/commit/6870eb3f7322ed64a1ca8bbda5cf4e121e181b54
commit: 6870eb3f7322ed64a1ca8bbda5cf4e121e181b54
branch: main
author: Tomas R.
committer: serhiy-storchaka
date: 2024-10-27T18:09:08Z
summary:
gh-124295: Skip translation tests when pygettext is missing (GH-126051)
files:
M Lib/test/test_argparse.py
diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py
index 06b616069c68ff..427e6bc6e7220c 100644
--- a/Lib/test/test_argparse.py
+++ b/Lib/test/test_argparse.py
@@ -27,6 +27,7 @@
from test.support import os_helper
from test.support import requires_subprocess
from test.support import script_helper
+from test.test_tools import skip_if_missing
from unittest import mock
@@ -7036,6 +7037,7 @@ class TestTranslations(unittest.TestCase):
def test_translations(self):
# Test messages extracted from the argparse module against a snapshot
+skip_if_missing('i18n')
res = generate_po_file(stdout_only=False)
self.assertEqual(res.returncode, 0)
self.assertEqual(res.stderr, '')
___
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]
[Python-checkins] [3.13] gh-124295: Add translation tests for argparse (GH-124803) (GH-126046)
https://github.com/python/cpython/commit/ff044ed8004e31c1896ca641b81b13ab04e92837
commit: ff044ed8004e31c1896ca641b81b13ab04e92837
branch: 3.13
author: Serhiy Storchaka
committer: serhiy-storchaka
date: 2024-10-27T18:35:30Z
summary:
[3.13] gh-124295: Add translation tests for argparse (GH-124803) (GH-126046)
(cherry picked from commit 0922a4ae0d2803e3a4e9f3d2ccd217364cfd700e)
Co-authored-by: Tomas R.
files:
A Lib/test/translationdata/argparse/msgids.txt
A Misc/NEWS.d/next/Tests/2024-09-30-22-52-44.gh-issue-124295.VZy5kx.rst
M Lib/test/test_argparse.py
M Makefile.pre.in
diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py
index f74105ca1ffb3e..4385b93ee56e7d 100644
--- a/Lib/test/test_argparse.py
+++ b/Lib/test/test_argparse.py
@@ -6,8 +6,10 @@
import io
import operator
import os
+import re
import shutil
import stat
+import subprocess
import sys
import textwrap
import tempfile
@@ -16,7 +18,15 @@
import warnings
from enum import StrEnum
-from test.support import os_helper, captured_stderr
+from pathlib import Path
+from test.support import REPO_ROOT
+from test.support import TEST_HOME_DIR
+from test.support import captured_stderr
+from test.support import import_helper
+from test.support import os_helper
+from test.support import requires_subprocess
+from test.support import script_helper
+from test.test_tools import skip_if_missing
from unittest import mock
@@ -6752,6 +6762,56 @@ def test_os_error(self):
self.parser.parse_args, ['@no-such-file'])
+# =
+# Translation tests
+# =
+
+pygettext = Path(REPO_ROOT) / 'Tools' / 'i18n' / 'pygettext.py'
+snapshot_path = Path(TEST_HOME_DIR) / 'translationdata' / 'argparse' /
'msgids.txt'
+
+msgid_pattern = re.compile(r'msgid(.*?)(?:msgid_plural|msgctxt|msgstr)',
re.DOTALL)
+msgid_string_pattern = re.compile(r'"((?:\\"|[^"])*)"')
+
+
+@requires_subprocess()
+class TestTranslations(unittest.TestCase):
+
+def test_translations(self):
+# Test messages extracted from the argparse module against a snapshot
+skip_if_missing('i18n')
+res = generate_po_file(stdout_only=False)
+self.assertEqual(res.returncode, 0)
+self.assertEqual(res.stderr, '')
+msgids = extract_msgids(res.stdout)
+snapshot = snapshot_path.read_text().splitlines()
+self.assertListEqual(msgids, snapshot)
+
+
+def generate_po_file(*, stdout_only=True):
+res = subprocess.run([sys.executable, pygettext,
+ '--no-location', '-o', '-', argparse.__file__],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,
text=True)
+if stdout_only:
+return res.stdout
+return res
+
+
+def extract_msgids(po):
+msgids = []
+for msgid in msgid_pattern.findall(po):
+msgid_string = ''.join(msgid_string_pattern.findall(msgid))
+msgid_string = msgid_string.replace(r'\"', '"')
+if msgid_string:
+msgids.append(msgid_string)
+return sorted(msgids)
+
+
+def update_translation_snapshots():
+contents = generate_po_file()
+msgids = extract_msgids(contents)
+snapshot_path.write_text('\n'.join(msgids))
+
+
def tearDownModule():
# Remove global references to avoid looking like we have refleaks.
RFile.seen = {}
@@ -6759,4 +6819,8 @@ def tearDownModule():
if __name__ == '__main__':
+# To regenerate translation snapshots
+if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update':
+update_translation_snapshots()
+sys.exit(0)
unittest.main()
diff --git a/Lib/test/translationdata/argparse/msgids.txt
b/Lib/test/translationdata/argparse/msgids.txt
new file mode 100644
index 00..97416aad520271
--- /dev/null
+++ b/Lib/test/translationdata/argparse/msgids.txt
@@ -0,0 +1,40 @@
+ (default: %(default)s)
+%(heading)s:
+%(prog)s: error: %(message)s\n
+%(prog)s: warning: %(message)s\n
+%r is not callable
+'required' is an invalid argument for positionals
+.__call__() not defined
+ambiguous option: %(option)s could match %(matches)s
+argument "-" with mode %r
+argument %(argument_name)s: %(message)s
+argument '%(argument_name)s' is deprecated
+can't open '%(filename)s': %(error)s
+cannot have multiple subparser arguments
+cannot merge actions - two groups are named %r
+command '%(parser_name)s' is deprecated
+conflicting subparser alias: %s
+conflicting subparser: %s
+dest= is required for options like %r
+expected at least one argument
+expected at most one argument
+expected one argument
+ignored explicit argument %r
+invalid %(type)s value: %(value)r
+invalid choice: %(value)r (choose from %(choices)s)
+invalid conflict_resolution value: %r
+invalid option string %(option)r: must start with a character %(prefix_chars)r
+mutually exclusive arguments must be optional
+not allowed with argument %s
+one of the arguments %s is required
+option '%(option)s' is deprecated
+options
+positional arguments
+show prog
[Python-checkins] [3.12] gh-124295: Add translation tests for argparse (GH-124803) (GH-126046) (GH-126054)
https://github.com/python/cpython/commit/4c039c0723d1db889269f89208751647999a7f43
commit: 4c039c0723d1db889269f89208751647999a7f43
branch: 3.12
author: Serhiy Storchaka
committer: serhiy-storchaka
date: 2024-10-27T18:57:11Z
summary:
[3.12] gh-124295: Add translation tests for argparse (GH-124803) (GH-126046)
(GH-126054)
(cherry picked from commit 0922a4ae0d2803e3a4e9f3d2ccd217364cfd700e)
(cherry picked from commit ff044ed8004e31c1896ca641b81b13ab04e92837)
Co-authored-by: Tomas R.
files:
A Lib/test/translationdata/argparse/msgids.txt
A Misc/NEWS.d/next/Tests/2024-09-30-22-52-44.gh-issue-124295.VZy5kx.rst
M Lib/test/test_argparse.py
M Makefile.pre.in
diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py
index 7f3c74be0ce6dd..c8adc640f2ccaf 100644
--- a/Lib/test/test_argparse.py
+++ b/Lib/test/test_argparse.py
@@ -6,8 +6,10 @@
import io
import operator
import os
+import re
import shutil
import stat
+import subprocess
import sys
import textwrap
import tempfile
@@ -16,7 +18,15 @@
import warnings
from enum import StrEnum
+from pathlib import Path
+from test.support import REPO_ROOT
+from test.support import TEST_HOME_DIR
+from test.support import captured_stderr
+from test.support import import_helper
from test.support import os_helper
+from test.support import requires_subprocess
+from test.support import script_helper
+from test.test_tools import skip_if_missing
from unittest import mock
@@ -6388,6 +6398,56 @@ def test_os_error(self):
self.parser.parse_args, ['@no-such-file'])
+# =
+# Translation tests
+# =
+
+pygettext = Path(REPO_ROOT) / 'Tools' / 'i18n' / 'pygettext.py'
+snapshot_path = Path(TEST_HOME_DIR) / 'translationdata' / 'argparse' /
'msgids.txt'
+
+msgid_pattern = re.compile(r'msgid(.*?)(?:msgid_plural|msgctxt|msgstr)',
re.DOTALL)
+msgid_string_pattern = re.compile(r'"((?:\\"|[^"])*)"')
+
+
+@requires_subprocess()
+class TestTranslations(unittest.TestCase):
+
+def test_translations(self):
+# Test messages extracted from the argparse module against a snapshot
+skip_if_missing('i18n')
+res = generate_po_file(stdout_only=False)
+self.assertEqual(res.returncode, 0)
+self.assertEqual(res.stderr, '')
+msgids = extract_msgids(res.stdout)
+snapshot = snapshot_path.read_text().splitlines()
+self.assertListEqual(msgids, snapshot)
+
+
+def generate_po_file(*, stdout_only=True):
+res = subprocess.run([sys.executable, pygettext,
+ '--no-location', '-o', '-', argparse.__file__],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,
text=True)
+if stdout_only:
+return res.stdout
+return res
+
+
+def extract_msgids(po):
+msgids = []
+for msgid in msgid_pattern.findall(po):
+msgid_string = ''.join(msgid_string_pattern.findall(msgid))
+msgid_string = msgid_string.replace(r'\"', '"')
+if msgid_string:
+msgids.append(msgid_string)
+return sorted(msgids)
+
+
+def update_translation_snapshots():
+contents = generate_po_file()
+msgids = extract_msgids(contents)
+snapshot_path.write_text('\n'.join(msgids))
+
+
def tearDownModule():
# Remove global references to avoid looking like we have refleaks.
RFile.seen = {}
@@ -6395,4 +6455,8 @@ def tearDownModule():
if __name__ == '__main__':
+# To regenerate translation snapshots
+if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update':
+update_translation_snapshots()
+sys.exit(0)
unittest.main()
diff --git a/Lib/test/translationdata/argparse/msgids.txt
b/Lib/test/translationdata/argparse/msgids.txt
new file mode 100644
index 00..a1b4f94d8487b7
--- /dev/null
+++ b/Lib/test/translationdata/argparse/msgids.txt
@@ -0,0 +1,36 @@
+ (default: %(default)s)
+%(heading)s:
+%(prog)s: error: %(message)s\n
+%r is not callable
+'required' is an invalid argument for positionals
+.__call__() not defined
+ambiguous option: %(option)s could match %(matches)s
+argument "-" with mode %r
+argument %(argument_name)s: %(message)s
+can't open '%(filename)s': %(error)s
+cannot have multiple subparser arguments
+cannot merge actions - two groups are named %r
+conflicting subparser alias: %s
+conflicting subparser: %s
+dest= is required for options like %r
+expected at least one argument
+expected at most one argument
+expected one argument
+ignored explicit argument %r
+invalid %(type)s value: %(value)r
+invalid choice: %(value)r (choose from %(choices)s)
+invalid conflict_resolution value: %r
+invalid option string %(option)r: must start with a character %(prefix_chars)r
+mutually exclusive arguments must be optional
+not allowed with argument %s
+one of the arguments %s is required
+options
+positional arguments
+show program's version number and exit
+show this help message and exit
+subcommands
+the following arguments are required: %s
+unexpe
[Python-checkins] gh-126035: add missing whitespace to *Py_EnterRecursiveCall() messages (#126036)
https://github.com/python/cpython/commit/19e93e2e269889ecb3c4c039091abff489f247c2 commit: 19e93e2e269889ecb3c4c039091abff489f247c2 branch: main author: Bénédikt Tran <[email protected]> committer: erlend-aasland date: 2024-10-27T22:55:48+01:00 summary: gh-126035: add missing whitespace to *Py_EnterRecursiveCall() messages (#126036) files: M Modules/_bisectmodule.c M Modules/_ctypes/_ctypes.c diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 56322c48b7cd35..9b146265445d9a 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -70,7 +70,7 @@ internal_bisect_right(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t if (sq_item == NULL) { return -1; } -if (Py_EnterRecursiveCall("in _bisect.bisect_right")) { +if (Py_EnterRecursiveCall(" in _bisect.bisect_right")) { return -1; } PyTypeObject *tp = Py_TYPE(item); @@ -254,7 +254,7 @@ internal_bisect_left(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t h if (sq_item == NULL) { return -1; } -if (Py_EnterRecursiveCall("in _bisect.bisect_left")) { +if (Py_EnterRecursiveCall(" in _bisect.bisect_left")) { return -1; } PyTypeObject *tp = Py_TYPE(item); diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index d6a5b75a03d3fe..eae69e484e1660 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2425,7 +2425,7 @@ PyCSimpleType_from_param_impl(PyObject *type, PyTypeObject *cls, return NULL; } if (as_parameter) { -if (_Py_EnterRecursiveCall("while processing _as_parameter_")) { +if (_Py_EnterRecursiveCall(" while processing _as_parameter_")) { Py_DECREF(as_parameter); Py_XDECREF(exc); return NULL; ___ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3/lists/python-checkins.python.org/ Member address: [email protected]
[Python-checkins] [3.12] gh-126035: add missing whitespace to *Py_EnterRecursiveCall() messages (GH-126036) (#126059)
https://github.com/python/cpython/commit/9f35fbc068a1bcaf7eca5dae3ef75eb8449bd9c8 commit: 9f35fbc068a1bcaf7eca5dae3ef75eb8449bd9c8 branch: 3.12 author: Miss Islington (bot) <[email protected]> committer: erlend-aasland date: 2024-10-27T22:12:20Z summary: [3.12] gh-126035: add missing whitespace to *Py_EnterRecursiveCall() messages (GH-126036) (#126059) (cherry picked from commit 19e93e2e269889ecb3c4c039091abff489f247c2) Co-authored-by: Bénédikt Tran <[email protected]> files: M Modules/_bisectmodule.c M Modules/_ctypes/_ctypes.c diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 0773bbd191931d..d79946c84061ae 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -66,7 +66,7 @@ internal_bisect_right(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t if (sq_item == NULL) { return -1; } -if (Py_EnterRecursiveCall("in _bisect.bisect_right")) { +if (Py_EnterRecursiveCall(" in _bisect.bisect_right")) { return -1; } PyTypeObject *tp = Py_TYPE(item); @@ -250,7 +250,7 @@ internal_bisect_left(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t h if (sq_item == NULL) { return -1; } -if (Py_EnterRecursiveCall("in _bisect.bisect_left")) { +if (Py_EnterRecursiveCall(" in _bisect.bisect_left")) { return -1; } PyTypeObject *tp = Py_TYPE(item); diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index d72614d2b8d972..b6d45e92647684 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2269,7 +2269,7 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) return NULL; } if (as_parameter) { -if (_Py_EnterRecursiveCall("while processing _as_parameter_")) { +if (_Py_EnterRecursiveCall(" while processing _as_parameter_")) { Py_DECREF(as_parameter); Py_XDECREF(exc); return NULL; ___ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3/lists/python-checkins.python.org/ Member address: [email protected]
[Python-checkins] [3.13] gh-126035: add missing whitespace to *Py_EnterRecursiveCall() messages (GH-126036) (#126058)
https://github.com/python/cpython/commit/97c2e6ac279e3cd016259a2457ccf4a1505649b3 commit: 97c2e6ac279e3cd016259a2457ccf4a1505649b3 branch: 3.13 author: Miss Islington (bot) <[email protected]> committer: erlend-aasland date: 2024-10-27T22:19:37Z summary: [3.13] gh-126035: add missing whitespace to *Py_EnterRecursiveCall() messages (GH-126036) (#126058) (cherry picked from commit 19e93e2e269889ecb3c4c039091abff489f247c2) Co-authored-by: Bénédikt Tran <[email protected]> files: M Modules/_bisectmodule.c M Modules/_ctypes/_ctypes.c diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 56322c48b7cd35..9b146265445d9a 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -70,7 +70,7 @@ internal_bisect_right(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t if (sq_item == NULL) { return -1; } -if (Py_EnterRecursiveCall("in _bisect.bisect_right")) { +if (Py_EnterRecursiveCall(" in _bisect.bisect_right")) { return -1; } PyTypeObject *tp = Py_TYPE(item); @@ -254,7 +254,7 @@ internal_bisect_left(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t h if (sq_item == NULL) { return -1; } -if (Py_EnterRecursiveCall("in _bisect.bisect_left")) { +if (Py_EnterRecursiveCall(" in _bisect.bisect_left")) { return -1; } PyTypeObject *tp = Py_TYPE(item); diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 236adaf6c4c418..bd98f5edc23188 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2413,7 +2413,7 @@ PyCSimpleType_from_param_impl(PyObject *type, PyTypeObject *cls, return NULL; } if (as_parameter) { -if (_Py_EnterRecursiveCall("while processing _as_parameter_")) { +if (_Py_EnterRecursiveCall(" while processing _as_parameter_")) { Py_DECREF(as_parameter); Py_XDECREF(exc); return NULL; ___ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3/lists/python-checkins.python.org/ Member address: [email protected]
