https://github.com/python/cpython/commit/55eef52e801be5fafa76c26b96a84ee737741dd1 commit: 55eef52e801be5fafa76c26b96a84ee737741dd1 branch: main author: Serhiy Storchaka <[email protected]> committer: serhiy-storchaka <[email protected]> date: 2026-08-29T22:27:09+03:00 summary:
gh-108885: Report the examples of a doctest run by unittest (GH-155377) DocTestCase ran its examples with verbose=False, so there was no way to ask unittest for the details which doctest reports on its own. It now takes the verbosity from the test result, and reports every example if the test runner is asked for more than the test names. They are written to the stream of the test runner, so that they are not lost when it buffers the output of the test. To make this reachable: * unittest.TestResult has now a verbosity attribute, which it accepted but ignored. The test runner sets it, because a result class is free to filter what its constructor gets. * The unittest -v option is now counted, so that -vv means 3. * The verbosity of regrtest is one less, so it is translated where the test runner is created: -v reports the test names, as before, and -vv reports also the examples. Co-authored-by: Claude Opus 5 (1M context) <[email protected]> files: A Misc/NEWS.d/next/Library/2026-08-08-09-14-22.gh-issue-108885.C4ktfw.rst A Misc/NEWS.d/next/Tests/2026-08-08-09-15-08.gh-issue-108885.Qm3vLp.rst M Doc/library/doctest.rst M Doc/library/test.rst M Doc/library/unittest.rst M Lib/doctest.py M Lib/test/libregrtest/testresult.py M Lib/test/test_doctest/test_doctest.py M Lib/test/test_regrtest.py M Lib/test/test_unittest/test_discovery.py M Lib/test/test_unittest/test_program.py M Lib/test/test_unittest/test_result.py M Lib/test/test_unittest/test_runner.py M Lib/unittest/main.py M Lib/unittest/result.py M Lib/unittest/runner.py diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 3298697af8511b6..4b61c07919dd025 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -1162,6 +1162,12 @@ from text files and modules with doctests: .. versionchanged:: 3.15 Run each example as a :ref:`subtest <subtests>`. + .. versionchanged:: next + Report every example, as in verbose mode, if the test runner reports + more than the test names, i.e. its + :attr:`~unittest.TestResult.verbosity` is 3 or higher (for example + with ``python -m unittest -vv``). + Under the covers, :func:`DocTestSuite` creates a :class:`unittest.TestSuite` out of :class:`!doctest.DocTestCase` instances, and :class:`!DocTestCase` is a subclass of :class:`unittest.TestCase`. :class:`!DocTestCase` isn't documented diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 72ffc2a0b760ead..765811eba32fd40 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -232,9 +232,10 @@ The :mod:`!test.support` module defines the following constants: .. data:: verbose - ``True`` when verbose output is enabled. Should be checked when more - detailed information is desired about a running test. *verbose* is set by - :mod:`test.regrtest`. + How verbose the output is: the number of :option:`!-v` options which + :mod:`test.regrtest` was run with, and therefore ``0`` when verbose output + is not enabled. Should be checked when more detailed information is + desired about a running test. .. data:: is_jython diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 7afcdb368a3562e..e1bc32fb79c2eb1 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -180,6 +180,9 @@ You can run tests with more detail (higher verbosity) by passing in the -v flag: python -m unittest -v test_module +Repeat it for even more detail: ``-vv`` reports also the individual examples +of a :mod:`doctest`. + When executed without arguments :ref:`unittest-test-discovery` is started:: python -m unittest @@ -291,7 +294,11 @@ The ``discover`` sub-command has the following options: .. option:: -v, --verbose - Verbose output + Verbose output. May be repeated: ``-vv`` reports also the individual + examples of a :mod:`doctest`. + + .. versionchanged:: next + The option can be repeated. .. option:: -s, --start-directory directory @@ -2149,6 +2156,15 @@ Loading and running tests .. versionadded:: 3.5 + .. attribute:: verbosity + + The level of details which the test runner reports: ``0`` -- quiet, + ``1`` -- progress dots, ``2`` -- test names, ``3`` -- also the + individual examples of a :mod:`doctest`. A test runner is expected to + set it to its own verbosity. + + .. versionadded:: next + .. method:: wasSuccessful() Return ``True`` if all tests run so far have passed, otherwise returns diff --git a/Lib/doctest.py b/Lib/doctest.py index d5541ab41f38d4f..a8f08868cf4957e 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1201,6 +1201,18 @@ def _find_lineno(self, obj, source_lines): ## 5. DocTest Runner ###################################################################### +def _make_output_function(stream): + """Return a function writing to *stream*, whatever it can encode.""" + encoding = getattr(stream, 'encoding', None) + if encoding is None or encoding.lower() == 'utf-8': + return stream.write + def out(s): + # Use backslashreplace error handling on write + s = str(s.encode(encoding, 'backslashreplace'), encoding) + stream.write(s) + return out + + class DocTestRunner: """ A class used to run DocTest test cases, and accumulate statistics. @@ -1561,14 +1573,7 @@ def run(self, test, compileflags=None, out=None, clear_globs=True): save_stdout = sys.stdout if out is None: - encoding = save_stdout.encoding - if encoding is None or encoding.lower() == 'utf-8': - out = save_stdout.write - else: - # Use backslashreplace error handling on write - def out(s): - s = str(s.encode(encoding, 'backslashreplace'), encoding) - save_stdout.write(s) + out = _make_output_function(save_stdout) sys.stdout = self._fakeout # Patch pdb.set_trace to restore sys.stdout during interactive @@ -2322,6 +2327,9 @@ def report_skip(self, out, test, example): unittest.case._addSkip(self._test_result, self._subTest(), '') def report_success(self, out, test, example, got): + # Report "ok" if verbose, to close what report_start() opened. A + # failed or skipped example is reported by the test result instead. + super().report_success(out, test, example, got) self._test_result.addSubTest(self._test_case, self._subTest(), None) def report_unexpected_exception(self, out, test, example, exc_info): @@ -2401,10 +2409,23 @@ def runTest(self): if getattr(result, 'failfast', False): optionflags |= FAIL_FAST + # Report every example only if the test runner is asked for more than + # the test names it reports at verbosity 2. Write them to its stream, + # so that they are not swallowed by result.buffer. + verbose = getattr(result, 'verbosity', 1) >= 3 + stream = getattr(result, 'stream', None) + out = None + if verbose and stream is not None: + out = _make_output_function(stream) + if test.examples and not getattr(result, '_newline', True): + # End the line which startTest() left open. + out('\n') + result._newline = True + runner = _DocTestCaseRunner(optionflags=optionflags, - checker=self._dt_checker, verbose=False, + checker=self._dt_checker, verbose=verbose, test_case=self, test_result=result) - results = runner.run(test, clear_globs=False) + results = runner.run(test, out=out, clear_globs=False) if results.skipped == results.attempted: raise unittest.SkipTest("all examples were skipped") diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 1820f354572521a..605f1f4e6a89fb6 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -16,7 +16,7 @@ class RegressionTestResult(unittest.TextTestResult): def __init__(self, stream, descriptions, verbosity): super().__init__(stream=stream, descriptions=descriptions, - verbosity=2 if verbosity else 0) + verbosity=verbosity) self.buffer = True if self.USE_XML: from xml.etree import ElementTree as ET @@ -150,10 +150,12 @@ def run(self, test): def get_test_runner_class(verbosity, buffer=False): if verbosity: + # The verbosity of regrtest is one less than the verbosity of + # unittest: -v reports the test names, -vv also the doctest examples. return functools.partial(unittest.TextTestRunner, resultclass=RegressionTestResult, buffer=buffer, - verbosity=verbosity) + verbosity=verbosity + 1) return functools.partial(QuietRegressionTestRunner, buffer=buffer) def get_test_runner(stream, verbosity, capture_output=False): diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index b125693ab0891cd..776ad83ee6d6ce5 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -6,6 +6,7 @@ from test.support import import_helper import doctest import functools +import io import os import sys import importlib @@ -469,7 +470,7 @@ def basics(): r""" >>> tests = finder.find(sample_func) >>> print(tests) # doctest: +ELLIPSIS - [<DocTest sample_func from test_doctest.py:36 (1 example)>] + [<DocTest sample_func from test_doctest.py:37 (1 example)>] The exact name depends on how test_doctest was invoked, so allow for leading path components. @@ -803,6 +804,59 @@ def myfunc(): self.assertEqual((x, y), (2, 3)) +class TestDocTestSuiteVerbosity(unittest.TestCase): + + def run_suite(self, module='test.test_doctest.sample_doctest', **kwargs): + """Return what the test runner wrote and what leaked to stdout.""" + suite = doctest.DocTestSuite(module) + stream = io.StringIO() + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + unittest.TextTestRunner(stream=stream, **kwargs).run(suite) + return stream.getvalue(), stdout.getvalue() + + def test_quiet(self): + for verbosity in range(3): + with self.subTest(verbosity=verbosity): + output, stdout = self.run_suite(verbosity=verbosity) + self.assertNotIn('Trying:', output) + self.assertNotIn('Expecting:', output) + self.assertEqual(stdout, '') + + def test_verbose(self): + output, stdout = self.run_suite(verbosity=3) + self.assertIn('Trying:\n 2+2\n', output) + self.assertIn('Expecting:\n 4\n', output) + self.assertIn('\nok\n', output) + # Reported to the stream of the test runner, not to the stdout. + self.assertEqual(stdout, '') + + def test_verbose_buffered(self): + # result.buffer replaces sys.stdout, which would swallow the examples. + output, stdout = self.run_suite(verbosity=3, buffer=True) + self.assertIn('Trying:\n 2+2\n', output) + self.assertEqual(stdout, '') + + def test_verbose_failure_not_duplicated(self): + module = 'test.test_doctest.sample_doctest_errors' + quiet, _ = self.run_suite(module, verbosity=2) + verbose, _ = self.run_suite(module, verbosity=3) + self.assertIn('Trying:', verbose) + self.assertNotIn('Trying:', quiet) + # Reporting the examples does not report the failures once more. + self.assertEqual(verbose.count('Failed example:'), + quiet.count('Failed example:')) + self.assertGreater(quiet.count('Failed example:'), 0) + + def test_plain_result(self): + # A result which is not from a text test runner has no stream. + suite = doctest.DocTestSuite('test.test_doctest.sample_doctest') + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + suite.run(unittest.TestResult()) + self.assertEqual(stdout.getvalue(), '') + + class TestDocTestFinder(unittest.TestCase): def test_issue35753(self): diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index f4baa965b3f632b..c966f8659e2abb0 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -2184,6 +2184,18 @@ def load_tests(loader, tests, pattern): failed=[testname], parallel=True, stats=TestStats(1, 2, 1)) + # A single -v reports the test names, not the examples. + self.assertNotIn('Trying:', output) + + # -vv reports every example, without changing what is run. + output = self.run_tests("--fail-env-changed", "-vv", "-j1", testname, + exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, [testname], + failed=[testname], + parallel=True, + stats=TestStats(1, 2, 1)) + self.assertIn('Trying:\n 1 + 1\n', output) + self.assertIn('Expecting:\n 2\n', output) def _check_random_seed(self, run_workers: bool): # gh-109276: When -r/--randomize is used, random.seed() is called diff --git a/Lib/test/test_unittest/test_discovery.py b/Lib/test/test_unittest/test_discovery.py index 38c9779daaf87d4..da184bd12be8d17 100644 --- a/Lib/test/test_unittest/test_discovery.py +++ b/Lib/test/test_unittest/test_discovery.py @@ -635,7 +635,7 @@ def test_command_line_handling_discover_by_default_with_options(self): program._do_discovery = args.append program.parseArgs(['something', '-v', '-b', '-v', '-c', '-f']) self.assertEqual(args, [[]]) - self.assertEqual(program.verbosity, 2) + self.assertEqual(program.verbosity, 3) # -v is passed twice self.assertIs(program.buffer, True) self.assertIs(program.catchbreak, True) self.assertIs(program.failfast, True) diff --git a/Lib/test/test_unittest/test_program.py b/Lib/test/test_unittest/test_program.py index 8ed92373e5e984a..a9a73f0d057ee81 100644 --- a/Lib/test/test_unittest/test_program.py +++ b/Lib/test/test_unittest/test_program.py @@ -267,6 +267,32 @@ def testVerbosity(self): program.parseArgs([None, opt]) self.assertEqual(program.verbosity, 2) + # -v can be repeated to ask for more details. + for args, verbosity in ( + (['-vv'], 3), + (['-v', '-v'], 3), + (['--verbose', '--verbose'], 3), + (['-vvv'], 4), + # -q overrides any number of -v. + (['-v', '-q'], 0), + ): + with self.subTest(args=args): + program.verbosity = 1 + program.parseArgs([None, *args]) + self.assertEqual(program.verbosity, verbosity) + + def testVerbosityCountedOnce(self): + # "python -m unittest -v" falls back to test discovery, which parses + # arguments again: -v must not be counted twice. + program = self.program + program.verbosity = 1 + program.parseArgs([None, '-v']) + self.assertEqual(program.verbosity, 2) + + program.verbosity = 1 + program.parseArgs([None, 'discover', '-vv']) + self.assertEqual(program.verbosity, 3) + def testBufferCatchFailfast(self): program = self.program for arg, attr in (('buffer', 'buffer'), ('failfast', 'failfast'), diff --git a/Lib/test/test_unittest/test_result.py b/Lib/test/test_unittest/test_result.py index 3f44e617303f81d..cb591c23fc964fd 100644 --- a/Lib/test/test_unittest/test_result.py +++ b/Lib/test/test_unittest/test_result.py @@ -54,6 +54,13 @@ def test_init(self): self.assertEqual(result.shouldStop, False) self.assertIsNone(result._stdout_buffer) self.assertIsNone(result._stderr_buffer) + self.assertEqual(result.verbosity, 1) + + def test_init_verbosity(self): + for verbosity in range(4): + with self.subTest(verbosity=verbosity): + result = unittest.TestResult(None, None, verbosity) + self.assertEqual(result.verbosity, verbosity) # "This method can be called to signal that the set of tests being # run should be aborted by setting the TestResult's shouldStop diff --git a/Lib/test/test_unittest/test_runner.py b/Lib/test/test_unittest/test_runner.py index a47e2ebb59da028..195d5e928030927 100644 --- a/Lib/test/test_unittest/test_runner.py +++ b/Lib/test/test_unittest/test_runner.py @@ -1363,6 +1363,35 @@ def MockResultClass(*args): expectedresult = (runner.stream, DESCRIPTIONS, VERBOSITY) self.assertEqual(runner._makeResult(), expectedresult) + def test_verbosity_set_on_result(self): + class Suite: + def __call__(self, result): + pass + + for verbosity in range(4): + with self.subTest(verbosity=verbosity): + runner = unittest.TextTestRunner(io.StringIO(), + verbosity=verbosity) + result = runner.run(Suite()) + self.assertEqual(result.verbosity, verbosity) + + def test_verbosity_set_on_filtering_result(self): + # A result class is free to filter the verbosity which its + # constructor gets, as test.libregrtest does. + class FilteringResult(unittest.TextTestResult): + def __init__(self, stream, descriptions, verbosity): + super().__init__(stream, descriptions, + 2 if verbosity else 0) + + class Suite: + def __call__(self, result): + pass + + runner = unittest.TextTestRunner(io.StringIO(), verbosity=3, + resultclass=FilteringResult) + result = runner.run(Suite()) + self.assertEqual(result.verbosity, 3) + @support.force_not_colorized @support.requires_subprocess() def test_warnings(self): diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index 6eeebf9657a3c7b..850c825a7366579 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -161,9 +161,12 @@ def _initArgParsers(self): def _getParentArgParser(self): parser = argparse.ArgumentParser(add_help=False) + # Counted, not a constant: the namespace is the TestProgram, whose + # verbosity is already 1, so -v still gives 2 and -vv gives 3. parser.add_argument('-v', '--verbose', dest='verbosity', - action='store_const', const=2, - help='Verbose output') + action='count', default=1, + help='Verbose output, twice to also report ' + 'the examples of a doctest') parser.add_argument('-q', '--quiet', dest='verbosity', action='store_const', const=0, help='Quiet output') diff --git a/Lib/unittest/result.py b/Lib/unittest/result.py index b8ea396db6772ed..a787686a5bea4ce 100644 --- a/Lib/unittest/result.py +++ b/Lib/unittest/result.py @@ -37,6 +37,9 @@ class TestResult(object): _moduleSetUpFailed = False def __init__(self, stream=None, descriptions=None, verbosity=None): self.failfast = False + # How much the test runner reports: 0 -- quiet, 1 -- progress dots, + # 2 -- test names, 3 -- also the examples of a doctest. + self.verbosity = 1 if verbosity is None else verbosity self.failures = [] self.errors = [] self.testsRun = 0 diff --git a/Lib/unittest/runner.py b/Lib/unittest/runner.py index 893fcba968c3ef8..f19d7b6e446d2ac 100644 --- a/Lib/unittest/runner.py +++ b/Lib/unittest/runner.py @@ -244,6 +244,9 @@ def run(self, test): result.failfast = self.failfast result.buffer = self.buffer result.tb_locals = self.tb_locals + # Not left to _makeResult(): a result class is free to filter the + # verbosity which its constructor gets. + result.verbosity = self.verbosity with warnings.catch_warnings(): if self.warnings: # if self.warnings is set, use it to filter all the warnings diff --git a/Misc/NEWS.d/next/Library/2026-08-08-09-14-22.gh-issue-108885.C4ktfw.rst b/Misc/NEWS.d/next/Library/2026-08-08-09-14-22.gh-issue-108885.C4ktfw.rst new file mode 100644 index 000000000000000..e504f9b7d684f88 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-08-09-14-22.gh-issue-108885.C4ktfw.rst @@ -0,0 +1,6 @@ +Doctests run by the :mod:`unittest` test runner now report every example, as +in verbose mode, if the runner reports more than the test names, i.e. its +verbosity is 3 or higher. The :option:`!-v` option of :mod:`unittest` can now +be repeated, so ``python -m unittest -vv`` asks for this. Added also the +:attr:`~unittest.TestResult.verbosity` attribute of +:class:`unittest.TestResult`. diff --git a/Misc/NEWS.d/next/Tests/2026-08-08-09-15-08.gh-issue-108885.Qm3vLp.rst b/Misc/NEWS.d/next/Tests/2026-08-08-09-15-08.gh-issue-108885.Qm3vLp.rst new file mode 100644 index 000000000000000..3cdac196dd35c0c --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-08-08-09-15-08.gh-issue-108885.Qm3vLp.rst @@ -0,0 +1,2 @@ +Running the Python test suite with ``-vv`` now reports every example of a +doctest. A single ``-v`` reports the test names, as before. _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
