diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 3298697af8511b6..4b61c07919dd025 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -1162,6 +1162,12 @@ from text files and modules with doctests: .. versionchanged:: 3.15 Run each example as a :ref:`subtest `. + .. versionchanged:: next + Report every example, as in verbose mode, if the test runner reports + more than the test names, i.e. its + :attr:`~unittest.TestResult.verbosity` is 3 or higher (for example + with ``python -m unittest -vv``). + Under the covers, :func:`DocTestSuite` creates a :class:`unittest.TestSuite` out of :class:`!doctest.DocTestCase` instances, and :class:`!DocTestCase` is a subclass of :class:`unittest.TestCase`. :class:`!DocTestCase` isn't documented diff --git a/Doc/library/test.rst b/Doc/library/test.rst index f3b5383658b5ac4..ba6600fc46679f9 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -232,9 +232,10 @@ The :mod:`!test.support` module defines the following constants: .. data:: verbose - ``True`` when verbose output is enabled. Should be checked when more - detailed information is desired about a running test. *verbose* is set by - :mod:`test.regrtest`. + How verbose the output is: the number of :option:`!-v` options which + :mod:`test.regrtest` was run with, and therefore ``0`` when verbose output + is not enabled. Should be checked when more detailed information is + desired about a running test. .. data:: is_jython diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 7afcdb368a3562e..e1bc32fb79c2eb1 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -180,6 +180,9 @@ You can run tests with more detail (higher verbosity) by passing in the -v flag: python -m unittest -v test_module +Repeat it for even more detail: ``-vv`` reports also the individual examples +of a :mod:`doctest`. + When executed without arguments :ref:`unittest-test-discovery` is started:: python -m unittest @@ -291,7 +294,11 @@ The ``discover`` sub-command has the following options: .. option:: -v, --verbose - Verbose output + Verbose output. May be repeated: ``-vv`` reports also the individual + examples of a :mod:`doctest`. + + .. versionchanged:: next + The option can be repeated. .. option:: -s, --start-directory directory @@ -2149,6 +2156,15 @@ Loading and running tests .. versionadded:: 3.5 + .. attribute:: verbosity + + The level of details which the test runner reports: ``0`` -- quiet, + ``1`` -- progress dots, ``2`` -- test names, ``3`` -- also the + individual examples of a :mod:`doctest`. A test runner is expected to + set it to its own verbosity. + + .. versionadded:: next + .. method:: wasSuccessful() Return ``True`` if all tests run so far have passed, otherwise returns diff --git a/Lib/doctest.py b/Lib/doctest.py index 8a55fe3ddd26154..dceea82f54bcd79 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1200,6 +1200,18 @@ def _find_lineno(self, obj, source_lines): ## 5. DocTest Runner ###################################################################### +def _make_output_function(stream): + """Return a function writing to *stream*, whatever it can encode.""" + encoding = getattr(stream, 'encoding', None) + if encoding is None or encoding.lower() == 'utf-8': + return stream.write + def out(s): + # Use backslashreplace error handling on write + s = str(s.encode(encoding, 'backslashreplace'), encoding) + stream.write(s) + return out + + class DocTestRunner: """ A class used to run DocTest test cases, and accumulate statistics. @@ -1560,14 +1572,7 @@ def run(self, test, compileflags=None, out=None, clear_globs=True): save_stdout = sys.stdout if out is None: - encoding = save_stdout.encoding - if encoding is None or encoding.lower() == 'utf-8': - out = save_stdout.write - else: - # Use backslashreplace error handling on write - def out(s): - s = str(s.encode(encoding, 'backslashreplace'), encoding) - save_stdout.write(s) + out = _make_output_function(save_stdout) sys.stdout = self._fakeout # Patch pdb.set_trace to restore sys.stdout during interactive @@ -2321,6 +2326,9 @@ def report_skip(self, out, test, example): unittest.case._addSkip(self._test_result, self._subTest(), '') def report_success(self, out, test, example, got): + # Report "ok" if verbose, to close what report_start() opened. A + # failed or skipped example is reported by the test result instead. + super().report_success(out, test, example, got) self._test_result.addSubTest(self._test_case, self._subTest(), None) def report_unexpected_exception(self, out, test, example, exc_info): @@ -2400,10 +2408,23 @@ def runTest(self): if getattr(result, 'failfast', False): optionflags |= FAIL_FAST + # Report every example only if the test runner is asked for more than + # the test names it reports at verbosity 2. Write them to its stream, + # so that they are not swallowed by result.buffer. + verbose = getattr(result, 'verbosity', 1) >= 3 + stream = getattr(result, 'stream', None) + out = None + if verbose and stream is not None: + out = _make_output_function(stream) + if test.examples and not getattr(result, '_newline', True): + # End the line which startTest() left open. + out('\n') + result._newline = True + runner = _DocTestCaseRunner(optionflags=optionflags, - checker=self._dt_checker, verbose=False, + checker=self._dt_checker, verbose=verbose, test_case=self, test_result=result) - results = runner.run(test, clear_globs=False) + results = runner.run(test, out=out, clear_globs=False) if results.skipped == results.attempted: raise unittest.SkipTest("all examples were skipped") diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 1820f354572521a..605f1f4e6a89fb6 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -16,7 +16,7 @@ class RegressionTestResult(unittest.TextTestResult): def __init__(self, stream, descriptions, verbosity): super().__init__(stream=stream, descriptions=descriptions, - verbosity=2 if verbosity else 0) + verbosity=verbosity) self.buffer = True if self.USE_XML: from xml.etree import ElementTree as ET @@ -150,10 +150,12 @@ def run(self, test): def get_test_runner_class(verbosity, buffer=False): if verbosity: + # The verbosity of regrtest is one less than the verbosity of + # unittest: -v reports the test names, -vv also the doctest examples. return functools.partial(unittest.TextTestRunner, resultclass=RegressionTestResult, buffer=buffer, - verbosity=verbosity) + verbosity=verbosity + 1) return functools.partial(QuietRegressionTestRunner, buffer=buffer) def get_test_runner(stream, verbosity, capture_output=False): diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index b125693ab0891cd..776ad83ee6d6ce5 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -6,6 +6,7 @@ from test.support import import_helper import doctest import functools +import io import os import sys import importlib @@ -469,7 +470,7 @@ def basics(): r""" >>> tests = finder.find(sample_func) >>> print(tests) # doctest: +ELLIPSIS - [] + [] The exact name depends on how test_doctest was invoked, so allow for leading path components. @@ -803,6 +804,59 @@ def myfunc(): self.assertEqual((x, y), (2, 3)) +class TestDocTestSuiteVerbosity(unittest.TestCase): + + def run_suite(self, module='test.test_doctest.sample_doctest', **kwargs): + """Return what the test runner wrote and what leaked to stdout.""" + suite = doctest.DocTestSuite(module) + stream = io.StringIO() + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + unittest.TextTestRunner(stream=stream, **kwargs).run(suite) + return stream.getvalue(), stdout.getvalue() + + def test_quiet(self): + for verbosity in range(3): + with self.subTest(verbosity=verbosity): + output, stdout = self.run_suite(verbosity=verbosity) + self.assertNotIn('Trying:', output) + self.assertNotIn('Expecting:', output) + self.assertEqual(stdout, '') + + def test_verbose(self): + output, stdout = self.run_suite(verbosity=3) + self.assertIn('Trying:\n 2+2\n', output) + self.assertIn('Expecting:\n 4\n', output) + self.assertIn('\nok\n', output) + # Reported to the stream of the test runner, not to the stdout. + self.assertEqual(stdout, '') + + def test_verbose_buffered(self): + # result.buffer replaces sys.stdout, which would swallow the examples. + output, stdout = self.run_suite(verbosity=3, buffer=True) + self.assertIn('Trying:\n 2+2\n', output) + self.assertEqual(stdout, '') + + def test_verbose_failure_not_duplicated(self): + module = 'test.test_doctest.sample_doctest_errors' + quiet, _ = self.run_suite(module, verbosity=2) + verbose, _ = self.run_suite(module, verbosity=3) + self.assertIn('Trying:', verbose) + self.assertNotIn('Trying:', quiet) + # Reporting the examples does not report the failures once more. + self.assertEqual(verbose.count('Failed example:'), + quiet.count('Failed example:')) + self.assertGreater(quiet.count('Failed example:'), 0) + + def test_plain_result(self): + # A result which is not from a text test runner has no stream. + suite = doctest.DocTestSuite('test.test_doctest.sample_doctest') + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + suite.run(unittest.TestResult()) + self.assertEqual(stdout.getvalue(), '') + + class TestDocTestFinder(unittest.TestCase): def test_issue35753(self): diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 6ba440053089161..1245c229722f04b 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -2151,6 +2151,18 @@ def load_tests(loader, tests, pattern): failed=[testname], parallel=True, stats=TestStats(1, 2, 1)) + # A single -v reports the test names, not the examples. + self.assertNotIn('Trying:', output) + + # -vv reports every example, without changing what is run. + output = self.run_tests("--fail-env-changed", "-vv", "-j1", testname, + exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, [testname], + failed=[testname], + parallel=True, + stats=TestStats(1, 2, 1)) + self.assertIn('Trying:\n 1 + 1\n', output) + self.assertIn('Expecting:\n 2\n', output) def _check_random_seed(self, run_workers: bool): # gh-109276: When -r/--randomize is used, random.seed() is called diff --git a/Lib/test/test_unittest/test_discovery.py b/Lib/test/test_unittest/test_discovery.py index 38c9779daaf87d4..da184bd12be8d17 100644 --- a/Lib/test/test_unittest/test_discovery.py +++ b/Lib/test/test_unittest/test_discovery.py @@ -635,7 +635,7 @@ def test_command_line_handling_discover_by_default_with_options(self): program._do_discovery = args.append program.parseArgs(['something', '-v', '-b', '-v', '-c', '-f']) self.assertEqual(args, [[]]) - self.assertEqual(program.verbosity, 2) + self.assertEqual(program.verbosity, 3) # -v is passed twice self.assertIs(program.buffer, True) self.assertIs(program.catchbreak, True) self.assertIs(program.failfast, True) diff --git a/Lib/test/test_unittest/test_program.py b/Lib/test/test_unittest/test_program.py index 8ed92373e5e984a..a9a73f0d057ee81 100644 --- a/Lib/test/test_unittest/test_program.py +++ b/Lib/test/test_unittest/test_program.py @@ -267,6 +267,32 @@ def testVerbosity(self): program.parseArgs([None, opt]) self.assertEqual(program.verbosity, 2) + # -v can be repeated to ask for more details. + for args, verbosity in ( + (['-vv'], 3), + (['-v', '-v'], 3), + (['--verbose', '--verbose'], 3), + (['-vvv'], 4), + # -q overrides any number of -v. + (['-v', '-q'], 0), + ): + with self.subTest(args=args): + program.verbosity = 1 + program.parseArgs([None, *args]) + self.assertEqual(program.verbosity, verbosity) + + def testVerbosityCountedOnce(self): + # "python -m unittest -v" falls back to test discovery, which parses + # arguments again: -v must not be counted twice. + program = self.program + program.verbosity = 1 + program.parseArgs([None, '-v']) + self.assertEqual(program.verbosity, 2) + + program.verbosity = 1 + program.parseArgs([None, 'discover', '-vv']) + self.assertEqual(program.verbosity, 3) + def testBufferCatchFailfast(self): program = self.program for arg, attr in (('buffer', 'buffer'), ('failfast', 'failfast'), diff --git a/Lib/test/test_unittest/test_result.py b/Lib/test/test_unittest/test_result.py index 3f44e617303f81d..cb591c23fc964fd 100644 --- a/Lib/test/test_unittest/test_result.py +++ b/Lib/test/test_unittest/test_result.py @@ -54,6 +54,13 @@ def test_init(self): self.assertEqual(result.shouldStop, False) self.assertIsNone(result._stdout_buffer) self.assertIsNone(result._stderr_buffer) + self.assertEqual(result.verbosity, 1) + + def test_init_verbosity(self): + for verbosity in range(4): + with self.subTest(verbosity=verbosity): + result = unittest.TestResult(None, None, verbosity) + self.assertEqual(result.verbosity, verbosity) # "This method can be called to signal that the set of tests being # run should be aborted by setting the TestResult's shouldStop diff --git a/Lib/test/test_unittest/test_runner.py b/Lib/test/test_unittest/test_runner.py index a47e2ebb59da028..195d5e928030927 100644 --- a/Lib/test/test_unittest/test_runner.py +++ b/Lib/test/test_unittest/test_runner.py @@ -1363,6 +1363,35 @@ def MockResultClass(*args): expectedresult = (runner.stream, DESCRIPTIONS, VERBOSITY) self.assertEqual(runner._makeResult(), expectedresult) + def test_verbosity_set_on_result(self): + class Suite: + def __call__(self, result): + pass + + for verbosity in range(4): + with self.subTest(verbosity=verbosity): + runner = unittest.TextTestRunner(io.StringIO(), + verbosity=verbosity) + result = runner.run(Suite()) + self.assertEqual(result.verbosity, verbosity) + + def test_verbosity_set_on_filtering_result(self): + # A result class is free to filter the verbosity which its + # constructor gets, as test.libregrtest does. + class FilteringResult(unittest.TextTestResult): + def __init__(self, stream, descriptions, verbosity): + super().__init__(stream, descriptions, + 2 if verbosity else 0) + + class Suite: + def __call__(self, result): + pass + + runner = unittest.TextTestRunner(io.StringIO(), verbosity=3, + resultclass=FilteringResult) + result = runner.run(Suite()) + self.assertEqual(result.verbosity, 3) + @support.force_not_colorized @support.requires_subprocess() def test_warnings(self): diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index 6eeebf9657a3c7b..850c825a7366579 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -161,9 +161,12 @@ def _initArgParsers(self): def _getParentArgParser(self): parser = argparse.ArgumentParser(add_help=False) + # Counted, not a constant: the namespace is the TestProgram, whose + # verbosity is already 1, so -v still gives 2 and -vv gives 3. parser.add_argument('-v', '--verbose', dest='verbosity', - action='store_const', const=2, - help='Verbose output') + action='count', default=1, + help='Verbose output, twice to also report ' + 'the examples of a doctest') parser.add_argument('-q', '--quiet', dest='verbosity', action='store_const', const=0, help='Quiet output') diff --git a/Lib/unittest/result.py b/Lib/unittest/result.py index b8ea396db6772ed..a787686a5bea4ce 100644 --- a/Lib/unittest/result.py +++ b/Lib/unittest/result.py @@ -37,6 +37,9 @@ class TestResult(object): _moduleSetUpFailed = False def __init__(self, stream=None, descriptions=None, verbosity=None): self.failfast = False + # How much the test runner reports: 0 -- quiet, 1 -- progress dots, + # 2 -- test names, 3 -- also the examples of a doctest. + self.verbosity = 1 if verbosity is None else verbosity self.failures = [] self.errors = [] self.testsRun = 0 diff --git a/Lib/unittest/runner.py b/Lib/unittest/runner.py index 893fcba968c3ef8..f19d7b6e446d2ac 100644 --- a/Lib/unittest/runner.py +++ b/Lib/unittest/runner.py @@ -244,6 +244,9 @@ def run(self, test): result.failfast = self.failfast result.buffer = self.buffer result.tb_locals = self.tb_locals + # Not left to _makeResult(): a result class is free to filter the + # verbosity which its constructor gets. + result.verbosity = self.verbosity with warnings.catch_warnings(): if self.warnings: # if self.warnings is set, use it to filter all the warnings diff --git a/Misc/NEWS.d/next/Library/2026-08-08-09-14-22.gh-issue-108885.C4ktfw.rst b/Misc/NEWS.d/next/Library/2026-08-08-09-14-22.gh-issue-108885.C4ktfw.rst new file mode 100644 index 000000000000000..e504f9b7d684f88 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-08-09-14-22.gh-issue-108885.C4ktfw.rst @@ -0,0 +1,6 @@ +Doctests run by the :mod:`unittest` test runner now report every example, as +in verbose mode, if the runner reports more than the test names, i.e. its +verbosity is 3 or higher. The :option:`!-v` option of :mod:`unittest` can now +be repeated, so ``python -m unittest -vv`` asks for this. Added also the +:attr:`~unittest.TestResult.verbosity` attribute of +:class:`unittest.TestResult`. diff --git a/Misc/NEWS.d/next/Tests/2026-08-08-09-15-08.gh-issue-108885.Qm3vLp.rst b/Misc/NEWS.d/next/Tests/2026-08-08-09-15-08.gh-issue-108885.Qm3vLp.rst new file mode 100644 index 000000000000000..3cdac196dd35c0c --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-08-08-09-15-08.gh-issue-108885.Qm3vLp.rst @@ -0,0 +1,2 @@ +Running the Python test suite with ``-vv`` now reports every example of a +doctest. A single ``-v`` reports the test names, as before.