Added REPORT_ONLY_FIRST_FAILURE flag, which supresses output after the
first failing example in each test.
diff --git a/Doc/lib/libdoctest.tex b/Doc/lib/libdoctest.tex
index dca79f7..99fbeb4 100644
--- a/Doc/lib/libdoctest.tex
+++ b/Doc/lib/libdoctest.tex
@@ -364,6 +364,17 @@
positions.
\end{datadesc}
+\begin{datadesc}{REPORT_ONLY_FIRST_FAILURE}
+ When specified, display the first failing example in each doctest,
+ but suppress output for all remaining examples. This will prevent
+ doctest from reporting correct examples that break because of
+ earlier failures; but it might also hide incorrect examples that
+ fail independently of the first failure. When
+ \constant{REPORT_ONLY_FIRST_FAILURE} is specified, the remaining
+ examples are still run, and still count towards the total number of
+ failures reported; only the output is suppressed.
+\end{datadesc}
+
A "doctest directive" is a trailing Python comment on a line of a doctest
example:
@@ -421,8 +432,8 @@
\versionchanged[Constants \constant{DONT_ACCEPT_BLANKLINE},
\constant{NORMALIZE_WHITESPACE}, \constant{ELLIPSIS},
- \constant{REPORT_UDIFF}, \constant{REPORT_CDIFF}, and
- \constant{REPORT_NDIFF}
+ \constant{REPORT_UDIFF}, \constant{REPORT_CDIFF},
+ \constant{REPORT_NDIFF}, and \constant{REPORT_ONLY_FIRST_FAILURE}
were added; by default \code{<BLANKLINE>} in expected output
matches an empty line in actual output; and doctest directives
were added]{2.4}
diff --git a/Lib/doctest.py b/Lib/doctest.py
index 5e83c18..d7978fd 100644
--- a/Lib/doctest.py
+++ b/Lib/doctest.py
@@ -260,6 +260,7 @@
REPORT_UDIFF = register_optionflag('REPORT_UDIFF')
REPORT_CDIFF = register_optionflag('REPORT_CDIFF')
REPORT_NDIFF = register_optionflag('REPORT_NDIFF')
+REPORT_ONLY_FIRST_FAILURE = register_optionflag('REPORT_ONLY_FIRST_FAILURE')
# Special string markers for use in `want` strings:
BLANKLINE_MARKER = '<BLANKLINE>'
@@ -1280,7 +1281,6 @@
"""
Report that the given example failed.
"""
- # Print an error message.
out(self._failure_header(test, example) +
self._checker.output_difference(example.want, got,
self.optionflags))
@@ -1331,6 +1331,11 @@
# Process each example.
for example in test.examples:
+ # If REPORT_ONLY_FIRST_FAILURE is set, then supress
+ # reporting after the first failure.
+ quiet = (self.optionflags & REPORT_ONLY_FIRST_FAILURE and
+ failures > 0)
+
# Merge in the example's options.
self.optionflags = original_optionflags
if example.options:
@@ -1342,7 +1347,8 @@
# Record that we started this example.
tries += 1
- self.report_start(out, test, example)
+ if not quiet:
+ self.report_start(out, test, example)
# Run the example in the given context (globs), and record
# any exception that gets raised. (But don't intercept
@@ -1365,9 +1371,11 @@
if exception is None:
if self._checker.check_output(example.want, got,
self.optionflags):
- self.report_success(out, test, example, got)
+ if not quiet:
+ self.report_success(out, test, example, got)
else:
- self.report_failure(out, test, example, got)
+ if not quiet:
+ self.report_failure(out, test, example, got)
failures += 1
# If the example raised an exception, then check if it was
@@ -1379,19 +1387,22 @@
# If `example.exc_msg` is None, then we weren't
# expecting an exception.
if example.exc_msg is None:
- self.report_unexpected_exception(out, test, example,
- exc_info)
+ if not quiet:
+ self.report_unexpected_exception(out, test, example,
+ exc_info)
failures += 1
# If `example.exc_msg` matches the actual exception
# message (`exc_msg`), then the example succeeds.
elif (self._checker.check_output(example.exc_msg, exc_msg,
self.optionflags)):
- self.report_success(out, test, example,
- got + _exception_traceback(exc_info))
+ if not quiet:
+ got += _exception_traceback(exc_info)
+ self.report_success(out, test, example, got)
# Otherwise, the example fails.
else:
- self.report_failure(out, test, example,
- got + _exception_traceback(exc_info))
+ if not quiet:
+ got += _exception_traceback(exc_info)
+ self.report_failure(out, test, example, got)
failures += 1
# Restore the option flags (in case they were modified)
@@ -1842,6 +1853,7 @@
REPORT_UDIFF
REPORT_CDIFF
REPORT_NDIFF
+ REPORT_ONLY_FIRST_FAILURE
Optional keyword arg "raise_on_error" raises an exception on the
first unexpected exception or failure. This allows failures to be
diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py
index d683051..2464b23 100644
--- a/Lib/test/test_doctest.py
+++ b/Lib/test/test_doctest.py
@@ -1042,6 +1042,87 @@
? + ++ ^
<BLANKLINE>
(1, 1)
+
+The REPORT_ONLY_FIRST_FAILURE supresses result output after the first
+failing example:
+
+ >>> def f(x):
+ ... r'''
+ ... >>> print 1 # first success
+ ... 1
+ ... >>> print 2 # first failure
+ ... 200
+ ... >>> print 3 # second failure
+ ... 300
+ ... >>> print 4 # second success
+ ... 4
+ ... >>> print 5 # third failure
+ ... 500
+ ... '''
+ >>> test = doctest.DocTestFinder().find(f)[0]
+ >>> flags = doctest.REPORT_ONLY_FIRST_FAILURE
+ >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test)
+ **********************************************************************
+ Line 4, in f
+ Failed example:
+ print 2 # first failure
+ Expected:
+ 200
+ Got:
+ 2
+ (3, 5)
+
+However, output from `report_start` is not supressed:
+
+ >>> doctest.DocTestRunner(verbose=True, optionflags=flags).run(test)
+ Trying:
+ print 1 # first success
+ Expecting:
+ 1
+ ok
+ Trying:
+ print 2 # first failure
+ Expecting:
+ 200
+ **********************************************************************
+ Line 4, in f
+ Failed example:
+ print 2 # first failure
+ Expected:
+ 200
+ Got:
+ 2
+ (3, 5)
+
+For the purposes of REPORT_ONLY_FIRST_FAILURE, unexpected exceptions
+count as failures:
+
+ >>> def f(x):
+ ... r'''
+ ... >>> print 1 # first success
+ ... 1
+ ... >>> raise ValueError(2) # first failure
+ ... 200
+ ... >>> print 3 # second failure
+ ... 300
+ ... >>> print 4 # second success
+ ... 4
+ ... >>> print 5 # third failure
+ ... 500
+ ... '''
+ >>> test = doctest.DocTestFinder().find(f)[0]
+ >>> flags = doctest.REPORT_ONLY_FIRST_FAILURE
+ >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test)
+ ... # doctest: +ELLIPSIS
+ **********************************************************************
+ Line 4, in f
+ Failed example:
+ raise ValueError(2) # first failure
+ Exception raised:
+ ...
+ ValueError: 2
+ (3, 5)
+
"""
def option_directives(): r"""