bpo-34582: Adds JUnit XML output for regression tests (GH-9210)
diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py
index bd126b3..538ff05 100644
--- a/Lib/test/libregrtest/cmdline.py
+++ b/Lib/test/libregrtest/cmdline.py
@@ -268,6 +268,10 @@
help='if a test file alters the environment, mark '
'the test as failed')
+ group.add_argument('--junit-xml', dest='xmlpath', metavar='FILENAME',
+ help='writes JUnit-style XML results to the specified '
+ 'file')
+
return parser
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index a176db5..1fd4132 100644
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -100,8 +100,11 @@
self.next_single_test = None
self.next_single_filename = None
+ # used by --junit-xml
+ self.testsuite_xml = None
+
def accumulate_result(self, test, result):
- ok, test_time = result
+ ok, test_time, xml_data = result
if ok not in (CHILD_ERROR, INTERRUPTED):
self.test_times.append((test_time, test))
if ok == PASSED:
@@ -118,6 +121,15 @@
elif ok != INTERRUPTED:
raise ValueError("invalid test result: %r" % ok)
+ if xml_data:
+ import xml.etree.ElementTree as ET
+ for e in xml_data:
+ try:
+ self.testsuite_xml.append(ET.fromstring(e))
+ except ET.ParseError:
+ print(xml_data, file=sys.__stderr__)
+ raise
+
def display_progress(self, test_index, test):
if self.ns.quiet:
return
@@ -164,6 +176,9 @@
file=sys.stderr)
ns.findleaks = False
+ if ns.xmlpath:
+ support.junit_xml_list = self.testsuite_xml = []
+
# Strip .py extensions.
removepy(ns.args)
@@ -384,7 +399,7 @@
result = runtest(self.ns, test)
except KeyboardInterrupt:
self.interrupted = True
- self.accumulate_result(test, (INTERRUPTED, None))
+ self.accumulate_result(test, (INTERRUPTED, None, None))
break
else:
self.accumulate_result(test, result)
@@ -508,6 +523,31 @@
if self.ns.runleaks:
os.system("leaks %d" % os.getpid())
+ def save_xml_result(self):
+ if not self.ns.xmlpath and not self.testsuite_xml:
+ return
+
+ import xml.etree.ElementTree as ET
+ root = ET.Element("testsuites")
+
+ # Manually count the totals for the overall summary
+ totals = {'tests': 0, 'errors': 0, 'failures': 0}
+ for suite in self.testsuite_xml:
+ root.append(suite)
+ for k in totals:
+ try:
+ totals[k] += int(suite.get(k, 0))
+ except ValueError:
+ pass
+
+ for k, v in totals.items():
+ root.set(k, str(v))
+
+ xmlpath = os.path.join(support.SAVEDCWD, self.ns.xmlpath)
+ with open(xmlpath, 'wb') as f:
+ for s in ET.tostringlist(root):
+ f.write(s)
+
def main(self, tests=None, **kwargs):
global TEMPDIR
@@ -570,6 +610,9 @@
self.rerun_failed_tests()
self.finalize()
+
+ self.save_xml_result()
+
if self.bad:
sys.exit(2)
if self.interrupted:
diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py
index 3e1afd4..4f41080 100644
--- a/Lib/test/libregrtest/runtest.py
+++ b/Lib/test/libregrtest/runtest.py
@@ -85,8 +85,8 @@
ns -- regrtest namespace of options
test -- the name of the test
- Returns the tuple (result, test_time), where result is one of the
- constants:
+ Returns the tuple (result, test_time, xml_data), where result is one
+ of the constants:
INTERRUPTED KeyboardInterrupt when run under -j
RESOURCE_DENIED test skipped because resource denied
@@ -94,6 +94,9 @@
ENV_CHANGED test failed because it changed the execution environment
FAILED test failed
PASSED test passed
+
+ If ns.xmlpath is not None, xml_data is a list containing each
+ generated testsuite element.
"""
output_on_failure = ns.verbose3
@@ -106,22 +109,13 @@
# reset the environment_altered flag to detect if a test altered
# the environment
support.environment_altered = False
+ support.junit_xml_list = xml_list = [] if ns.xmlpath else None
if ns.failfast:
support.failfast = True
if output_on_failure:
support.verbose = True
- # Reuse the same instance to all calls to runtest(). Some
- # tests keep a reference to sys.stdout or sys.stderr
- # (eg. test_argparse).
- if runtest.stringio is None:
- stream = io.StringIO()
- runtest.stringio = stream
- else:
- stream = runtest.stringio
- stream.seek(0)
- stream.truncate()
-
+ stream = io.StringIO()
orig_stdout = sys.stdout
orig_stderr = sys.stderr
try:
@@ -138,12 +132,18 @@
else:
support.verbose = ns.verbose # Tell tests to be moderately quiet
result = runtest_inner(ns, test, display_failure=not ns.verbose)
- return result
+
+ if xml_list:
+ import xml.etree.ElementTree as ET
+ xml_data = [ET.tostring(x).decode('us-ascii') for x in xml_list]
+ else:
+ xml_data = None
+ return result + (xml_data,)
finally:
if use_timeout:
faulthandler.cancel_dump_traceback_later()
cleanup_test_droppings(test, ns.verbose)
-runtest.stringio = None
+ support.junit_xml_list = None
def post_test_cleanup():
diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py
index 1f07cfb..6190574 100644
--- a/Lib/test/libregrtest/runtest_mp.py
+++ b/Lib/test/libregrtest/runtest_mp.py
@@ -67,7 +67,7 @@
try:
result = runtest(ns, testname)
except KeyboardInterrupt:
- result = INTERRUPTED, ''
+ result = INTERRUPTED, '', None
except BaseException as e:
traceback.print_exc()
result = CHILD_ERROR, str(e)
@@ -122,7 +122,7 @@
self.current_test = None
if retcode != 0:
- result = (CHILD_ERROR, "Exit code %s" % retcode)
+ result = (CHILD_ERROR, "Exit code %s" % retcode, None)
self.output.put((test, stdout.rstrip(), stderr.rstrip(),
result))
return False
@@ -133,6 +133,7 @@
return True
result = json.loads(result)
+ assert len(result) == 3, f"Invalid result tuple: {result!r}"
self.output.put((test, stdout.rstrip(), stderr.rstrip(),
result))
return False
@@ -195,7 +196,7 @@
regrtest.accumulate_result(test, result)
# Display progress
- ok, test_time = result
+ ok, test_time, xml_data = result
text = format_test_result(test, ok)
if (ok not in (CHILD_ERROR, INTERRUPTED)
and test_time >= PROGRESS_MIN_TIME