pythongh-121188: Escape XML characters in regrtest

When creating the JUnit XML file, regrtest now escapes characters
which are invalid in XML such as the chr(27) control character used
in ANSI escape sequences.

Author: vstinner

Lib/test/libregrtest/testresult.py

@@ -9,6 +9,7 @@
 import traceback
 import unittest
 from test import support
+from test.libregrtest.utils import escape_xml
 
 class RegressionTestResult(unittest.TextTestResult):
     USE_XML = False
@@ -65,23 +66,24 @@ def _add_result(self, test, capture=False, **args):
         if capture:
             if self._stdout_buffer is not None:
                 stdout = self._stdout_buffer.getvalue().rstrip()
-                ET.SubElement(e, 'system-out').text = stdout
+                ET.SubElement(e, 'system-out').text = escape_xml(stdout)
             if self._stderr_buffer is not None:
                 stderr = self._stderr_buffer.getvalue().rstrip()
-                ET.SubElement(e, 'system-err').text = stderr
+                ET.SubElement(e, 'system-err').text = escape_xml(stderr)
 
         for k, v in args.items():
             if not k or not v:
                 continue
+
             e2 = ET.SubElement(e, k)
             if hasattr(v, 'items'):
                 for k2, v2 in v.items():
                     if k2:
-                        e2.set(k2, str(v2))
+                        e2.set(k2, escape_xml(str(v2)))
                     else:
-                        e2.text = str(v2)
+                        e2.text = escape_xml(str(v2))
             else:
-                e2.text = str(v)
+                e2.text = escape_xml(str(v))
 
     @classmethod
     def __makeErrorDict(cls, err_type, err_value, err_tb):

Lib/test/libregrtest/utils.py

@@ -5,6 +5,7 @@
 import os.path
 import platform
 import random
+import re
 import shlex
 import signal
 import subprocess
@@ -712,3 +713,19 @@ def get_signal_name(exitcode):
         pass
 
     return None
+
+
+ILLEGAL_XML_CHARS_RE = re.compile(
+    '['
+    '\x00-\x1F'      # ASCII control characters
+    '\uD800-\uDFFF'  # surrogate characters
+    '\uFFFE'
+    '\uFFFF'
+    ']')
+
+def _escape_xml_replace(regs):
+    code_point = ord(regs[0])
+    return f"&#{code_point};"
+
+def escape_xml(text):
+    return ILLEGAL_XML_CHARS_RE.sub(_escape_xml_replace, text)

Lib/test/test_regrtest.py

@@ -21,6 +21,8 @@
 import tempfile
 import textwrap
 import unittest
+from xml.etree import ElementTree
+
 from test import support
 from test.support import import_helper
 from test.support import os_helper
@@ -2254,6 +2256,44 @@ def test_pass(self):
             self.check_executed_tests(output, testname, stats=1, parallel=True)
             self.assertNotIn('SPAM SPAM SPAM', output)
 
+    def test_xml(self):
+        code = textwrap.dedent(r"""
+            import unittest
+            from test import support
+
+            class VerboseTests(unittest.TestCase):
+                def test_failed(self):
+                    print("abc \x1b def")
+                    self.fail()
+        """)
+        testname = self.create_test(code=code)
+
+        # Run sequentially
+        filename = os_helper.TESTFN
+        self.addCleanup(os_helper.unlink, filename)
+
+        output = self.run_tests(testname, "--junit-xml", filename,
+                                exitcode=EXITCODE_BAD_TEST)
+        self.check_executed_tests(output, testname,
+                                  failed=testname,
+                                  stats=TestStats(1, 1, 0))
+
+        # Test generated XML
+        with open(filename, encoding="utf8") as fp:
+            content = fp.read()
+
+        testsuite = ElementTree.fromstring(content)
+        self.assertEqual(int(testsuite.get('tests')), 1)
+        self.assertEqual(int(testsuite.get('errors')), 0)
+        self.assertEqual(int(testsuite.get('failures')), 1)
+
+        testcase = testsuite[0][0]
+        self.assertEqual(testcase.get('status'), 'run')
+        self.assertEqual(testcase.get('result'), 'completed')
+        self.assertGreater(float(testcase.get('time')), 0)
+        for out in testcase.iter('system-out'):
+            self.assertEqual(out.text, "abc  def")
+
 
 class TestUtils(unittest.TestCase):
     def test_format_duration(self):
@@ -2437,6 +2477,11 @@ def id(self):
             self.assertTrue(match_test(test_chdir))
             self.assertFalse(match_test(test_copy))
 
+    def test_escape_xml(self):
+        escape_xml = utils.escape_xml
+        self.assertEqual(escape_xml('abc \x1b def'),
+                         'abc  def')
+
 
 if __name__ == '__main__':
     unittest.main()

Misc/NEWS.d/next/Tests/2024-07-01-09-04-32.gh-issue-121188.XbuTVa.rst

@@ -0,0 +1,3 @@
+When creating the JUnit XML file, regrtest now escapes characters which are
+invalid in XML such as the chr(27) control character used in ANSI escape
+sequences. Patch by Victor Stinner.