https://github.com/python/cpython/commit/af8c3d7a26d605099f5b3406a8d33ecddb77e8fb
commit: af8c3d7a26d605099f5b3406a8d33ecddb77e8fb
branch: main
author: Victor Stinner <vstin...@python.org>
committer: vstinner <vstin...@python.org>
date: 2024-07-01T08:30:33Z
summary:
gh-121188: Sanitize invalid XML characters in regrtest (#121195)
When creating the JUnit XML file, regrtest now escapes characters
which are invalid in XML, such as the chr(27) control character used
in ANSI escape sequences.
files:
A Misc/NEWS.d/next/Tests/2024-07-01-09-04-32.gh-issue-121188.XbuTVa.rst
M Lib/test/libregrtest/testresult.py
M Lib/test/libregrtest/utils.py
M Lib/test/test_regrtest.py
diff --git a/Lib/test/libregrtest/testresult.py
b/Lib/test/libregrtest/testresult.py
index de23fdd59ded95..1820f354572521 100644
--- a/Lib/test/libregrtest/testresult.py
+++ b/Lib/test/libregrtest/testresult.py
@@ -9,6 +9,7 @@
import traceback
import unittest
from test import support
+from test.libregrtest.utils import sanitize_xml
class RegressionTestResult(unittest.TextTestResult):
USE_XML = False
@@ -65,23 +66,24 @@ def _add_result(self, test, capture=False, **args):
if capture:
if self._stdout_buffer is not None:
stdout = self._stdout_buffer.getvalue().rstrip()
- ET.SubElement(e, 'system-out').text = stdout
+ ET.SubElement(e, 'system-out').text = sanitize_xml(stdout)
if self._stderr_buffer is not None:
stderr = self._stderr_buffer.getvalue().rstrip()
- ET.SubElement(e, 'system-err').text = stderr
+ ET.SubElement(e, 'system-err').text = sanitize_xml(stderr)
for k, v in args.items():
if not k or not v:
continue
+
e2 = ET.SubElement(e, k)
if hasattr(v, 'items'):
for k2, v2 in v.items():
if k2:
- e2.set(k2, str(v2))
+ e2.set(k2, sanitize_xml(str(v2)))
else:
- e2.text = str(v2)
+ e2.text = sanitize_xml(str(v2))
else:
- e2.text = str(v)
+ e2.text = sanitize_xml(str(v))
@classmethod
def __makeErrorDict(cls, err_type, err_value, err_tb):
diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py
index 8253d330b95b81..0167742d388a2c 100644
--- a/Lib/test/libregrtest/utils.py
+++ b/Lib/test/libregrtest/utils.py
@@ -5,6 +5,7 @@
import os.path
import platform
import random
+import re
import shlex
import signal
import subprocess
@@ -712,3 +713,24 @@ def get_signal_name(exitcode):
pass
return None
+
+
+ILLEGAL_XML_CHARS_RE = re.compile(
+ '['
+ # Control characters; newline (\x0A and \x0D) and TAB (\x09) are legal
+ '\x00-\x08\x0B\x0C\x0E-\x1F'
+ # Surrogate characters
+ '\uD800-\uDFFF'
+ # Special Unicode characters
+ '\uFFFE'
+ '\uFFFF'
+ # Match multiple sequential invalid characters for better effiency
+ ']+')
+
+def _sanitize_xml_replace(regs):
+ text = regs[0]
+ return ''.join(f'\\x{ord(ch):02x}' if ch <= '\xff' else ascii(ch)[1:-1]
+ for ch in text)
+
+def sanitize_xml(text):
+ return ILLEGAL_XML_CHARS_RE.sub(_sanitize_xml_replace, text)
diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
index 0a15170a99e757..44fd11bfdc3fcb 100644
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -21,6 +21,8 @@
import tempfile
import textwrap
import unittest
+from xml.etree import ElementTree
+
from test import support
from test.support import import_helper
from test.support import os_helper
@@ -2254,6 +2256,44 @@ def test_pass(self):
self.check_executed_tests(output, testname, stats=1, parallel=True)
self.assertNotIn('SPAM SPAM SPAM', output)
+ def test_xml(self):
+ code = textwrap.dedent(r"""
+ import unittest
+ from test import support
+
+ class VerboseTests(unittest.TestCase):
+ def test_failed(self):
+ print("abc \x1b def")
+ self.fail()
+ """)
+ testname = self.create_test(code=code)
+
+ # Run sequentially
+ filename = os_helper.TESTFN
+ self.addCleanup(os_helper.unlink, filename)
+
+ output = self.run_tests(testname, "--junit-xml", filename,
+ exitcode=EXITCODE_BAD_TEST)
+ self.check_executed_tests(output, testname,
+ failed=testname,
+ stats=TestStats(1, 1, 0))
+
+ # Test generated XML
+ with open(filename, encoding="utf8") as fp:
+ content = fp.read()
+
+ testsuite = ElementTree.fromstring(content)
+ self.assertEqual(int(testsuite.get('tests')), 1)
+ self.assertEqual(int(testsuite.get('errors')), 0)
+ self.assertEqual(int(testsuite.get('failures')), 1)
+
+ testcase = testsuite[0][0]
+ self.assertEqual(testcase.get('status'), 'run')
+ self.assertEqual(testcase.get('result'), 'completed')
+ self.assertGreater(float(testcase.get('time')), 0)
+ for out in testcase.iter('system-out'):
+ self.assertEqual(out.text, r"abc \x1b def")
+
class TestUtils(unittest.TestCase):
def test_format_duration(self):
@@ -2437,6 +2477,25 @@ def id(self):
self.assertTrue(match_test(test_chdir))
self.assertFalse(match_test(test_copy))
+ def test_sanitize_xml(self):
+ sanitize_xml = utils.sanitize_xml
+
+ # escape invalid XML characters
+ self.assertEqual(sanitize_xml('abc \x1b\x1f def'),
+ r'abc \x1b\x1f def')
+ self.assertEqual(sanitize_xml('nul:\x00, bell:\x07'),
+ r'nul:\x00, bell:\x07')
+ self.assertEqual(sanitize_xml('surrogate:\uDC80'),
+ r'surrogate:\udc80')
+ self.assertEqual(sanitize_xml('illegal \uFFFE and \uFFFF'),
+ r'illegal \ufffe and \uffff')
+
+ # no escape for valid XML characters
+ self.assertEqual(sanitize_xml('a\n\tb'),
+ 'a\n\tb')
+ self.assertEqual(sanitize_xml('valid t\xe9xt \u20ac'),
+ 'valid t\xe9xt \u20ac')
+
if __name__ == '__main__':
unittest.main()
diff --git
a/Misc/NEWS.d/next/Tests/2024-07-01-09-04-32.gh-issue-121188.XbuTVa.rst
b/Misc/NEWS.d/next/Tests/2024-07-01-09-04-32.gh-issue-121188.XbuTVa.rst
new file mode 100644
index 00000000000000..c92002d8fe3cd2
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2024-07-01-09-04-32.gh-issue-121188.XbuTVa.rst
@@ -0,0 +1,3 @@
+When creating the JUnit XML file, regrtest now escapes characters which are
+invalid in XML, such as the chr(27) control character used in ANSI escape
+sequences. Patch by Victor Stinner.
_______________________________________________
Python-checkins mailing list -- python-checkins@python.org
To unsubscribe send an email to python-checkins-le...@python.org
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: arch...@mail-archive.com
