https://github.com/python/cpython/commit/9259e8ba4de8e2f534d97233b8e0f9c2537691d3
commit: 9259e8ba4de8e2f534d97233b8e0f9c2537691d3
branch: main
author: Serhiy Storchaka <[email protected]>
committer: serhiy-storchaka <[email protected]>
date: 2026-09-01T13:18:07Z
summary:
gh-83895: Accept input larger than 2 GiB in the C implementation of ElementTree
(GH-156746)
XMLParser.feed() and ElementTree.parse() raised OverflowError, because Expat
takes the length as an int. The data is now fed to Expat in chunks of 1 MiB,
as xml.parsers.expat does since bpo-17089, so the pure Python implementation
already accepted such input.
files:
A Misc/NEWS.d/next/Library/2026-08-31-23-10-00.gh-issue-83895.Jm5tR3.rst
M Lib/test/test_xml_etree.py
M Lib/test/test_xml_etree_c.py
M Modules/_elementtree.c
diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py
index fb35bb6a5f442f..fae212e7d2d369 100644
--- a/Lib/test/test_xml_etree.py
+++ b/Lib/test/test_xml_etree.py
@@ -1096,6 +1096,41 @@ def test_parse_text_source_multiple_chunks(self):
xml = "<?xml version='1.0' encoding='ISO-8859-1'?><xml>%s</xml>" % body
self.assertEqual(ET.parse(io.StringIO(xml)).getroot().text, body)
+ def test_parse_input_larger_than_chunk(self):
+ # gh-83895: the C implementation feeds Expat in chunks of 1 MiB
+ size = 3 * (1 << 20)
+ xml = '<r><a>%s</a><b/></r>' % ('x' * size)
+ for source in xml, xml.encode():
+ with self.subTest(type=type(source).__name__):
+ root = ET.fromstring(source)
+ self.assertEqual(len(root[0].text), size)
+ self.assertEqual(root[1].tag, 'b')
+
+ # gh-83895: input larger than INT_MAX is fed to Expat in chunks.
+ # memuse is 3 for the Python implementation, which joins the collected
+ # data, 2 would be enough for the C implementation.
+ @support.bigmemtest(size=support._2G + 100, memuse=3, dry_run=False)
+ def test_large_input(self, size):
+ data = b'<r>' + b'x' * size + b'</r>'
+ root = None
+ try:
+ parser = ET.XMLParser()
+ parser.feed(data)
+ data = None
+ root = parser.close()
+ self.assertEqual(len(root.text), size)
+ finally:
+ data = None
+ root = None
+
+ def test_parse_error_after_chunk_boundary(self):
+ # the reported position accounts for the preceding chunks
+ size = 2 * (1 << 20)
+ with self.assertRaises(ET.ParseError) as cm:
+ ET.fromstring('<r>%s<</r>' % ('x' * size))
+ self.assertEqual(cm.exception.position, (1, size + 4))
+
+
@support.subTests('sample,exception', [
(b'<x> \xa1</x>', UnicodeDecodeError), # crashed
(b'<x> \xa1</x', UnicodeDecodeError), # crashed
diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py
index 2a18396afb6088..deb2bd8d572581 100644
--- a/Lib/test/test_xml_etree_c.py
+++ b/Lib/test/test_xml_etree_c.py
@@ -15,16 +15,6 @@
@unittest.skipUnless(cET, 'requires _elementtree')
class MiscTests(unittest.TestCase):
- # Issue #8651.
- @support.bigmemtest(size=support._2G + 100, memuse=1, dry_run=False)
- def test_length_overflow(self, size):
- data = b'x' * size
- parser = cET.XMLParser()
- try:
- self.assertRaises(OverflowError, parser.feed, data)
- finally:
- data = None
-
def test_del_attribute(self):
element = cET.Element('tag')
diff --git
a/Misc/NEWS.d/next/Library/2026-08-31-23-10-00.gh-issue-83895.Jm5tR3.rst
b/Misc/NEWS.d/next/Library/2026-08-31-23-10-00.gh-issue-83895.Jm5tR3.rst
new file mode 100644
index 00000000000000..c6e2c77897cbc2
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-08-31-23-10-00.gh-issue-83895.Jm5tR3.rst
@@ -0,0 +1,4 @@
+:mod:`xml.etree.ElementTree` now accepts input larger than 2 GiB
+in the C implementation.
+The data is fed to Expat in chunks, as :mod:`xml.parsers.expat` already did,
+instead of raising :exc:`OverflowError`.
diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c
index a49811a338e625..fa178f53e9b3ff 100644
--- a/Modules/_elementtree.c
+++ b/Modules/_elementtree.c
@@ -3940,6 +3940,27 @@ expat_parse(elementtreestate *st, XMLParserObject *self,
const char *data,
Py_RETURN_NONE;
}
+/* Expat takes the length as an int, feed larger data in chunks. */
+#define MAX_CHUNK_SIZE (1 << 20)
+
+LOCAL(PyObject*)
+expat_parse_large(elementtreestate *st, XMLParserObject *self,
+ const char *data, Py_ssize_t data_len, int final)
+{
+ static_assert(MAX_CHUNK_SIZE <= INT_MAX,
+ "MAX_CHUNK_SIZE is larger than INT_MAX");
+ while (data_len > MAX_CHUNK_SIZE) {
+ PyObject *res = expat_parse(st, self, data, MAX_CHUNK_SIZE, 0);
+ if (res == NULL) {
+ return NULL;
+ }
+ Py_DECREF(res);
+ data += MAX_CHUNK_SIZE;
+ data_len -= MAX_CHUNK_SIZE;
+ }
+ return expat_parse(st, self, data, (int)data_len, final);
+}
+
/*[clinic input]
_elementtree.XMLParser.close
@@ -4031,26 +4052,17 @@ _elementtree_XMLParser_feed_impl(XMLParserObject *self,
PyObject *data)
const char *data_ptr = PyUnicode_AsUTF8AndSize(data, &data_len);
if (data_ptr == NULL)
return NULL;
- if (data_len > INT_MAX) {
- PyErr_SetString(PyExc_OverflowError, "size does not fit in an
int");
- return NULL;
- }
/* Explicitly set UTF-8 encoding. Return code ignored. */
(void)EXPAT(st, SetEncoding)(self->parser, "utf-8");
- return expat_parse(st, self, data_ptr, (int)data_len, 0);
+ return expat_parse_large(st, self, data_ptr, data_len, 0);
}
else {
Py_buffer view;
PyObject *res;
if (PyObject_GetBuffer(data, &view, PyBUF_SIMPLE) < 0)
return NULL;
- if (view.len > INT_MAX) {
- PyBuffer_Release(&view);
- PyErr_SetString(PyExc_OverflowError, "size does not fit in an
int");
- return NULL;
- }
- res = expat_parse(st, self, view.buf, (int)view.len, 0);
+ res = expat_parse_large(st, self, view.buf, view.len, 0);
PyBuffer_Release(&view);
return res;
}
@@ -4120,14 +4132,8 @@ _elementtree_XMLParser__parse_whole_impl(XMLParserObject
*self,
break;
}
- if (PyBytes_GET_SIZE(buffer) > INT_MAX) {
- Py_DECREF(buffer);
- Py_DECREF(reader);
- PyErr_SetString(PyExc_OverflowError, "size does not fit in an
int");
- return NULL;
- }
- res = expat_parse(
- st, self, PyBytes_AS_STRING(buffer), (int)PyBytes_GET_SIZE(buffer),
+ res = expat_parse_large(
+ st, self, PyBytes_AS_STRING(buffer), PyBytes_GET_SIZE(buffer),
0);
first = 0;
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
