This is an automated email from the ASF dual-hosted git repository.
lewismc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new 773b4321c NUTCH-3206 XML parsers should not be vulnerable to XXE
attacks (#954)
773b4321c is described below
commit 773b4321ca25921085ca7bee78499a50b62f34ea
Author: Lewis John McGibbney <[email protected]>
AuthorDate: Sun Aug 16 13:46:11 2026 -0600
NUTCH-3206 XML parsers should not be vulnerable to XXE attacks (#954)
---
src/java/org/apache/nutch/exchange/Exchanges.java | 3 +-
.../org/apache/nutch/indexer/IndexWriters.java | 3 +-
.../org/apache/nutch/parse/ParsePluginsReader.java | 3 +-
.../apache/nutch/plugin/PluginManifestParser.java | 3 +-
src/java/org/apache/nutch/tools/DmozParser.java | 5 +-
src/java/org/apache/nutch/util/DomUtil.java | 4 +-
src/java/org/apache/nutch/util/XmlUtil.java | 135 +++++++++++++++++++++
.../org/creativecommons/nutch/CCParseFilter.java | 13 +-
.../org/apache/nutch/protocol/httpclient/Http.java | 4 +-
.../urlnormalizer/regex/RegexURLNormalizer.java | 5 +-
src/test/org/apache/nutch/util/TestXmlUtil.java | 90 ++++++++++++++
11 files changed, 251 insertions(+), 17 deletions(-)
diff --git a/src/java/org/apache/nutch/exchange/Exchanges.java
b/src/java/org/apache/nutch/exchange/Exchanges.java
index 1e0518b6b..37b2406bf 100644
--- a/src/java/org/apache/nutch/exchange/Exchanges.java
+++ b/src/java/org/apache/nutch/exchange/Exchanges.java
@@ -22,6 +22,7 @@ import org.apache.nutch.plugin.Extension;
import org.apache.nutch.plugin.ExtensionPoint;
import org.apache.nutch.plugin.PluginRepository;
import org.apache.nutch.plugin.PluginRuntimeException;
+import org.apache.nutch.util.XmlUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Element;
@@ -104,7 +105,7 @@ public class Exchanges {
final List<ExchangeConfig> configList = new LinkedList<>();
try {
- DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+ DocumentBuilderFactory factory =
XmlUtil.newSecureDocumentBuilderFactory();
DocumentBuilder builder = factory.newDocumentBuilder();
Element rootElement = builder.parse(inputSource).getDocumentElement();
NodeList exchangeList = rootElement.getElementsByTagName("exchange");
diff --git a/src/java/org/apache/nutch/indexer/IndexWriters.java
b/src/java/org/apache/nutch/indexer/IndexWriters.java
index f8ae8ee86..4b809d1e4 100644
--- a/src/java/org/apache/nutch/indexer/IndexWriters.java
+++ b/src/java/org/apache/nutch/indexer/IndexWriters.java
@@ -26,6 +26,7 @@ import org.apache.nutch.plugin.ExtensionPoint;
import org.apache.nutch.plugin.PluginRepository;
import org.apache.nutch.plugin.PluginRuntimeException;
import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.XmlUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
@@ -121,7 +122,7 @@ public class IndexWriters {
InputSource inputSource = new InputSource(ssInputStream);
try {
- DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+ DocumentBuilderFactory factory =
XmlUtil.newSecureDocumentBuilderFactory();
DocumentBuilder builder = factory.newDocumentBuilder();
Document document = builder.parse(inputSource);
Element rootElement = document.getDocumentElement();
diff --git a/src/java/org/apache/nutch/parse/ParsePluginsReader.java
b/src/java/org/apache/nutch/parse/ParsePluginsReader.java
index 978e3c097..8b84d4479 100644
--- a/src/java/org/apache/nutch/parse/ParsePluginsReader.java
+++ b/src/java/org/apache/nutch/parse/ParsePluginsReader.java
@@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.XmlUtil;
/**
* A reader to load the information stored in the
@@ -102,7 +103,7 @@ class ParsePluginsReader {
inputSource = new InputSource(ppInputStream);
try {
- factory = DocumentBuilderFactory.newInstance();
+ factory = XmlUtil.newSecureDocumentBuilderFactory();
parser = factory.newDocumentBuilder();
document = parser.parse(inputSource);
} catch (Exception e) {
diff --git a/src/java/org/apache/nutch/plugin/PluginManifestParser.java
b/src/java/org/apache/nutch/plugin/PluginManifestParser.java
index 95208fa43..dca202910 100644
--- a/src/java/org/apache/nutch/plugin/PluginManifestParser.java
+++ b/src/java/org/apache/nutch/plugin/PluginManifestParser.java
@@ -31,6 +31,7 @@ import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.util.XmlUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
@@ -158,7 +159,7 @@ public class PluginManifestParser {
*/
private Document parseXML(URL url)
throws SAXException, IOException, ParserConfigurationException {
- DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+ DocumentBuilderFactory factory = XmlUtil.newSecureDocumentBuilderFactory();
DocumentBuilder builder = factory.newDocumentBuilder();
return builder.parse(url.openStream());
}
diff --git a/src/java/org/apache/nutch/tools/DmozParser.java
b/src/java/org/apache/nutch/tools/DmozParser.java
index 4924e3b1b..5de457fb3 100644
--- a/src/java/org/apache/nutch/tools/DmozParser.java
+++ b/src/java/org/apache/nutch/tools/DmozParser.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.MD5Hash;
import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.XmlUtil;
import org.apache.xerces.util.XMLChar;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -296,9 +297,7 @@ public class DmozParser {
boolean includeAdult, int skew, Pattern topicPattern)
throws IOException, SAXException, ParserConfigurationException {
- SAXParserFactory parserFactory = SAXParserFactory.newInstance();
-
parserFactory.setFeature("http://xml.org/sax/features/external-general-entities";,
false);
-
parserFactory.setFeature("http://apache.org/xml/features/disallow-doctype-decl";,
true);
+ SAXParserFactory parserFactory = XmlUtil.newSecureSAXParserFactory();
SAXParser parser = parserFactory.newSAXParser();
XMLReader reader = parser.getXMLReader();
reader.setFeature("http://xml.org/sax/features/external-general-entities";,
false);
diff --git a/src/java/org/apache/nutch/util/DomUtil.java
b/src/java/org/apache/nutch/util/DomUtil.java
index 50cc43c98..c31c2a1f0 100644
--- a/src/java/org/apache/nutch/util/DomUtil.java
+++ b/src/java/org/apache/nutch/util/DomUtil.java
@@ -60,6 +60,7 @@ public class DomUtil {
InputSource input;
try {
+ XmlUtil.configureSecure(parser);
input = new InputSource(is);
input.setEncoding("UTF-8");
parser.parse(input);
@@ -87,9 +88,10 @@ public class DomUtil {
public static void saveDom(OutputStream os, Element e) {
DOMSource source = new DOMSource(e);
- TransformerFactory transFactory = TransformerFactory.newInstance();
+ TransformerFactory transFactory;
Transformer transformer;
try {
+ transFactory = XmlUtil.newSecureTransformerFactory();
transformer = transFactory.newTransformer();
transformer.setOutputProperty("indent", "yes");
transformer.setOutputProperty(OutputKeys.ENCODING,
diff --git a/src/java/org/apache/nutch/util/XmlUtil.java
b/src/java/org/apache/nutch/util/XmlUtil.java
new file mode 100644
index 000000000..a8ce9b64f
--- /dev/null
+++ b/src/java/org/apache/nutch/util/XmlUtil.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.util;
+
+import javax.xml.XMLConstants;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParserFactory;
+import javax.xml.transform.TransformerConfigurationException;
+import javax.xml.transform.TransformerFactory;
+
+import org.apache.xerces.parsers.DOMParser;
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXNotRecognizedException;
+import org.xml.sax.SAXNotSupportedException;
+
+/**
+ * Helpers that return XML parser/transformer factories configured to reject
+ * XXE (external entity) attacks. Prefer these over bare
+ * {@link DocumentBuilderFactory#newInstance()} /
+ * {@link SAXParserFactory#newInstance()} /
+ * {@link TransformerFactory#newInstance()}.
+ */
+public final class XmlUtil {
+
+ private static final String DISALLOW_DOCTYPE_DECL =
+ "http://apache.org/xml/features/disallow-doctype-decl";;
+ private static final String EXTERNAL_GENERAL_ENTITIES =
+ "http://xml.org/sax/features/external-general-entities";;
+ private static final String EXTERNAL_PARAMETER_ENTITIES =
+ "http://xml.org/sax/features/external-parameter-entities";;
+ private static final String LOAD_EXTERNAL_DTD =
+ "http://apache.org/xml/features/nonvalidating/load-external-dtd";;
+
+ private XmlUtil() {
+ }
+
+ /**
+ * @return a {@link DocumentBuilderFactory} hardened against XXE
+ * @throws ParserConfigurationException if a required secure feature is
+ * unsupported
+ */
+ public static DocumentBuilderFactory newSecureDocumentBuilderFactory()
+ throws ParserConfigurationException {
+ return newSecureDocumentBuilderFactory(false);
+ }
+
+ /**
+ * @param namespaceAware whether the factory should be namespace-aware
+ * @return a {@link DocumentBuilderFactory} hardened against XXE
+ * @throws ParserConfigurationException if a required secure feature is
+ * unsupported
+ */
+ public static DocumentBuilderFactory newSecureDocumentBuilderFactory(
+ boolean namespaceAware) throws ParserConfigurationException {
+ DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+ factory.setNamespaceAware(namespaceAware);
+ factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+ factory.setFeature(DISALLOW_DOCTYPE_DECL, true);
+ factory.setFeature(EXTERNAL_GENERAL_ENTITIES, false);
+ factory.setFeature(EXTERNAL_PARAMETER_ENTITIES, false);
+ factory.setFeature(LOAD_EXTERNAL_DTD, false);
+ factory.setXIncludeAware(false);
+ // Sonar java:S2755 "Going the extra mile" — does not alone block XXE
+ factory.setExpandEntityReferences(false);
+ return factory;
+ }
+
+ /**
+ * @return a {@link SAXParserFactory} hardened against XXE
+ * @throws ParserConfigurationException if a required secure feature is
+ * unsupported
+ * @throws SAXNotRecognizedException if a feature name is not recognized
+ * @throws SAXNotSupportedException if a feature value is not supported
+ */
+ public static SAXParserFactory newSecureSAXParserFactory()
+ throws ParserConfigurationException, SAXNotRecognizedException,
+ SAXNotSupportedException {
+ SAXParserFactory factory = SAXParserFactory.newInstance();
+ factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+ factory.setFeature(DISALLOW_DOCTYPE_DECL, true);
+ factory.setFeature(EXTERNAL_GENERAL_ENTITIES, false);
+ factory.setFeature(EXTERNAL_PARAMETER_ENTITIES, false);
+ factory.setXIncludeAware(false);
+ return factory;
+ }
+
+ /**
+ * @return a {@link TransformerFactory} that cannot load external DTDs or
+ * stylesheets
+ * @throws TransformerConfigurationException if secure attributes cannot be
+ * applied
+ */
+ public static TransformerFactory newSecureTransformerFactory()
+ throws TransformerConfigurationException {
+ TransformerFactory factory = TransformerFactory.newInstance();
+ factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+ factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+ factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_STYLESHEET, "");
+ return factory;
+ }
+
+ /**
+ * Apply XXE-hardening features to an Xerces {@link DOMParser}.
+ *
+ * @param parser parser to configure
+ * @throws SAXException if a secure feature cannot be set
+ */
+ public static void configureSecure(DOMParser parser) throws SAXException {
+ parser.setFeature(DISALLOW_DOCTYPE_DECL, true);
+ parser.setFeature(EXTERNAL_GENERAL_ENTITIES, false);
+ parser.setFeature(EXTERNAL_PARAMETER_ENTITIES, false);
+ parser.setFeature(LOAD_EXTERNAL_DTD, false);
+ try {
+ parser.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+ } catch (SAXNotRecognizedException | SAXNotSupportedException e) {
+ // Xerces' DOMParser does not expose the JAXP secure-processing feature.
+ // DTDs and external entities are already disabled above.
+ }
+ }
+}
diff --git
a/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java
b/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java
index 030c6bf66..ee4e22ec2 100644
---
a/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java
+++
b/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java
@@ -26,6 +26,7 @@ import org.apache.nutch.parse.ParseException;
import org.apache.nutch.parse.ParseResult;
import org.apache.nutch.parse.ParseStatus;
import org.apache.nutch.parse.ParseText;
+import org.apache.nutch.util.XmlUtil;
import org.apache.hadoop.conf.Configuration;
import org.slf4j.Logger;
@@ -45,6 +46,7 @@ import java.util.HashMap;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
import org.xml.sax.InputSource;
@@ -167,11 +169,14 @@ public class CCParseFilter implements HtmlParseFilter {
}
}
- /** Configure a namespace aware XML parser. */
- private static final DocumentBuilderFactory FACTORY =
DocumentBuilderFactory
- .newInstance();
+ /** Configure a namespace aware XML parser hardened against XXE. */
+ private static final DocumentBuilderFactory FACTORY;
static {
- FACTORY.setNamespaceAware(true);
+ try {
+ FACTORY = XmlUtil.newSecureDocumentBuilderFactory(true);
+ } catch (ParserConfigurationException e) {
+ throw new ExceptionInInitializerError(e);
+ }
}
/** Creative Commons' namespace URI. */
diff --git
a/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
b/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
index ebeb652dd..6d66b9b9d 100644
---
a/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
+++
b/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
@@ -26,7 +26,6 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
-import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import org.xml.sax.SAXException;
@@ -55,6 +54,7 @@ import org.apache.nutch.protocol.ProtocolException;
import org.apache.nutch.protocol.http.api.HttpBase;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.XmlUtil;
/**
* <p>
@@ -275,7 +275,7 @@ public class Http extends HttpBase {
InputStream is = conf.getConfResourceAsInputStream(authFile);
if (is != null) {
- Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder()
+ Document doc =
XmlUtil.newSecureDocumentBuilderFactory().newDocumentBuilder()
.parse(is);
Element rootElement = doc.getDocumentElement();
diff --git
a/src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java
b/src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java
index 2aeb722b2..2714e3e19 100644
---
a/src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java
+++
b/src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java
@@ -32,13 +32,12 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
-import javax.xml.parsers.DocumentBuilderFactory;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.nutch.net.URLNormalizer;
import org.apache.nutch.net.URLNormalizers;
import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.XmlUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
@@ -224,7 +223,7 @@ public class RegexURLNormalizer extends Configured
implements URLNormalizer {
try {
// borrowed heavily from code in Configuration.java
- Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder()
+ Document doc =
XmlUtil.newSecureDocumentBuilderFactory().newDocumentBuilder()
.parse(new InputSource(reader));
Element root = doc.getDocumentElement();
if (!"regex-normalize".equals(root.getTagName())) {
diff --git a/src/test/org/apache/nutch/util/TestXmlUtil.java
b/src/test/org/apache/nutch/util/TestXmlUtil.java
new file mode 100644
index 000000000..d5c9d7dae
--- /dev/null
+++ b/src/test/org/apache/nutch/util/TestXmlUtil.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.util;
+
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import org.junit.jupiter.api.Test;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.xml.sax.SAXException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * Unit tests for {@link XmlUtil} XXE hardening.
+ */
+public class TestXmlUtil {
+
+ @Test
+ public void testSecureFactoryParsesSimpleDocument() throws Exception {
+ DocumentBuilderFactory factory = XmlUtil.newSecureDocumentBuilderFactory();
+ DocumentBuilder builder = factory.newDocumentBuilder();
+ Document doc = builder.parse(new ByteArrayInputStream(
+ "<root attr=\"ok\"><child/></root>".getBytes(StandardCharsets.UTF_8)));
+ assertEquals("root", doc.getDocumentElement().getTagName());
+ assertEquals("ok", doc.getDocumentElement().getAttribute("attr"));
+ }
+
+ @Test
+ public void testSecureFactoryRejectsDoctype() throws Exception {
+ DocumentBuilderFactory factory = XmlUtil.newSecureDocumentBuilderFactory();
+ DocumentBuilder builder = factory.newDocumentBuilder();
+ String xxe = "<?xml version=\"1.0\"?>"
+ + "<!DOCTYPE foo [<!ENTITY xxe SYSTEM \"file:///etc/passwd\">]>"
+ + "<foo>&xxe;</foo>";
+ assertThrows(SAXException.class, () -> builder
+ .parse(new
ByteArrayInputStream(xxe.getBytes(StandardCharsets.UTF_8))));
+ }
+
+ @Test
+ public void testDomUtilRejectsDoctype() {
+ String xxe = "<?xml version=\"1.0\"?>"
+ + "<!DOCTYPE foo [<!ENTITY xxe SYSTEM \"file:///etc/passwd\">]>"
+ + "<foo>&xxe;</foo>";
+ Element element = DomUtil.getDom(
+ new ByteArrayInputStream(xxe.getBytes(StandardCharsets.UTF_8)));
+ // DomUtil swallows parse errors and returns null
+ assertEquals(null, element);
+ }
+
+ @Test
+ public void testDomUtilParsesDocumentWithLeadingComment() {
+ String xml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+ + "<!-- just a comment -->"
+ + "<subcollections><subcollection><id>nutch</id></subcollection>"
+ + "</subcollections>";
+ Element element = DomUtil.getDom(
+ new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)));
+ assertNotNull(element);
+ assertEquals("subcollections", element.getTagName());
+ assertEquals(1, element.getElementsByTagName("subcollection").getLength());
+ }
+
+ @Test
+ public void testNamespaceAwareFactory() throws Exception {
+ DocumentBuilderFactory factory = XmlUtil
+ .newSecureDocumentBuilderFactory(true);
+ assertNotNull(factory.newDocumentBuilder());
+ }
+}
