This is an automated email from the ASF dual-hosted git repository. rzo1 pushed a commit to branch fix/hosturlfilter-domain-config in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
commit 47f31637416d9a70f2f68e9b540f83e0b10546f9 Author: Richard Zowalla <[email protected]> AuthorDate: Thu Aug 27 14:20:10 2026 +0200 Honour ignoreOutsideDomain when set on its own in HostURLFilter HostURLFilter.configure() read the ignoreOutsideDomain parameter but tested the ignoreOutsideHost node for null. A configuration naming only ignoreOutsideDomain was therefore discarded and the filter let every URL through, while one naming only ignoreOutsideHost failed with a NullPointerException while the topology was starting. Now each parameter is tested independently. Topologies which set only ignoreOutsideDomain start restricting the crawl to the source domain, as configured. --- .../stormcrawler/filtering/host/HostURLFilter.java | 2 +- .../stormcrawler/filtering/HostURLFilterTest.java | 37 +++++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java index 645b7b6d..d14541fc 100644 --- a/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java +++ b/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java @@ -62,7 +62,7 @@ public class HostURLFilter extends URLFilter { // always the same if (!ignoreOutsideHost) { JsonNode filterByDomainNode = filterParams.get("ignoreOutsideDomain"); - if (filterByHostNode == null) { + if (filterByDomainNode == null) { ignoreOutsideDomain = false; } else { ignoreOutsideDomain = filterByDomainNode.asBoolean(false); diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java index f70ca90b..c4457903 100644 --- a/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java +++ b/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java @@ -36,10 +36,14 @@ import org.junit.jupiter.api.Test; class HostURLFilterTest { private HostURLFilter createFilter(boolean ignoreOutsideHost, boolean ignoreOutsideDomain) { - HostURLFilter filter = new HostURLFilter(); ObjectNode filterParams = new ObjectNode(JsonNodeFactory.instance); filterParams.put("ignoreOutsideHost", Boolean.valueOf(ignoreOutsideHost)); filterParams.put("ignoreOutsideDomain", Boolean.valueOf(ignoreOutsideDomain)); + return createFilter(filterParams); + } + + private HostURLFilter createFilter(ObjectNode filterParams) { + HostURLFilter filter = new HostURLFilter(); Map<String, Object> conf = new HashMap<>(); filter.configure(conf, filterParams); return filter; @@ -108,4 +112,35 @@ class HostURLFilterTest { allAllowed.filter(sourceURL, metadata, "http://sub.sourcedomain.com/index.html";); Assertions.assertEquals("http://sub.sourcedomain.com/index.html";, filterResult); } + + /** The two modes are independent, so ignoreOutsideDomain must be honoured on its own. */ + @Test + void testWithinDomainWithoutHostParameter() throws MalformedURLException { + ObjectNode filterParams = new ObjectNode(JsonNodeFactory.instance); + filterParams.put("ignoreOutsideDomain", Boolean.TRUE); + HostURLFilter withinDomain = createFilter(filterParams); + URL sourceURL = URLUtil.toURL("http://www.sourcedomain.com/index.html";); + Metadata metadata = new Metadata(); + String filterResult = + withinDomain.filter(sourceURL, metadata, "http://sub.sourcedomain.com/index.html";); + Assertions.assertEquals("http://sub.sourcedomain.com/index.html";, filterResult); + filterResult = + withinDomain.filter(sourceURL, metadata, "http://www.anotherDomain.com/index.html";); + Assertions.assertNull(filterResult); + } + + /** + * A configuration which sets ignoreOutsideHost only must not fail on the missing domain key. + */ + @Test + void testHostParameterWithoutDomainParameter() throws MalformedURLException { + ObjectNode filterParams = new ObjectNode(JsonNodeFactory.instance); + filterParams.put("ignoreOutsideHost", Boolean.FALSE); + HostURLFilter allAllowed = createFilter(filterParams); + URL sourceURL = URLUtil.toURL("http://www.sourcedomain.com/index.html";); + Metadata metadata = new Metadata(); + String filterResult = + allAllowed.filter(sourceURL, metadata, "http://www.anotherDomain.com/index.html";); + Assertions.assertEquals("http://www.anotherDomain.com/index.html";, filterResult); + } }
