(stormcrawler) branch main updated: Honour ignoreOutsideDomain when set on its own in HostURLFilter (#2074)

[jnioche]

This is an automated email from the ASF dual-hosted git repository.

jnioche pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git


The following commit(s) were added to refs/heads/main by this push:
     new 2223fe8f Honour ignoreOutsideDomain when set on its own in 
HostURLFilter (#2074)
2223fe8f is described below

commit 2223fe8f586e5a388a9c139107d5e4f30cb9fc72
Author: Richard Zowalla <13417392+r...@users.noreply.github.com>
AuthorDate: Fri Aug 28 12:52:13 2026 +0200

    Honour ignoreOutsideDomain when set on its own in HostURLFilter (#2074)
    
    HostURLFilter.configure() read the ignoreOutsideDomain parameter but tested
    the ignoreOutsideHost node for null. A configuration naming only
    ignoreOutsideDomain was therefore discarded and the filter let every URL
    through, while one naming only ignoreOutsideHost: false failed with a
    NullPointerException while the topology was starting. ignoreOutsideHost: 
true
    was never affected, as it short-circuits before the domain parameter is 
read.
    Now each parameter is tested independently. Topologies which set only
    ignoreOutsideDomain: true start restricting the crawl to the source domain,
    as configured.
---
 .../stormcrawler/filtering/host/HostURLFilter.java |  2 +-
 .../stormcrawler/filtering/HostURLFilterTest.java  | 37 +++++++++++++++++++++-
 2 files changed, 37 insertions(+), 2 deletions(-)

diff --git 
a/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java 
b/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java
index 645b7b6d..d14541fc 100644
--- 
a/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java
+++ 
b/core/src/main/java/org/apache/stormcrawler/filtering/host/HostURLFilter.java
@@ -62,7 +62,7 @@ public class HostURLFilter extends URLFilter {
         // always the same
         if (!ignoreOutsideHost) {
             JsonNode filterByDomainNode = 
filterParams.get("ignoreOutsideDomain");
-            if (filterByHostNode == null) {
+            if (filterByDomainNode == null) {
                 ignoreOutsideDomain = false;
             } else {
                 ignoreOutsideDomain = filterByDomainNode.asBoolean(false);
diff --git 
a/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java 
b/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java
index f70ca90b..c4457903 100644
--- 
a/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java
+++ 
b/core/src/test/java/org/apache/stormcrawler/filtering/HostURLFilterTest.java
@@ -36,10 +36,14 @@ import org.junit.jupiter.api.Test;
 class HostURLFilterTest {
 
     private HostURLFilter createFilter(boolean ignoreOutsideHost, boolean 
ignoreOutsideDomain) {
-        HostURLFilter filter = new HostURLFilter();
         ObjectNode filterParams = new ObjectNode(JsonNodeFactory.instance);
         filterParams.put("ignoreOutsideHost", 
Boolean.valueOf(ignoreOutsideHost));
         filterParams.put("ignoreOutsideDomain", 
Boolean.valueOf(ignoreOutsideDomain));
+        return createFilter(filterParams);
+    }
+
+    private HostURLFilter createFilter(ObjectNode filterParams) {
+        HostURLFilter filter = new HostURLFilter();
         Map<String, Object> conf = new HashMap<>();
         filter.configure(conf, filterParams);
         return filter;
@@ -108,4 +112,35 @@ class HostURLFilterTest {
                 allAllowed.filter(sourceURL, metadata, 
"http://sub.sourcedomain.com/index.html";);
         Assertions.assertEquals("http://sub.sourcedomain.com/index.html";, 
filterResult);
     }
+
+    /** The two modes are independent, so ignoreOutsideDomain must be honoured 
on its own. */
+    @Test
+    void testWithinDomainWithoutHostParameter() throws MalformedURLException {
+        ObjectNode filterParams = new ObjectNode(JsonNodeFactory.instance);
+        filterParams.put("ignoreOutsideDomain", Boolean.TRUE);
+        HostURLFilter withinDomain = createFilter(filterParams);
+        URL sourceURL = 
URLUtil.toURL("http://www.sourcedomain.com/index.html";);
+        Metadata metadata = new Metadata();
+        String filterResult =
+                withinDomain.filter(sourceURL, metadata, 
"http://sub.sourcedomain.com/index.html";);
+        Assertions.assertEquals("http://sub.sourcedomain.com/index.html";, 
filterResult);
+        filterResult =
+                withinDomain.filter(sourceURL, metadata, 
"http://www.anotherDomain.com/index.html";);
+        Assertions.assertNull(filterResult);
+    }
+
+    /**
+     * A configuration which sets ignoreOutsideHost only must not fail on the 
missing domain key.
+     */
+    @Test
+    void testHostParameterWithoutDomainParameter() throws 
MalformedURLException {
+        ObjectNode filterParams = new ObjectNode(JsonNodeFactory.instance);
+        filterParams.put("ignoreOutsideHost", Boolean.FALSE);
+        HostURLFilter allAllowed = createFilter(filterParams);
+        URL sourceURL = 
URLUtil.toURL("http://www.sourcedomain.com/index.html";);
+        Metadata metadata = new Metadata();
+        String filterResult =
+                allAllowed.filter(sourceURL, metadata, 
"http://www.anotherDomain.com/index.html";);
+        Assertions.assertEquals("http://www.anotherDomain.com/index.html";, 
filterResult);
+    }
 }