This is an automated email from the ASF dual-hosted git repository.
dpol1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git
The following commit(s) were added to refs/heads/main by this push:
new 3a49d077 #2130 MetadataTransfer: single pass over the metadata for
wildcard keys (#2133)
3a49d077 is described below
commit 3a49d077b35c48ad6ac135918d829ffd33e23ec3
Author: Gianluca Graziadei <[email protected]>
AuthorDate: Mon Sep 7 19:10:46 2026 +0200
#2130 MetadataTransfer: single pass over the metadata for wildcard keys
(#2133)
Pre-compile the keys to transfer into exact keys and wildcard prefixes
and copy the matching entries in one pass over the metadata, instead of
building an intermediate key set per wildcard for every outlink.
Micro-benchmark on a 12-key metadata with 3 wildcards: ~780 ns to
~265 ns per outlink.
The compiled form is built lazily and keyed on a snapshot of the key
set, so a subclass editing mdToTransfer / mdToPersistOnly (in an
overridden configure() or later, even without changing the size) is
always honoured. Null value arrays in a caller-supplied map are skipped
as before.
Fixes #2130.
---
.../apache/stormcrawler/util/MetadataTransfer.java | 104 ++++++++++++++++++---
.../stormcrawler/util/MetadataTransferTest.java | 101 ++++++++++++++++++++
2 files changed, 191 insertions(+), 14 deletions(-)
diff --git
a/core/src/main/java/org/apache/stormcrawler/util/MetadataTransfer.java
b/core/src/main/java/org/apache/stormcrawler/util/MetadataTransfer.java
index 38592bfe..f8033a34 100644
--- a/core/src/main/java/org/apache/stormcrawler/util/MetadataTransfer.java
+++ b/core/src/main/java/org/apache/stormcrawler/util/MetadataTransfer.java
@@ -17,7 +17,11 @@
package org.apache.stormcrawler.util;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
@@ -123,7 +127,7 @@ public class MetadataTransfer {
* the URL path.
*/
public Metadata getMetaForOutlink(String targetUrl, String sourceUrl,
Metadata parentMetadata) {
- Metadata md = filter(parentMetadata, mdToTransfer);
+ Metadata md = filter(parentMetadata, transferFilter());
// keep the path?
if (trackPath) {
@@ -150,11 +154,11 @@ public class MetadataTransfer {
* not necessarily transferred to the outlinks.
*/
public Metadata filter(Metadata metadata) {
- Metadata filteredMetadata = filter(metadata, mdToTransfer);
+ Metadata filteredMetadata = filter(metadata, transferFilter());
// add the features that are only persisted but
// not transferred like __redirTo_
- filteredMetadata.putAll(filter(metadata, mdToPersistOnly));
+ filteredMetadata.putAll(filter(metadata, persistOnlyFilter()));
return filteredMetadata;
}
@@ -163,20 +167,92 @@ public class MetadataTransfer {
* Filter the metadata based on a set of keys. If a key ends with a * then
all the keys starting
* with the prefix will be added.
*/
- private Metadata filter(Metadata metadata, Set<String> filter) {
- Metadata filteredMetadata = new Metadata();
-
- for (String key : filter) {
- if (key.endsWith("*")) {
- String prefix = key.substring(0, key.length() - 1);
- for (String k : metadata.keySet(prefix)) {
- metadata.copy(filteredMetadata, k);
+ private static Metadata filter(Metadata metadata, CompiledFilter compiled)
{
+ final Map<String, String[]> source = metadata.asMap();
+ final Map<String, String[]> target = new HashMap<>();
+
+ // exact keys: direct lookups
+ for (String key : compiled.exactKeys) {
+ final String[] values = source.get(key);
+ if (values != null && values.length > 0) {
+ target.put(key, values);
+ }
+ }
+
+ // wildcards: a single pass over the metadata for all the prefixes,
+ // without allocating an intermediate key set per prefix
+ if (compiled.prefixes.length > 0) {
+ for (Map.Entry<String, String[]> entry : source.entrySet()) {
+ final String key = entry.getKey();
+ final String[] values = entry.getValue();
+ if (values == null || values.length == 0 ||
target.containsKey(key)) {
+ continue;
+ }
+ for (String prefix : compiled.prefixes) {
+ if (key.startsWith(prefix)) {
+ target.put(key, values);
+ break;
+ }
}
- } else {
- metadata.copy(filteredMetadata, key);
}
}
- return filteredMetadata;
+ return new Metadata(target);
+ }
+
+ /**
+ * Pre-computed, normalised form of a set of keys to transfer: exact keys
and wildcard prefixes.
+ * Keeps a snapshot of the keys it was built from so that a stale instance
can be detected.
+ */
+ private static final class CompiledFilter {
+ private final Set<String> snapshot;
+ private final Set<String> exactKeys;
+ private final String[] prefixes;
+
+ private CompiledFilter(Set<String> filter) {
+ this.snapshot = new HashSet<>(filter);
+ final Set<String> exact = new HashSet<>();
+ final List<String> prefixList = new ArrayList<>();
+ for (String key : snapshot) {
+ final String normalised = key.toLowerCase(Locale.ROOT);
+ if (normalised.endsWith("*")) {
+ prefixList.add(normalised.substring(0, normalised.length()
- 1));
+ } else {
+ exact.add(normalised);
+ }
+ }
+ this.exactKeys = exact;
+ this.prefixes = prefixList.toArray(new String[0]);
+ }
+
+ /** True if this instance was built from exactly the given keys. */
+ private boolean isFor(Set<String> filter) {
+ return snapshot.equals(filter);
+ }
+ }
+
+ // Built lazily on first use and rebuilt whenever the underlying set no
longer matches the
+ // snapshot, so subclasses may keep editing mdToTransfer / mdToPersistOnly
at any time. The
+ // equality check is a handful of hash lookups with no allocation; the
compile itself only
+ // runs when the keys actually changed.
+ private CompiledFilter compiledTransfer;
+ private CompiledFilter compiledPersistOnly;
+
+ private CompiledFilter transferFilter() {
+ CompiledFilter compiled = compiledTransfer;
+ if (compiled == null || !compiled.isFor(mdToTransfer)) {
+ compiled = new CompiledFilter(mdToTransfer);
+ compiledTransfer = compiled;
+ }
+ return compiled;
+ }
+
+ private CompiledFilter persistOnlyFilter() {
+ CompiledFilter compiled = compiledPersistOnly;
+ if (compiled == null || !compiled.isFor(mdToPersistOnly)) {
+ compiled = new CompiledFilter(mdToPersistOnly);
+ compiledPersistOnly = compiled;
+ }
+ return compiled;
}
}
diff --git
a/core/src/test/java/org/apache/stormcrawler/util/MetadataTransferTest.java
b/core/src/test/java/org/apache/stormcrawler/util/MetadataTransferTest.java
index 43368e97..ac212d4e 100644
--- a/core/src/test/java/org/apache/stormcrawler/util/MetadataTransferTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/util/MetadataTransferTest.java
@@ -152,4 +152,105 @@ class MetadataTransferTest {
}
static class MyCustomTransferClass extends MetadataTransfer {}
+
+ @Test
+ void testWildcardPrefixIsCaseInsensitiveAndSelective() throws
MalformedURLException {
+ Map<String, Object> conf = new HashMap<>();
+ conf.put(MetadataTransfer.trackPathParamName, false);
+ conf.put(MetadataTransfer.trackDepthParamName, false);
+ conf.put(MetadataTransfer.metadataTransferParamName,
List.of("Cookie.*", "exact"));
+ Metadata parentMD = new Metadata();
+ parentMD.addValue("cookie.id", "42");
+ parentMD.addValue("cookies", "not a prefix match");
+ parentMD.addValue("cook", "no");
+ parentMD.addValue("exact", "yes");
+ parentMD.addValue("exactly", "no");
+ Metadata outlinkMD =
+ MetadataTransfer.getInstance(conf)
+ .getMetaForOutlink(
+ "http://www.example.com/outlink.html";,
+ "http://www.example.com";,
+ parentMD);
+ Assertions.assertEquals(Set.of("cookie.id", "exact"),
outlinkMD.keySet());
+ Assertions.assertEquals("42", outlinkMD.getFirstValue("cookie.id"));
+ }
+
+ /** Subclass that extends the transfer set after the base configuration, a
supported pattern. */
+ static class ExtendingTransferClass extends MetadataTransfer {
+ @Override
+ protected void configure(Map<String, Object> conf) {
+ super.configure(conf);
+ mdToTransfer.add("added.*");
+ }
+ }
+
+ @Test
+ void testSubclassCanExtendTransferSetInConfigure() throws
MalformedURLException {
+ Map<String, Object> conf = new HashMap<>();
+ conf.put(MetadataTransfer.trackPathParamName, false);
+ conf.put(MetadataTransfer.trackDepthParamName, false);
+ conf.put(
+ MetadataTransfer.metadataTransferClassParamName,
+ ExtendingTransferClass.class.getName());
+ conf.put(MetadataTransfer.metadataTransferParamName,
List.of("cookie.*"));
+ Metadata parentMD = new Metadata();
+ parentMD.addValue("cookie.id", "42");
+ parentMD.addValue("added.key", "yes");
+ parentMD.addValue("other", "no");
+ Metadata outlinkMD =
+ MetadataTransfer.getInstance(conf)
+ .getMetaForOutlink(
+ "http://www.example.com/outlink.html";,
+ "http://www.example.com";,
+ parentMD);
+ Assertions.assertEquals(Set.of("cookie.id", "added.key"),
outlinkMD.keySet());
+ }
+
+ @Test
+ void testSameSizeMutationOfTransferSetIsHonoured() throws
MalformedURLException {
+ Map<String, Object> conf = new HashMap<>();
+ conf.put(MetadataTransfer.trackPathParamName, false);
+ conf.put(MetadataTransfer.trackDepthParamName, false);
+ conf.put(MetadataTransfer.metadataTransferParamName,
List.of("cookie.*"));
+ MetadataTransfer mdt = MetadataTransfer.getInstance(conf);
+ Metadata parentMD = new Metadata();
+ parentMD.addValue("cookie.id", "42");
+ parentMD.addValue("other", "yes");
+ Assertions.assertEquals(
+ Set.of("cookie.id"),
+ mdt.getMetaForOutlink(
+ "http://www.example.com/outlink.html";,
+ "http://www.example.com";,
+ parentMD)
+ .keySet());
+ // same size, different content: the compiled filter must not be stale
+ mdt.mdToTransfer.remove("cookie.*");
+ mdt.mdToTransfer.add("other");
+ Assertions.assertEquals(
+ Set.of("other"),
+ mdt.getMetaForOutlink(
+ "http://www.example.com/outlink.html";,
+ "http://www.example.com";,
+ parentMD)
+ .keySet());
+ }
+
+ @Test
+ void testNullValueArrayIsSkipped() throws MalformedURLException {
+ Map<String, Object> conf = new HashMap<>();
+ conf.put(MetadataTransfer.trackPathParamName, false);
+ conf.put(MetadataTransfer.trackDepthParamName, false);
+ conf.put(MetadataTransfer.metadataTransferParamName,
List.of("cookie.*", "exact"));
+ Map<String, String[]> backing = new HashMap<>();
+ backing.put("cookie.id", new String[] {"42"});
+ backing.put("cookie.broken", null);
+ backing.put("exact", null);
+ Metadata outlinkMD =
+ MetadataTransfer.getInstance(conf)
+ .getMetaForOutlink(
+ "http://www.example.com/outlink.html";,
+ "http://www.example.com";,
+ new Metadata(backing));
+ Assertions.assertEquals(Set.of("cookie.id"), outlinkMD.keySet());
+ }
}
