This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 137c0d16b TIKA-4288 -- users should be able to configure the metadata
filter per parse via the ParseContext on the ParseEmitTuple with PipesServer
(#1867)
137c0d16b is described below
commit 137c0d16b74fc300330e182ec2b9074b2afa05ca
Author: Tim Allison <[email protected]>
AuthorDate: Tue Jul 30 12:26:00 2024 -0400
TIKA-4288 -- users should be able to configure the metadata filter per
parse via the ParseContext on the ParseEmitTuple with PipesServer (#1867)
---
.../java/org/apache/tika/pipes/PipesServer.java | 13 +++++++++----
.../org/apache/tika/pipes/PipesClientTest.java | 22 ++++++++++++++++++++--
2 files changed, 29 insertions(+), 6 deletions(-)
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java
b/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java
index 802545380..54d2b439c 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java
@@ -57,6 +57,7 @@ import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.metadata.filter.MetadataFilter;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.DigestingParser;
@@ -398,8 +399,12 @@ public class PipesServer implements Runnable {
private void emitParseData(FetchEmitTuple t, MetadataListAndEmbeddedBytes
parseData) {
long start = System.currentTimeMillis();
String stack = getContainerStacktrace(t, parseData.getMetadataList());
- //we need to apply this after we pull out the stacktrace
- filterMetadata(parseData.getMetadataList());
+ //we need to apply the metadata filter after we pull out the stacktrace
+ MetadataFilter filter = t.getParseContext().get(MetadataFilter.class);
+ if (filter == null) {
+ filter = tikaConfig.getMetadataFilter();
+ }
+ filterMetadata(filter, parseData.getMetadataList());
ParseContext parseContext = t.getParseContext();
FetchEmitTuple.ON_PARSE_EXCEPTION onParseException =
t.getOnParseException();
EmbeddedDocumentBytesConfig embeddedDocumentBytesConfig =
parseContext.get(EmbeddedDocumentBytesConfig.class);
@@ -432,10 +437,10 @@ public class PipesServer implements Runnable {
}
}
- private void filterMetadata(List<Metadata> metadataList) {
+ private void filterMetadata(MetadataFilter metadataFilter, List<Metadata>
metadataList) {
for (Metadata m : metadataList) {
try {
- tikaConfig.getMetadataFilter().filter(m);
+ metadataFilter.filter(m);
} catch (TikaException e) {
LOG.warn("failed to filter metadata", e);
}
diff --git a/tika-core/src/test/java/org/apache/tika/pipes/PipesClientTest.java
b/tika-core/src/test/java/org/apache/tika/pipes/PipesClientTest.java
index 01cc86c0a..13b0dc312 100644
--- a/tika-core/src/test/java/org/apache/tika/pipes/PipesClientTest.java
+++ b/tika-core/src/test/java/org/apache/tika/pipes/PipesClientTest.java
@@ -19,6 +19,7 @@ package org.apache.tika.pipes;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.List;
import javax.xml.parsers.ParserConfigurationException;
import org.junit.jupiter.api.Assertions;
@@ -28,11 +29,14 @@ import org.xml.sax.SAXException;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.filter.CompositeMetadataFilter;
+import org.apache.tika.metadata.filter.MetadataFilter;
+import org.apache.tika.metadata.filter.MockUpperCaseFilter;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.pipes.emitter.EmitKey;
import org.apache.tika.pipes.fetcher.FetchKey;
-class PipesClientTest {
+public class PipesClientTest {
String fetcherName = "fs";
String testPdfFile = "testOverlappingText.pdf";
@@ -49,7 +53,7 @@ class PipesClientTest {
}
@Test
- void process() throws IOException, InterruptedException {
+ public void testBasic() throws IOException, InterruptedException {
PipesResult pipesResult = pipesClient.process(
new FetchEmitTuple(testPdfFile, new FetchKey(fetcherName,
testPdfFile),
new EmitKey(), new Metadata(), new ParseContext(),
FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP));
@@ -58,4 +62,18 @@ class PipesClientTest {
Metadata metadata = pipesResult.getEmitData().getMetadataList().get(0);
Assertions.assertEquals("testOverlappingText.pdf",
metadata.get("resourceName"));
}
+
+ @Test
+ public void testMetadataFilter() throws IOException, InterruptedException {
+ ParseContext parseContext = new ParseContext();
+ MetadataFilter metadataFilter = new
CompositeMetadataFilter(List.of(new MockUpperCaseFilter()));
+ parseContext.set(MetadataFilter.class, metadataFilter);
+ PipesResult pipesResult = pipesClient.process(
+ new FetchEmitTuple(testPdfFile, new FetchKey(fetcherName,
testPdfFile),
+ new EmitKey(), new Metadata(), parseContext,
FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP));
+ Assertions.assertNotNull(pipesResult.getEmitData().getMetadataList());
+ Assertions.assertEquals(1,
pipesResult.getEmitData().getMetadataList().size());
+ Metadata metadata = pipesResult.getEmitData().getMetadataList().get(0);
+ Assertions.assertEquals("TESTOVERLAPPINGTEXT.PDF",
metadata.get("resourceName"));
+ }
}
