This is an automated email from the ASF dual-hosted git repository. tallison pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/tika.git
commit ccc7bd841e097c3aa6d0c7c8494ddc5fa7596619 Author: tallison <[email protected]> AuthorDate: Fri Apr 8 14:33:27 2022 -0400 TIKA-3711 -- allow configuration of EmbeddedDocumentExtractors via tika-config.xml --- .../EmbeddedDocumentExtractorFactory.java | 28 +++++++++++++++ .../tika/extractor/EmbeddedDocumentUtil.java | 26 +++++++------- .../ParsingEmbeddedDocumentExtractor.java | 8 ++++- .../ParsingEmbeddedDocumentExtractorFactory.java | 40 ++++++++++++++++++++++ .../org/apache/tika/parser/AutoDetectParser.java | 28 +++++++++------ .../apache/tika/parser/AutoDetectParserConfig.java | 14 ++++++++ .../parser/microsoft/ooxml/OOXMLParserTest.java | 26 ++++++++++++++ .../resources/configs/tika-config-no-names.xml | 33 ++++++++++++++++++ .../resources/configs/tika-config-with-names.xml | 33 ++++++++++++++++++ 9 files changed, 211 insertions(+), 25 deletions(-) diff --git a/tika-core/src/main/java/org/apache/tika/extractor/EmbeddedDocumentExtractorFactory.java b/tika-core/src/main/java/org/apache/tika/extractor/EmbeddedDocumentExtractorFactory.java new file mode 100644 index 000000000..4a55052aa --- /dev/null +++ b/tika-core/src/main/java/org/apache/tika/extractor/EmbeddedDocumentExtractorFactory.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tika.extractor; + +import java.io.Serializable; + +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; + +public interface EmbeddedDocumentExtractorFactory extends Serializable { + + EmbeddedDocumentExtractor newInstance(Metadata metadata, ParseContext parseContext); +} diff --git a/tika-core/src/main/java/org/apache/tika/extractor/EmbeddedDocumentUtil.java b/tika-core/src/main/java/org/apache/tika/extractor/EmbeddedDocumentUtil.java index dfb38dbe8..5854aba28 100644 --- a/tika-core/src/main/java/org/apache/tika/extractor/EmbeddedDocumentUtil.java +++ b/tika-core/src/main/java/org/apache/tika/extractor/EmbeddedDocumentUtil.java @@ -78,21 +78,21 @@ public class EmbeddedDocumentUtil implements Serializable { */ public static EmbeddedDocumentExtractor getEmbeddedDocumentExtractor(ParseContext context) { EmbeddedDocumentExtractor extractor = context.get(EmbeddedDocumentExtractor.class); - if (extractor == null) { - //ensure that an AutoDetectParser is - //available for parsing embedded docs TIKA-2096 - Parser embeddedParser = context.get(Parser.class); - if (embeddedParser == null) { - TikaConfig tikaConfig = context.get(TikaConfig.class); - if (tikaConfig == null) { - context.set(Parser.class, new AutoDetectParser()); - } else { - context.set(Parser.class, new AutoDetectParser(tikaConfig)); - } + if (extractor != null) { + return extractor; + } + //ensure that an AutoDetectParser is + //available for parsing embedded docs TIKA-2096 + Parser embeddedParser = context.get(Parser.class); + if (embeddedParser == null) { + TikaConfig tikaConfig = context.get(TikaConfig.class); + if (tikaConfig == null) { + context.set(Parser.class, new AutoDetectParser()); + } else { + context.set(Parser.class, new AutoDetectParser(tikaConfig)); } - extractor = new ParsingEmbeddedDocumentExtractor(context); } - return extractor; + return new ParsingEmbeddedDocumentExtractor(context); } /** diff --git a/tika-core/src/main/java/org/apache/tika/extractor/ParsingEmbeddedDocumentExtractor.java b/tika-core/src/main/java/org/apache/tika/extractor/ParsingEmbeddedDocumentExtractor.java index 5a903d4de..f09963f9a 100644 --- a/tika-core/src/main/java/org/apache/tika/extractor/ParsingEmbeddedDocumentExtractor.java +++ b/tika-core/src/main/java/org/apache/tika/extractor/ParsingEmbeddedDocumentExtractor.java @@ -53,6 +53,8 @@ public class ParsingEmbeddedDocumentExtractor implements EmbeddedDocumentExtract private static final Parser DELEGATING_PARSER = new DelegatingParser(); + private boolean writeFileNameToContent = true; + private final ParseContext context; public ParsingEmbeddedDocumentExtractor(ParseContext context) { @@ -86,7 +88,7 @@ public class ParsingEmbeddedDocumentExtractor implements EmbeddedDocumentExtract } String name = metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY); - if (name != null && name.length() > 0 && outputHtml) { + if (writeFileNameToContent && name != null && name.length() > 0 && outputHtml) { handler.startElement(XHTML, "h1", "h1", new AttributesImpl()); char[] chars = name.toCharArray(); handler.characters(chars, 0, chars.length); @@ -125,4 +127,8 @@ public class ParsingEmbeddedDocumentExtractor implements EmbeddedDocumentExtract public Parser getDelegatingParser() { return DELEGATING_PARSER; } + + public void setWriteFileNameToContent(boolean writeFileNameToContent) { + this.writeFileNameToContent = writeFileNameToContent; + } } diff --git a/tika-core/src/main/java/org/apache/tika/extractor/ParsingEmbeddedDocumentExtractorFactory.java b/tika-core/src/main/java/org/apache/tika/extractor/ParsingEmbeddedDocumentExtractorFactory.java new file mode 100644 index 000000000..9136228c4 --- /dev/null +++ b/tika-core/src/main/java/org/apache/tika/extractor/ParsingEmbeddedDocumentExtractorFactory.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.extractor; + +import org.apache.tika.config.Field; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; + +public class ParsingEmbeddedDocumentExtractorFactory + implements EmbeddedDocumentExtractorFactory { + + private boolean writeFileNameToContent = true; + + @Field + public void setWriteFileNameToContent(boolean writeFileNameToContent) { + this.writeFileNameToContent = writeFileNameToContent; + } + + @Override + public EmbeddedDocumentExtractor newInstance(Metadata metadata, ParseContext parseContext) { + ParsingEmbeddedDocumentExtractor ex = + new ParsingEmbeddedDocumentExtractor(parseContext); + ex.setWriteFileNameToContent(writeFileNameToContent); + return ex; + } +} diff --git a/tika-core/src/main/java/org/apache/tika/parser/AutoDetectParser.java b/tika-core/src/main/java/org/apache/tika/parser/AutoDetectParser.java index 7650b23f7..f30809e6d 100644 --- a/tika-core/src/main/java/org/apache/tika/parser/AutoDetectParser.java +++ b/tika-core/src/main/java/org/apache/tika/parser/AutoDetectParser.java @@ -28,7 +28,6 @@ import org.apache.tika.detect.Detector; import org.apache.tika.exception.TikaException; import org.apache.tika.exception.ZeroByteFileException; import org.apache.tika.extractor.EmbeddedDocumentExtractor; -import org.apache.tika.extractor.ParsingEmbeddedDocumentExtractor; import org.apache.tika.io.TemporaryResources; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; @@ -172,16 +171,7 @@ public class AutoDetectParser extends CompositeParser { handler != null ? createSecureContentHandler(handler, tis, autoDetectParserConfig) : null; - //pass self to handle embedded documents if - //the caller hasn't specified one. - if (context.get(EmbeddedDocumentExtractor.class) == null) { - Parser p = context.get(Parser.class); - if (p == null) { - context.set(Parser.class, this); - } - context.set(EmbeddedDocumentExtractor.class, - new ParsingEmbeddedDocumentExtractor(context)); - } + initializeEmbeddedDocumentExtractor(metadata, context); try { // Parse the document @@ -196,6 +186,22 @@ public class AutoDetectParser extends CompositeParser { } } + private void initializeEmbeddedDocumentExtractor(Metadata metadata, ParseContext context) { + if (context.get(EmbeddedDocumentExtractor.class) != null) { + return; + } + //pass self to handle embedded documents if + //the caller hasn't specified one. + Parser p = context.get(Parser.class); + if (p == null) { + context.set(Parser.class, this); + } + EmbeddedDocumentExtractor edx = + autoDetectParserConfig.getEmbeddedDocumentExtractorFactory() + .newInstance(metadata, context); + context.set(EmbeddedDocumentExtractor.class, edx); + } + public void parse(InputStream stream, ContentHandler handler, Metadata metadata) throws IOException, SAXException, TikaException { ParseContext context = new ParseContext(); diff --git a/tika-core/src/main/java/org/apache/tika/parser/AutoDetectParserConfig.java b/tika-core/src/main/java/org/apache/tika/parser/AutoDetectParserConfig.java index f1447f99c..d4702a512 100644 --- a/tika-core/src/main/java/org/apache/tika/parser/AutoDetectParserConfig.java +++ b/tika-core/src/main/java/org/apache/tika/parser/AutoDetectParserConfig.java @@ -23,6 +23,8 @@ import org.w3c.dom.Element; import org.apache.tika.config.ConfigBase; import org.apache.tika.exception.TikaConfigException; +import org.apache.tika.extractor.EmbeddedDocumentExtractorFactory; +import org.apache.tika.extractor.ParsingEmbeddedDocumentExtractorFactory; import org.apache.tika.metadata.writefilter.MetadataWriteFilterFactory; /** @@ -72,6 +74,9 @@ public class AutoDetectParserConfig extends ConfigBase implements Serializable { private MetadataWriteFilterFactory metadataWriteFilterFactory = null; + private EmbeddedDocumentExtractorFactory embeddedDocumentExtractorFactory = + new ParsingEmbeddedDocumentExtractorFactory(); + /** * Creates a SecureContentHandlerConfig using the passed in parameters. * @@ -142,5 +147,14 @@ public class AutoDetectParserConfig extends ConfigBase implements Serializable { public void setMetadataWriteFilterFactory(MetadataWriteFilterFactory metadataWriteFilterFactory) { this.metadataWriteFilterFactory = metadataWriteFilterFactory; } + + public void setEmbeddedDocumentExtractorFactory( + EmbeddedDocumentExtractorFactory embeddedDocumentExtractorFactory) { + this.embeddedDocumentExtractorFactory = embeddedDocumentExtractorFactory; + } + + public EmbeddedDocumentExtractorFactory getEmbeddedDocumentExtractorFactory() { + return embeddedDocumentExtractorFactory; + } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLParserTest.java index b1c63cd72..fa7dc4cd6 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLParserTest.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLParserTest.java @@ -16,15 +16,19 @@ */ package org.apache.tika.parser.microsoft.ooxml; +import java.io.InputStream; import java.util.List; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.apache.tika.TikaTest; +import org.apache.tika.config.TikaConfig; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.parser.AutoDetectParser; import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.Parser; import org.apache.tika.parser.microsoft.OfficeParserConfig; public class OOXMLParserTest extends TikaTest { @@ -66,4 +70,26 @@ public class OOXMLParserTest extends TikaTest { //TIKA_2446 getRecursiveMetadata("testZIP_corrupted_oom.zip"); } + + @Test + public void testConfiguringEmbeddedDocExtractor() throws Exception { + + TikaConfig tikaConfig = null; + try (InputStream is = OOXMLParserTest.class.getResourceAsStream( + "/configs/tika-config-no-names.xml")) { + tikaConfig = new TikaConfig(is); + } + Parser p = new AutoDetectParser(tikaConfig); + String xml = getXML("testPPT_EmbeddedPDF.pptx", p).xml; + assertNotContained("<h1>/docProps/thumbnail.jpeg</h1>", xml); + + try (InputStream is = OOXMLParserTest.class.getResourceAsStream( + "/configs/tika-config-with-names.xml")) { + tikaConfig = new TikaConfig(is); + } + p = new AutoDetectParser(tikaConfig); + xml = getXML("testPPT_EmbeddedPDF.pptx", p).xml; + assertContains("<h1>/docProps/thumbnail.jpeg</h1>", xml); + System.out.println(xml); + } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/resources/configs/tika-config-no-names.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/resources/configs/tika-config-no-names.xml new file mode 100644 index 000000000..6f7cc95e1 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/resources/configs/tika-config-no-names.xml @@ -0,0 +1,33 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<properties> + <parsers> + <parser class="org.apache.tika.parser.DefaultParser"/> + </parsers> + <autoDetectParserConfig> + <params> + <spoolToDisk>123450</spoolToDisk> + <outputThreshold>678900</outputThreshold> + </params> + <embeddedDocumentExtractorFactory class="org.apache.tika.extractor.ParsingEmbeddedDocumentExtractorFactory"> + <params> + <writeFileNameToContent>false</writeFileNameToContent> + </params> + </embeddedDocumentExtractorFactory> + </autoDetectParserConfig> +</properties> diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/resources/configs/tika-config-with-names.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/resources/configs/tika-config-with-names.xml new file mode 100644 index 000000000..dae145194 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/resources/configs/tika-config-with-names.xml @@ -0,0 +1,33 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<properties> + <parsers> + <parser class="org.apache.tika.parser.DefaultParser"/> + </parsers> + <autoDetectParserConfig> + <params> + <spoolToDisk>123450</spoolToDisk> + <outputThreshold>678900</outputThreshold> + </params> + <embeddedDocumentExtractorFactory class="org.apache.tika.extractor.ParsingEmbeddedDocumentExtractorFactory"> + <params> + <writeFileNameToContent>true</writeFileNameToContent> + </params> + </embeddedDocumentExtractorFactory> + </autoDetectParserConfig> +</properties>
