Author: reschke
Date: Fri Jul 5 09:11:57 2019
New Revision: 1862598
URL: http://svn.apache.org/viewvc?rev=1862598&view=rev
Log:
OAK-5048 - Upgrade to Tika 1.15 version
OAK-6414 - Use Tika config to determine non indexed mimeTypes
Merge r1800609, r1800749, r1800756 into 1.6
Added:
jackrabbit/oak/branches/1.6/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/binary/TikaParserConfig.java
- copied unchanged from r1800749,
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/binary/TikaParserConfig.java
jackrabbit/oak/branches/1.6/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/binary/
- copied from r1800749,
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/binary/
Modified:
jackrabbit/oak/branches/1.6/ (props changed)
jackrabbit/oak/branches/1.6/oak-examples/standalone/pom.xml
jackrabbit/oak/branches/1.6/oak-examples/webapp/ (props changed)
jackrabbit/oak/branches/1.6/oak-examples/webapp/pom.xml
jackrabbit/oak/branches/1.6/oak-lucene/pom.xml
jackrabbit/oak/branches/1.6/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/binary/BinaryTextExtractor.java
jackrabbit/oak/branches/1.6/oak-parent/pom.xml
jackrabbit/oak/branches/1.6/oak-pojosr/pom.xml
jackrabbit/oak/branches/1.6/oak-run/pom.xml
jackrabbit/oak/branches/1.6/oak-solr-core/pom.xml
Propchange: jackrabbit/oak/branches/1.6/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 5 09:11:57 2019
@@ -1,4 +1,4 @@
/jackrabbit/oak/branches/1.0:1665962
/jackrabbit/oak/branches/1.8:1844835
-/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783110,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784034,1784130,1784162,1784251,1784401,1784551,1784574,1784689,1785095,1785108,1785161,1785172,1785283,1785652,1785838,1785916-1785917,1785919,1785946,1786122,1787074,1787145,1787151,1787217,1787425,1788056,1788378,1788387-1788389,1788463,1788476,1788850,1789056,1789534,1789925,1789940,1789987,1790006,1790013,1790069,1790077,1790079,1790382,1790502-1790503,1790549,1792049,1792463,1792742,1792746,1793013,1793088,1793618,1793627,1793644,1794393,1794417,1794683,1795138,1795314,1795330,1795475,1795488,1795491,1795502,1795594,1795613,1795618,1796144,1796230,1796239,1796274,1796278,1796988,1797378,1798035,1798832,1798834,1799219,1799389,1799393,1799924,1800244,1800269,1800606,1800613,1800974,1801011,1801013,1801118
-1801119,1801675,1802260,1802262,1802286,1802548,1802905,1802934,1802938,1802973,1803026,1803247-1803249,1803951,1803953-1803955,1804437,1805851-1805852,1806668,1807308,1807688,1808022,1808053,1808125,1808128,1808142,1808240,1808246,1808731,1809024,1809026,1809131,1809163,1809178-1809179,1809253,1809255-1809256,1809289,1809745,1811071-1811072,1811155,1811380,1811655,1811952,1811963,1811986,1813192,1813538,1814052,1814189,1814332,1814397,1814475,1815201,1815438,1815926,1816019,1817326,1817919,1817987-1817988,1817990,1818038,1818042,1818056,1818124,1818137,1818554,1818576,1818645,1819048,1819050,1821237,1821325,1821358,1821495,1821516,1821847,1822207,1822642,1822850,1823172,1823655,1824896,1825471,1825654,1826237,1826338,1826532,1826640,1826932,1826957,1827472,1827486,1827977,1828502,1829527,1829569,1829587,1829665,1829854,1829864,1829987,1829998,1830019,1830160,1830239,1830748,1831190,1831374,1832379,1832535,1833308,1834648-1834649,1834681,1835060,1835780,1836082,1837475,1837998,1838
637,1839746,1840024,1840226,1842677,1843175,1843222,1843231,1844549,1844642,1844728,1845135,1845405,1845415,1845730-1845731,1845863,1845865,1846057,1846396,1846617,1848181-1848182,1848191,1848217,1848822-1848823,1850837,1853429,1854044,1854461,1856818
+/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783110,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784034,1784130,1784162,1784251,1784401,1784551,1784574,1784689,1785095,1785108,1785161,1785172,1785283,1785652,1785838,1785916-1785917,1785919,1785946,1786122,1787074,1787145,1787151,1787217,1787425,1788056,1788378,1788387-1788389,1788463,1788476,1788850,1789056,1789534,1789925,1789940,1789987,1790006,1790013,1790069,1790077,1790079,1790382,1790502-1790503,1790549,1792049,1792463,1792742,1792746,1793013,1793088,1793618,1793627,1793644,1794393,1794417,1794683,1795138,1795314,1795330,1795475,1795488,1795491,1795502,1795594,1795613,1795618,1796144,1796230,1796239,1796274,1796278,1796988,1797378,1798035,1798832,1798834,1799219,1799389,1799393,1799924,1800244,1800269,1800606,1800609,1800613,1800749,1800756,1800974
,1801011,1801013,1801118-1801119,1801675,1802260,1802262,1802286,1802548,1802905,1802934,1802938,1802973,1803026,1803247-1803249,1803951,1803953-1803955,1804437,1805851-1805852,1806668,1807308,1807688,1808022,1808053,1808125,1808128,1808142,1808240,1808246,1808731,1809024,1809026,1809131,1809163,1809178-1809179,1809253,1809255-1809256,1809289,1809745,1811071-1811072,1811155,1811380,1811655,1811952,1811963,1811986,1813192,1813538,1814052,1814189,1814332,1814397,1814475,1815201,1815438,1815926,1816019,1817326,1817919,1817987-1817988,1817990,1818038,1818042,1818056,1818124,1818137,1818554,1818576,1818645,1819048,1819050,1821237,1821325,1821358,1821495,1821516,1821847,1822207,1822642,1822850,1823172,1823655,1824896,1825471,1825654,1826237,1826338,1826532,1826640,1826932,1826957,1827472,1827486,1827977,1828502,1829527,1829569,1829587,1829665,1829854,1829864,1829987,1829998,1830019,1830160,1830239,1830748,1831190,1831374,1832379,1832535,1833308,1834648-1834649,1834681,1835060,1835780,1836
082,1837475,1837998,1838637,1839746,1840024,1840226,1842677,1843175,1843222,1843231,1844549,1844642,1844728,1845135,1845405,1845415,1845730-1845731,1845863,1845865,1846057,1846396,1846617,1848181-1848182,1848191,1848217,1848822-1848823,1850837,1853429,1854044,1854461,1856818
/jackrabbit/trunk:1345480
Modified: jackrabbit/oak/branches/1.6/oak-examples/standalone/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-examples/standalone/pom.xml?rev=1862598&r1=1862597&r2=1862598&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-examples/standalone/pom.xml (original)
+++ jackrabbit/oak/branches/1.6/oak-examples/standalone/pom.xml Fri Jul 5
09:11:57 2019
@@ -35,7 +35,6 @@
<description>Standalone application based on Spring Boot then embeds a
Jackrabbit Oak content repository</description>
<properties>
- <tika.version>1.7</tika.version>
<skip.deployment>true</skip.deployment>
<slf4j.version>1.7.21</slf4j.version>
<slf4j.api.version>1.7.21</slf4j.api.version>
@@ -134,6 +133,10 @@
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
Propchange: jackrabbit/oak/branches/1.6/oak-examples/webapp/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jul 5 09:11:57 2019
@@ -1,3 +1,3 @@
/jackrabbit/branches/JCR-2272/jackrabbit-webapp:1173165-1176545
-/jackrabbit/oak/trunk/oak-examples/webapp:1782990,1785108
+/jackrabbit/oak/trunk/oak-examples/webapp:1782990,1785108,1800609,1800756
/jackrabbit/sandbox/JCR-2415-lucene-3.0/jackrabbit-webapp:1060860-1064038
Modified: jackrabbit/oak/branches/1.6/oak-examples/webapp/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-examples/webapp/pom.xml?rev=1862598&r1=1862597&r2=1862598&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-examples/webapp/pom.xml (original)
+++ jackrabbit/oak/branches/1.6/oak-examples/webapp/pom.xml Fri Jul 5 09:11:57
2019
@@ -36,7 +36,6 @@
<properties>
<tomcat.version>7.0.40</tomcat.version>
- <tika.version>1.7</tika.version>
<skip.deployment>true</skip.deployment>
</properties>
@@ -59,6 +58,10 @@
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
Modified: jackrabbit/oak/branches/1.6/oak-lucene/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-lucene/pom.xml?rev=1862598&r1=1862597&r2=1862598&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-lucene/pom.xml (original)
+++ jackrabbit/oak/branches/1.6/oak-lucene/pom.xml Fri Jul 5 09:11:57 2019
@@ -33,7 +33,6 @@
<description>Oak Lucene integration subproject</description>
<properties>
- <tika.version>1.5</tika.version>
<known.issues>
<!-- Jackrabbit query tests -->
org.apache.jackrabbit.core.query.ExcerptTest#testMoreTextDotsAtEnd
<!-- OAK-318 -->
Modified:
jackrabbit/oak/branches/1.6/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/binary/BinaryTextExtractor.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/binary/BinaryTextExtractor.java?rev=1862598&r1=1862597&r2=1862598&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/binary/BinaryTextExtractor.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/binary/BinaryTextExtractor.java
Fri Jul 5 09:11:57 2019
@@ -19,9 +19,11 @@
package org.apache.jackrabbit.oak.plugins.index.lucene.binary;
+import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Set;
@@ -40,6 +42,7 @@ import org.apache.jackrabbit.oak.spi.sta
import org.apache.jackrabbit.oak.util.BlobByteSource;
import org.apache.lucene.document.Field;
import org.apache.tika.config.TikaConfig;
+import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.AutoDetectParser;
@@ -48,6 +51,7 @@ import org.apache.tika.parser.Parser;
import org.apache.tika.sax.WriteOutContentHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.xml.sax.SAXException;
import static org.apache.jackrabbit.JcrConstants.JCR_DATA;
import static
org.apache.jackrabbit.oak.plugins.index.lucene.FieldFactory.newFulltextField;
@@ -66,6 +70,7 @@ public class BinaryTextExtractor {
* The media types supported by the parser used.
*/
private Set<MediaType> supportedMediaTypes;
+ private Set<MediaType> nonIndexedMediaType;
public BinaryTextExtractor(ExtractedTextCache extractedTextCache,
IndexDefinition definition, boolean reindex) {
this.extractedTextCache = extractedTextCache;
@@ -203,8 +208,36 @@ public class BinaryTextExtractor {
private boolean isSupportedMediaType(String type) {
if (supportedMediaTypes == null) {
supportedMediaTypes = getParser().getSupportedTypes(new
ParseContext());
+ nonIndexedMediaType = getNonIndexedMediaTypes();
}
- return supportedMediaTypes.contains(MediaType.parse(type));
+ MediaType mediaType = MediaType.parse(type);
+ return supportedMediaTypes.contains(mediaType) &&
!nonIndexedMediaType.contains(mediaType);
+ }
+
+ private Set<MediaType> getNonIndexedMediaTypes() {
+ InputStream configStream = null;
+ String configSource = null;
+ try {
+ if (definition.hasCustomTikaConfig()) {
+ configSource = String.format("Custom config at %s",
definition.getIndexPath());
+ configStream = definition.getTikaConfig();
+ } else {
+ URL configUrl =
LuceneIndexEditorContext.class.getResource("tika-config.xml");
+ configSource = "Default : tika-config.xml";
+ if (configUrl != null) {
+ configStream = configUrl.openStream();
+ }
+ }
+
+ if (configStream != null) {
+ return TikaParserConfig.getNonIndexedMediaTypes(configStream);
+ }
+ } catch (TikaException | IOException | SAXException e) {
+ log.warn("Tika configuration not available : " + configSource, e);
+ } finally {
+ IOUtils.closeQuietly(configStream);
+ }
+ return Collections.emptySet();
}
private static Parser initializeTikaParser(IndexDefinition definition) {
Modified: jackrabbit/oak/branches/1.6/oak-parent/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-parent/pom.xml?rev=1862598&r1=1862597&r2=1862598&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-parent/pom.xml (original)
+++ jackrabbit/oak/branches/1.6/oak-parent/pom.xml Fri Jul 5 09:11:57 2019
@@ -55,6 +55,7 @@
<slf4j.version>1.7.26</slf4j.version> <!-- sync with logback version -->
<logback.version>1.2.3</logback.version>
<h2.version>1.4.194</h2.version>
+ <tika.version>1.15</tika.version>
<findbugs.version>3.0.5</findbugs.version>
<derby.version>10.12.1.1</derby.version>
<jackson.version>2.9.6</jackson.version>
Modified: jackrabbit/oak/branches/1.6/oak-pojosr/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-pojosr/pom.xml?rev=1862598&r1=1862597&r2=1862598&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-pojosr/pom.xml (original)
+++ jackrabbit/oak/branches/1.6/oak-pojosr/pom.xml Fri Jul 5 09:11:57 2019
@@ -33,7 +33,6 @@
<properties>
<skip.deployment>true</skip.deployment>
<groovy.version>2.3.1</groovy.version>
- <tika.version>1.5</tika.version>
</properties>
<build>
Modified: jackrabbit/oak/branches/1.6/oak-run/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-run/pom.xml?rev=1862598&r1=1862597&r2=1862598&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-run/pom.xml (original)
+++ jackrabbit/oak/branches/1.6/oak-run/pom.xml Fri Jul 5 09:11:57 2019
@@ -400,12 +400,12 @@
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-parsers</artifactId>
- <version>1.5</version>
+ <version>${tika.version}</version>
</dependency>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-core</artifactId>
- <version>1.5</version>
+ <version>${tika.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
Modified: jackrabbit/oak/branches/1.6/oak-solr-core/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-solr-core/pom.xml?rev=1862598&r1=1862597&r2=1862598&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-solr-core/pom.xml (original)
+++ jackrabbit/oak/branches/1.6/oak-solr-core/pom.xml Fri Jul 5 09:11:57 2019
@@ -33,7 +33,6 @@
<packaging>bundle</packaging>
<properties>
- <tika.version>1.5</tika.version>
<known.issues>
org.apache.jackrabbit.core.query.FulltextQueryTest#testMultiByte
<!-- wildcards with multi byte support -->
org.apache.jackrabbit.core.query.FnNameQueryTest#testLikeWithPrefix
<!-- OAK-328 -->
