This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 9f9b452634 TIKA-4954: speed up parser loading and pipes fork startup:
build pars… (#3297)
9f9b452634 is described below
commit 9f9b452634bac9d0e04f78b8c113cd69aee13603
Author: Tim Allison <[email protected]>
AuthorDate: Wed Oct 7 15:01:12 2026 -0400
TIKA-4954: speed up parser loading and pipes fork startup: build pars…
(#3297)
* TIKA-4954: speed up parser loading and pipes fork startup: build parsers'
default EncodingDetector lazily, recognize a failed fork start within 100ms
* TIKA-4954: DBFParser honors the configured EncodingDetector
---------
Co-authored-by: Tilman Hausherr <[email protected]>
---
CHANGES.txt | 8 ++++++++
.../tika/parser/AbstractEncodingDetectorParser.java | 16 +++++++++++++---
.../apache/tika/config/TikaEncodingDetectorTest.java | 6 +++---
.../java/org/apache/tika/parser/dbf/DBFParser.java | 19 +++++++++++--------
.../org/apache/tika/parser/dbf/DBFParserTest.java | 17 +++++++++++++++++
.../tika/pipes/core/PerClientServerManager.java | 4 ++--
6 files changed, 54 insertions(+), 16 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index dc63817378..a8d1fcbf40 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -88,6 +88,14 @@ Release 4.2.0 - unreleased
exit, and the ilst length no longer counts its own header twice. Since
4.0.0 (TIKA-4957).
+ * Faster parser loading and pipes fork startup: parsers no longer build a
+ default EncodingDetector (loading the charset models) only to have the
+ loader replace it, saving about 1.3s per AutoDetectParser build and per
+ forked JVM; a fork that fails to start is recognized within 100ms instead
+ of 1s (TIKA-4954).
+ DBFParser now honors the configured EncodingDetector instead of always
+ building its own default (TIKA-4954).
+
* tika-core's OSGi manifest no longer requires a Service Loader Mediator or
providers for Parser, Detector, EncodingDetector, LanguageDetector and
MetadataFilter; 4.1.0 failed to install on its own in Equinox/p2
diff --git
a/tika-core/src/main/java/org/apache/tika/parser/AbstractEncodingDetectorParser.java
b/tika-core/src/main/java/org/apache/tika/parser/AbstractEncodingDetectorParser.java
index db104a4b2b..fcf70dbe4c 100644
---
a/tika-core/src/main/java/org/apache/tika/parser/AbstractEncodingDetectorParser.java
+++
b/tika-core/src/main/java/org/apache/tika/parser/AbstractEncodingDetectorParser.java
@@ -27,10 +27,10 @@ import org.apache.tika.detect.EncodingDetector;
public abstract class AbstractEncodingDetectorParser implements Parser {
- private EncodingDetector encodingDetector;
+ // lazy: loaders replace it with the shared detector, and building one
loads the models
+ private volatile EncodingDetector encodingDetector;
public AbstractEncodingDetectorParser() {
- encodingDetector = new DefaultEncodingDetector();
}
public AbstractEncodingDetectorParser(EncodingDetector encodingDetector) {
@@ -55,7 +55,17 @@ public abstract class AbstractEncodingDetectorParser
implements Parser {
}
public EncodingDetector getEncodingDetector() {
- return encodingDetector;
+ EncodingDetector detector = encodingDetector;
+ if (detector == null) {
+ synchronized (this) {
+ detector = encodingDetector;
+ if (detector == null) {
+ detector = new DefaultEncodingDetector();
+ encodingDetector = detector;
+ }
+ }
+ }
+ return detector;
}
public void setEncodingDetector(EncodingDetector encodingDetector) {
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
index fdbe71d0be..84828bb3d6 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
@@ -147,7 +147,7 @@ public class TikaEncodingDetectorTest extends TikaTest {
List<Parser> parsers = new ArrayList<>();
findEncodingDetectionParsers(p, parsers);
- assertEquals(8, parsers.size());
+ assertEquals(9, parsers.size());
EncodingDetector encodingDetector =
((AbstractEncodingDetectorParser)
parsers.get(0)).getEncodingDetector();
assertTrue(encodingDetector instanceof CompositeEncodingDetector);
@@ -177,7 +177,7 @@ public class TikaEncodingDetectorTest extends TikaTest {
List<Parser> parsers = new ArrayList<>();
findEncodingDetectionParsers(p, parsers);
- assertEquals(9, parsers.size());
+ assertEquals(10, parsers.size());
for (Parser encodingDetectingParser : parsers) {
EncodingDetector encodingDetector =
@@ -203,7 +203,7 @@ public class TikaEncodingDetectorTest extends TikaTest {
List<Parser> parsers = new ArrayList<>();
findEncodingDetectionParsers(p, parsers);
- assertEquals(8, parsers.size());
+ assertEquals(9, parsers.size());
for (Parser childParser : parsers) {
EncodingDetector encodingDetector =
((AbstractEncodingDetectorParser)
childParser).getEncodingDetector();
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/dbf/DBFParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/dbf/DBFParser.java
index 0be11be78c..7389ccb795 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/dbf/DBFParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/dbf/DBFParser.java
@@ -31,7 +31,6 @@ import org.xml.sax.SAXException;
import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.detect.CharsetSupersets;
-import org.apache.tika.detect.DefaultEncodingDetector;
import org.apache.tika.detect.EncodingDetector;
import org.apache.tika.detect.EncodingResult;
import org.apache.tika.exception.TikaException;
@@ -40,8 +39,8 @@ import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.AbstractEncodingDetectorParser;
import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.Parser;
import org.apache.tika.sax.XHTMLContentHandler;
/**
@@ -53,16 +52,23 @@ import org.apache.tika.sax.XHTMLContentHandler;
* on the "character" cells.
*/
@TikaComponent
-public class DBFParser implements Parser {
+public class DBFParser extends AbstractEncodingDetectorParser {
private static final int ROWS_TO_BUFFER_FOR_CHARSET_DETECTION = 10;
private static final int MAX_CHARS_FOR_CHARSET_DETECTION = 20000;
private static final Charset DEFAULT_CHARSET = StandardCharsets.ISO_8859_1;
- private static final EncodingDetector DEFAULT_ENCODING_DETECTOR = new
DefaultEncodingDetector();
private static final Set<MediaType> SUPPORTED_TYPES =
Collections.singleton(MediaType.application("x-dbf"));
+ public DBFParser() {
+ super();
+ }
+
+ public DBFParser(EncodingDetector encodingDetector) {
+ super(encodingDetector);
+ }
+
@Override
public Set<MediaType> getSupportedTypes(ParseContext context) {
return SUPPORTED_TYPES;
@@ -142,10 +148,7 @@ public class DBFParser implements Parser {
}
byte[] bytes = bos.toByteArray();
if (bytes.length > 20) {
- EncodingDetector detector =
parseContext.get(EncodingDetector.class);
- if (detector == null) {
- detector = DEFAULT_ENCODING_DETECTOR;
- }
+ EncodingDetector detector = getEncodingDetector(parseContext);
try (TikaInputStream tis = TikaInputStream.get(bytes)) {
List<EncodingResult> results =
detector.detect(tis, new Metadata(), parseContext);
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/dbf/DBFParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/dbf/DBFParserTest.java
index 772833cca7..cdf65e6005 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/dbf/DBFParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/dbf/DBFParserTest.java
@@ -21,16 +21,20 @@ import static org.junit.jupiter.api.Assertions.fail;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.Test;
import org.apache.tika.TikaTest;
+import org.apache.tika.detect.EncodingDetector;
+import org.apache.tika.detect.OverrideEncodingDetector;
import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
public class DBFParserTest extends TikaTest {
@@ -61,6 +65,19 @@ public class DBFParserTest extends TikaTest {
assertContains("<td>licour</td>", xml);
}
+ @Test
+ public void testConfiguredEncodingDetector() throws Exception {
+ Parser p = new DBFParser(new
OverrideEncodingDetector(StandardCharsets.ISO_8859_1));
+ XMLResult r = getXML("testDBF.dbf", p, new Metadata());
+ assertEquals("ISO-8859-1",
r.metadata.get(HttpHeaders.CONTENT_ENCODING));
+
+ ParseContext context = new ParseContext();
+ context.set(EncodingDetector.class,
+ new OverrideEncodingDetector(StandardCharsets.UTF_16LE));
+ r = getXML("testDBF.dbf", p, new Metadata(), context);
+ assertEquals("UTF-16LE", r.metadata.get(HttpHeaders.CONTENT_ENCODING));
+ }
+
@Test
public void testGB18030Encoded() throws Exception {
XMLResult r = getXML("testDBF_gb18030.dbf");
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PerClientServerManager.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PerClientServerManager.java
index 59390013f4..f6a7b17a45 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PerClientServerManager.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PerClientServerManager.java
@@ -414,8 +414,8 @@ public class PerClientServerManager implements
ServerManager {
throw new IllegalStateException("Server not started. Call
ensureRunning() first.");
}
- // Accept incoming connection from the server process
- ss.setSoTimeout(1000); // 1 second timeout for each poll
+ // Accept the server's connection; poll briefly so a child that dies
first is noticed fast
+ ss.setSoTimeout(100);
long startTime = System.currentTimeMillis();
while (true) {
