This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 68e5762116 TIKA-4889: engines config section and
openai-embedding-engine (#3154)
68e5762116 is described below
commit 68e57621168adf9e8e3004e3ff4ea6fd5e4c3cd7
Author: Tim Allison <[email protected]>
AuthorDate: Thu Sep 10 20:05:22 2026 -0400
TIKA-4889: engines config section and openai-embedding-engine (#3154)
---
CHANGES.txt | 6 +
.../org/apache/tika/parser/inference/Engine.java | 27 +++
.../tika/parser/inference/EngineRegistry.java | 39 ++++
.../org/apache/tika/inference/EmbeddingEngine.java | 34 ++++
.../tika/inference/OpenAIEmbeddingEngine.java | 210 +++++++++++++++++++++
.../tika/inference/OpenAIEmbeddingEngineTest.java | 113 +++++++++++
.../apache/tika/config/loader/EngineLoader.java | 77 ++++++++
.../apache/tika/config/loader/TikaJsonConfig.java | 1 +
.../org/apache/tika/config/loader/TikaLoader.java | 4 +
.../tika/config/loader/EngineLoaderTest.java | 78 ++++++++
.../org/apache/tika/config/loader/TestEngine.java | 35 ++++
11 files changed, 624 insertions(+)
diff --git a/CHANGES.txt b/CHANGES.txt
index 7d5a977ed0..9fded46d6c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,11 @@
Release 4.1.0 - unreleased
+ * New "engines" config section: a map of names to inference engines, each
+ an endpoint or local binary with its settings (openai-embedding-engine
+ posts up to maxBatchSize images per request to an OpenAI-compatible
+ /v1/embeddings). Engines are called by the inference bindings that
+ follow; nothing calls them yet (TIKA-4889).
+
* The image embedder writes a picture's vector onto the document the
picture appears in (INLINE and RENDERING children) with an "embedded"
locator naming the picture; attachments keep their own, and
diff --git
a/tika-core/src/main/java/org/apache/tika/parser/inference/Engine.java
b/tika-core/src/main/java/org/apache/tika/parser/inference/Engine.java
new file mode 100644
index 0000000000..ba71a2d0b9
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/parser/inference/Engine.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.inference;
+
+/**
+ * A model or service an inference binding calls: one endpoint or one local
binary with its
+ * settings, named in the {@code "engines"} map. An engine knows nothing about
documents; a
+ * binding says what it is fed and what to ask.
+ *
+ * @since Apache Tika 4.1
+ */
+public interface Engine {
+}
diff --git
a/tika-core/src/main/java/org/apache/tika/parser/inference/EngineRegistry.java
b/tika-core/src/main/java/org/apache/tika/parser/inference/EngineRegistry.java
new file mode 100644
index 0000000000..39a57d3cc7
--- /dev/null
+++
b/tika-core/src/main/java/org/apache/tika/parser/inference/EngineRegistry.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.inference;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/** The {@code "engines"} map: user-chosen name to engine, built once at
config load. */
+public final class EngineRegistry {
+
+ private final Map<String, Engine> engines;
+
+ public EngineRegistry(Map<String, Engine> engines) {
+ this.engines = Collections.unmodifiableMap(new
LinkedHashMap<>(engines));
+ }
+
+ public Engine get(String name) {
+ return engines.get(name);
+ }
+
+ public Map<String, Engine> getEngines() {
+ return engines;
+ }
+}
diff --git
a/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/EmbeddingEngine.java
b/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/EmbeddingEngine.java
new file mode 100644
index 0000000000..169b233967
--- /dev/null
+++
b/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/EmbeddingEngine.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.inference;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.inference.Engine;
+
+/** An engine that turns images into vectors, many per request. */
+public interface EmbeddingEngine extends Engine {
+
+ /** One request; the vectors come back in the order of the images. */
+ List<float[]> embedImages(List<byte[]> images, List<String> mimeTypes,
ParseContext context)
+ throws IOException, TikaException;
+
+ int getMaxBatchSize();
+}
diff --git
a/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/OpenAIEmbeddingEngine.java
b/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/OpenAIEmbeddingEngine.java
new file mode 100644
index 0000000000..90e85f5d2d
--- /dev/null
+++
b/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/OpenAIEmbeddingEngine.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.inference;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import org.apache.tika.annotation.TikaComponent;
+import org.apache.tika.config.Initializable;
+import org.apache.tika.config.ParseTimeout;
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.http.TikaHttpClient;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.utils.StringUtils;
+
+/**
+ * An OpenAI-compatible embeddings endpoint ({@code POST /v1/embeddings}),
named in
+ * {@code "engines"}. Sends up to {@code maxBatchSize} images per request as
+ * {@code {"image": "data:<mime>;base64,..."}} inputs and reads the vectors
back by index.
+ */
+@TikaComponent(name = "openai-embedding-engine", spi = false)
+public class OpenAIEmbeddingEngine implements EmbeddingEngine, Initializable,
Closeable {
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ private String baseUrl = "http://localhost:8000";;
+ private String model = "";
+ private String apiKey = "";
+ private long timeoutMillis = 120_000;
+ private int maxBatchSize = 32;
+ private String embeddingsPath = "/v1/embeddings";
+ private String apiKeyHeaderName = "Authorization";
+ private String apiKeyPrefix = "Bearer ";
+
+ private TikaHttpClient httpClient;
+
+ @Override
+ public void initialize() throws TikaConfigException {
+ if (StringUtils.isBlank(baseUrl)) {
+ throw new TikaConfigException("openai-embedding-engine needs a
baseUrl");
+ }
+ if (maxBatchSize < 1) {
+ throw new TikaConfigException("maxBatchSize must be at least 1");
+ }
+ httpClient = TikaHttpClient.build(30);
+ }
+
+ @Override
+ public List<float[]> embedImages(List<byte[]> images, List<String>
mimeTypes,
+ ParseContext context) throws IOException,
TikaException {
+ if (httpClient == null) {
+ httpClient = TikaHttpClient.build(30);
+ }
+ ObjectNode root = MAPPER.createObjectNode();
+ if (!StringUtils.isBlank(model)) {
+ root.put("model", model);
+ }
+ ArrayNode input = root.putArray("input");
+ for (int i = 0; i < images.size(); i++) {
+ input.addObject().put("image", "data:" + mimeTypes.get(i) +
";base64,"
+ + Base64.getEncoder().encodeToString(images.get(i)));
+ }
+ Map<String, String> headers = new HashMap<>();
+ if (!StringUtils.isBlank(apiKey)) {
+ headers.put(apiKeyHeaderName, apiKeyPrefix + apiKey);
+ }
+ String url = baseUrl.replaceAll("/+$", "") + embeddingsPath;
+ String body = httpClient.postJson(url, root.toString(), headers,
timeoutMillis, context);
+ ParseTimeout.checkpoint(context);
+ return parseResponse(body, images.size());
+ }
+
+ static List<float[]> parseResponse(String body, int expected) throws
TikaException {
+ try {
+ JsonNode data = MAPPER.readTree(body).get("data");
+ if (data == null || !data.isArray() || data.size() != expected) {
+ throw new TikaException("Embedding response has " + (data ==
null ? "no"
+ : data.size()) + " data entries; expected " +
expected);
+ }
+ List<float[]> vectors = new ArrayList<>(expected);
+ for (int i = 0; i < expected; i++) {
+ vectors.add(null);
+ }
+ for (int i = 0; i < data.size(); i++) {
+ JsonNode item = data.get(i);
+ int index = item.path("index").asInt(i);
+ JsonNode embedding = item.get("embedding");
+ if (index < 0 || index >= expected || embedding == null ||
!embedding.isArray()) {
+ throw new TikaException("Embedding response entry " + i +
" is malformed");
+ }
+ float[] vector = new float[embedding.size()];
+ for (int d = 0; d < vector.length; d++) {
+ vector[d] = (float) embedding.get(d).asDouble();
+ if (Float.isNaN(vector[d]) || Float.isInfinite(vector[d]))
{
+ throw new TikaException("Embedding " + index + " has
an invalid float");
+ }
+ }
+ if (vectors.get(index) != null) {
+ throw new TikaException("Embedding response repeats index
" + index);
+ }
+ vectors.set(index, vector);
+ }
+ for (int i = 0; i < expected; i++) {
+ if (vectors.get(i) == null) {
+ throw new TikaException("Embedding response has no entry
for index " + i);
+ }
+ }
+ return vectors;
+ } catch (IOException e) {
+ throw new TikaException("Failed to parse embedding response", e);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (httpClient != null) {
+ httpClient.close();
+ }
+ }
+
+ public String getBaseUrl() {
+ return baseUrl;
+ }
+
+ public void setBaseUrl(String baseUrl) {
+ this.baseUrl = baseUrl;
+ }
+
+ public String getModel() {
+ return model;
+ }
+
+ public void setModel(String model) {
+ this.model = model;
+ }
+
+ public String getApiKey() {
+ return apiKey;
+ }
+
+ public void setApiKey(String apiKey) {
+ this.apiKey = apiKey;
+ }
+
+ public long getTimeoutMillis() {
+ return timeoutMillis;
+ }
+
+ public void setTimeoutMillis(long timeoutMillis) {
+ this.timeoutMillis = timeoutMillis;
+ }
+
+ @Override
+ public int getMaxBatchSize() {
+ return maxBatchSize;
+ }
+
+ public void setMaxBatchSize(int maxBatchSize) {
+ this.maxBatchSize = maxBatchSize;
+ }
+
+ public String getEmbeddingsPath() {
+ return embeddingsPath;
+ }
+
+ public void setEmbeddingsPath(String embeddingsPath) {
+ this.embeddingsPath = embeddingsPath;
+ }
+
+ public String getApiKeyHeaderName() {
+ return apiKeyHeaderName;
+ }
+
+ public void setApiKeyHeaderName(String apiKeyHeaderName) {
+ this.apiKeyHeaderName = apiKeyHeaderName;
+ }
+
+ public String getApiKeyPrefix() {
+ return apiKeyPrefix;
+ }
+
+ public void setApiKeyPrefix(String apiKeyPrefix) {
+ this.apiKeyPrefix = apiKeyPrefix;
+ }
+}
diff --git
a/tika-parsers/tika-parsers-ml/tika-inference/src/test/java/org/apache/tika/inference/OpenAIEmbeddingEngineTest.java
b/tika-parsers/tika-parsers-ml/tika-inference/src/test/java/org/apache/tika/inference/OpenAIEmbeddingEngineTest.java
new file mode 100644
index 0000000000..04ed8d387c
--- /dev/null
+++
b/tika-parsers/tika-parsers-ml/tika-inference/src/test/java/org/apache/tika/inference/OpenAIEmbeddingEngineTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.inference;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.http.TikaTestHttpServer;
+import org.apache.tika.parser.ParseContext;
+
+public class OpenAIEmbeddingEngineTest {
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+ private TikaTestHttpServer server;
+ private OpenAIEmbeddingEngine engine;
+
+ @BeforeEach
+ void setUp() throws Exception {
+ server = new TikaTestHttpServer();
+ engine = new OpenAIEmbeddingEngine();
+ engine.setBaseUrl(server.url());
+ engine.setModel("clip");
+ engine.setApiKey("secret");
+ engine.initialize();
+ }
+
+ @AfterEach
+ void tearDown() throws Exception {
+ engine.close();
+ server.shutdown();
+ }
+
+ private static byte[] bytes(String s) {
+ return s.getBytes(StandardCharsets.UTF_8);
+ }
+
+ @Test
+ public void testManyImagesOneRequestPlacedByIndex() throws Exception {
+ // out of order on purpose: vectors are placed by index, not position
+ server.enqueue(new TikaTestHttpServer.MockResponse(200, "{\"data\":["
+ +
"{\"index\":1,\"embedding\":[1.0,0.5]},{\"index\":0,\"embedding\":[0.0,0.5]}]}"));
+ List<float[]> vectors = engine.embedImages(List.of(bytes("a"),
bytes("b")),
+ List.of("image/png", "image/jpeg"), new ParseContext());
+ assertEquals(2, vectors.size());
+ assertEquals(0.0f, vectors.get(0)[0]);
+ assertEquals(1.0f, vectors.get(1)[0]);
+
+ TikaTestHttpServer.RecordedRequest request = server.takeRequest();
+ assertEquals("/v1/embeddings", request.path());
+ assertEquals("Bearer secret", request.header("Authorization"));
+ JsonNode body = MAPPER.readTree(request.body());
+ assertEquals("clip", body.get("model").asText());
+ assertEquals(2, body.get("input").size());
+ assertEquals("data:image/png;base64,YQ==",
body.get("input").get(0).get("image").asText());
+ assertEquals("data:image/jpeg;base64,Yg==",
body.get("input").get(1).get("image").asText());
+ }
+
+ @Test
+ public void testMalformedResponsesAreRefused() throws Exception {
+ String[] bad = {
+ "{\"data\":[{\"index\":0,\"embedding\":[1.0]}]}",
+
"{\"data\":[{\"index\":0,\"embedding\":[1.0]},{\"index\":0,\"embedding\":[2.0]}]}",
+
"{\"data\":[{\"index\":0,\"embedding\":[1.0]},{\"index\":5,\"embedding\":[2.0]}]}",
+
"{\"data\":[{\"index\":0,\"embedding\":[1.0]},{\"index\":1,\"embedding\":\"x\"}]}",
+
"{\"data\":[{\"index\":0,\"embedding\":[1.0]},{\"index\":1,\"embedding\":[\"NaN\"]}]}",
+ "{}",
+ };
+ for (String body : bad) {
+ server.enqueue(new TikaTestHttpServer.MockResponse(200, body));
+ assertThrows(TikaException.class, () -> engine.embedImages(
+ List.of(bytes("a"), bytes("b")), List.of("image/png",
"image/png"),
+ new ParseContext()), body);
+ }
+ }
+
+ @Test
+ public void testHttpErrorIsAnException() throws Exception {
+ server.enqueue(new TikaTestHttpServer.MockResponse(500, "boom"));
+ assertThrows(TikaException.class, () ->
engine.embedImages(List.of(bytes("a")),
+ List.of("image/png"), new ParseContext()));
+ }
+
+ @Test
+ public void testInitializeValidates() {
+ OpenAIEmbeddingEngine bad = new OpenAIEmbeddingEngine();
+ bad.setMaxBatchSize(0);
+ assertThrows(org.apache.tika.exception.TikaConfigException.class,
bad::initialize);
+ }
+}
diff --git
a/tika-serialization/src/main/java/org/apache/tika/config/loader/EngineLoader.java
b/tika-serialization/src/main/java/org/apache/tika/config/loader/EngineLoader.java
new file mode 100644
index 0000000000..cc01e342f5
--- /dev/null
+++
b/tika-serialization/src/main/java/org/apache/tika/config/loader/EngineLoader.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.config.loader;
+
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.parser.inference.Engine;
+import org.apache.tika.parser.inference.EngineRegistry;
+
+/**
+ * Loads {@code "engines"}: a map of user-chosen name to one engine,
+ * {@code {"clip": {"openai-embedding-engine": {...}}}}. Null when the key is
absent.
+ */
+class EngineLoader implements ComponentLoader<EngineRegistry> {
+
+ static final String KEY = "engines";
+ private static final Pattern LEGAL_NAME =
Pattern.compile("[A-Za-z0-9._-]+");
+
+ @Override
+ public EngineRegistry load(TikaJsonConfig config, LoaderContext context)
+ throws TikaConfigException {
+ JsonNode node = config.getRootNode().get(KEY);
+ if (node == null) {
+ return null;
+ }
+ if (!node.isObject()) {
+ throw new TikaConfigException("\"" + KEY + "\" must be an object
keyed by engine "
+ + "name, e.g. {\"clip\": {\"openai-embedding-engine\":
{...}}}");
+ }
+ Map<String, Engine> engines = new LinkedHashMap<>();
+ Iterator<Map.Entry<String, JsonNode>> fields = node.fields();
+ while (fields.hasNext()) {
+ Map.Entry<String, JsonNode> e = fields.next();
+ String name = e.getKey();
+ if (!LEGAL_NAME.matcher(name).matches()) {
+ throw new TikaConfigException("engine name \"" + name + "\"
may use only "
+ + "letters, digits, '.', '_' and '-'");
+ }
+ JsonNode value = e.getValue();
+ if (!value.isObject() || value.size() != 1) {
+ throw new TikaConfigException("engine \"" + name + "\" must be
one "
+ + "{\"<engine-type>\": {...}} object");
+ }
+ Map.Entry<String, JsonNode> type = value.fields().next();
+ Engine engine;
+ try {
+ engine =
ComponentInstantiator.instantiateComponent(type.getKey(),
+ type.getValue(), context.getObjectMapper(),
context.getClassLoader(),
+ Engine.class);
+ } catch (TikaConfigException ex) {
+ throw new TikaConfigException("engine \"" + name + "\": " +
ex.getMessage(), ex);
+ }
+ engines.put(name, engine);
+ }
+ return new EngineRegistry(engines);
+ }
+}
diff --git
a/tika-serialization/src/main/java/org/apache/tika/config/loader/TikaJsonConfig.java
b/tika-serialization/src/main/java/org/apache/tika/config/loader/TikaJsonConfig.java
index a3aecf240c..04fcf89073 100644
---
a/tika-serialization/src/main/java/org/apache/tika/config/loader/TikaJsonConfig.java
+++
b/tika-serialization/src/main/java/org/apache/tika/config/loader/TikaJsonConfig.java
@@ -114,6 +114,7 @@ public class TikaJsonConfig {
"content-handler-factory",
"renderers",
"text-recognizers",
+ "engines",
"translator",
"auto-detect-parser",
"parse-context",
diff --git
a/tika-serialization/src/main/java/org/apache/tika/config/loader/TikaLoader.java
b/tika-serialization/src/main/java/org/apache/tika/config/loader/TikaLoader.java
index adb09a86db..594f93f815 100644
---
a/tika-serialization/src/main/java/org/apache/tika/config/loader/TikaLoader.java
+++
b/tika-serialization/src/main/java/org/apache/tika/config/loader/TikaLoader.java
@@ -53,6 +53,7 @@ import org.apache.tika.parser.CompositeParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.enricher.CompositeContentEnricher;
+import org.apache.tika.parser.inference.EngineRegistry;
import org.apache.tika.renderer.CompositeRenderer;
import org.apache.tika.renderer.Renderer;
import org.apache.tika.sax.BasicContentHandlerFactory;
@@ -138,6 +139,9 @@ public class TikaLoader {
ComponentConfig.builder("text-recognizers",
CompositeContentEnricher.class)
.customLoader(new ContentEnricherLoader())
.register();
+ ComponentConfig.builder(EngineLoader.KEY, EngineRegistry.class)
+ .customLoader(new EngineLoader())
+ .register();
ComponentConfig.builder("translator", Translator.class)
.loadAsList()
diff --git
a/tika-serialization/src/test/java/org/apache/tika/config/loader/EngineLoaderTest.java
b/tika-serialization/src/test/java/org/apache/tika/config/loader/EngineLoaderTest.java
new file mode 100644
index 0000000000..034083f947
--- /dev/null
+++
b/tika-serialization/src/test/java/org/apache/tika/config/loader/EngineLoaderTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.config.loader;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.parser.inference.EngineRegistry;
+
+public class EngineLoaderTest {
+
+ @TempDir
+ Path tmp;
+
+ private TikaLoader load(String json) throws Exception {
+ Path config = Files.createTempFile(tmp, "tika-config", ".json");
+ Files.writeString(config, json);
+ return TikaLoader.load(config);
+ }
+
+ @Test
+ public void testEnginesLoadByName() throws Exception {
+ TikaLoader loader = load("{ \"engines\": {"
+ + " \"one\": { \"test-engine\": { \"label\": \"first\" } },"
+ + " \"two\": { \"test-engine\": { \"label\": \"second\" } } }
}");
+ EngineRegistry engines = loader.get(EngineRegistry.class);
+ assertEquals("first", ((TestEngine) engines.get("one")).getLabel());
+ assertEquals("second", ((TestEngine) engines.get("two")).getLabel());
+ assertNull(engines.get("three"));
+ assertEquals(2, engines.getEngines().size());
+ }
+
+ @Test
+ public void testAbsentSectionLoadsNothing() throws Exception {
+ assertNull(load("{ \"parsers\": [ { \"default-parser\": {} } ] }")
+ .get(EngineRegistry.class));
+ }
+
+ @Test
+ public void testMisconfigurationsFailLoad() throws Exception {
+ String[] bad = {
+ "{ \"engines\": [ { \"test-engine\": {} } ] }",
+ "{ \"engines\": { \"one\": { \"test-engine\": {},
\"test-engine-2\": {} } } }",
+ "{ \"engines\": { \"one\": {} } }",
+ "{ \"engines\": { \"bad name\": { \"test-engine\": {} } } }",
+ "{ \"engines\": { \"one\": { \"no-such-engine\": {} } } }",
+ "{ \"engines\": { \"one\": { \"test-engine\": { \"nope\": 1 } } }
}",
+ };
+ for (String json : bad) {
+ TikaConfigException e = assertThrows(TikaConfigException.class,
+ () -> load(json).get(EngineRegistry.class), json);
+ assertNotNull(e.getMessage());
+ }
+ }
+}
diff --git
a/tika-serialization/src/test/java/org/apache/tika/config/loader/TestEngine.java
b/tika-serialization/src/test/java/org/apache/tika/config/loader/TestEngine.java
new file mode 100644
index 0000000000..d5559c7b00
--- /dev/null
+++
b/tika-serialization/src/test/java/org/apache/tika/config/loader/TestEngine.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.config.loader;
+
+import org.apache.tika.annotation.TikaComponent;
+import org.apache.tika.parser.inference.Engine;
+
+/** An engine that does nothing but carry a label, for loader tests. */
+@TikaComponent(name = "test-engine", spi = false)
+public class TestEngine implements Engine {
+
+ private String label = "";
+
+ public String getLabel() {
+ return label;
+ }
+
+ public void setLabel(String label) {
+ this.label = label;
+ }
+}
