This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch improve-serialization in repository https://gitbox.apache.org/repos/asf/tika.git
commit 710ba49c7ed19dd48f96201cd427fdb6363e3d8b Author: tallison <[email protected]> AuthorDate: Wed Jun 24 21:03:11 2026 -0400 further improvements to serialization --- .../ROOT/pages/developers/serialization.adoc | 20 +- .../serialization/FetchEmitTupleDeserializer.java | 7 +- .../tika/pipes/core/server/ConnectionHandler.java | 2 +- .../apache/tika/pipes/core/server/PipesServer.java | 4 +- .../tika/pipes/core/server/ServerProtocolIO.java | 17 +- .../core/serialization/JsonFetchEmitTupleTest.java | 16 +- .../WireRestrictedFetchEmitTupleTest.java | 96 ++++++++++ .../config/loader/AbstractSpiComponentLoader.java | 10 +- .../tika/config/loader/ComponentInstantiator.java | 87 --------- .../tika/config/loader/ComponentRegistry.java | 14 -- .../apache/tika/config/loader/DetectorLoader.java | 6 - .../tika/config/loader/EncodingDetectorLoader.java | 9 - .../org/apache/tika/config/loader/TikaLoader.java | 27 +-- .../tika/serialization/ComponentNameResolver.java | 92 +++++---- .../tika/serialization/ParseContextUtils.java | 6 +- .../serdes/ParseContextDeserializer.java | 209 ++++++++++----------- .../serdes/ParseContextSerializer.java | 40 +--- .../WireRestrictedParseContextTest.java | 138 ++++++++++++++ .../apache/tika/server/core/TikaServerProcess.java | 11 ++ .../tika/server/core/resource/AsyncResource.java | 3 + .../tika/server/core/resource/TikaResource.java | 4 +- .../core/TikaServerPipesIntegrationTest.java | 28 +++ .../configs/tika-config-server-basic.json | 7 - .../apache/tika/server/standard/TikaPipesTest.java | 10 +- .../tika/server/standard/TikaResourceTest.java | 26 +++ 25 files changed, 524 insertions(+), 365 deletions(-) diff --git a/docs/modules/ROOT/pages/developers/serialization.adoc b/docs/modules/ROOT/pages/developers/serialization.adoc index a5adc3e0e5..238adf1552 100644 --- a/docs/modules/ROOT/pages/developers/serialization.adoc +++ b/docs/modules/ROOT/pages/developers/serialization.adoc @@ -226,23 +226,9 @@ Benefits: } ---- -=== Typed Section - -For components that need immediate deserialization (not lazy loading): - -[source,json] ----- -{ - "parse-context": { - "typed": { - "handler-config": { - "type": "XML", - "writeLimit": 100000 - } - } - } -} ----- +All entries use this flat, friendly-named form and are resolved lazily: a config +is parsed into its component only when first needed (see `resolveAll`). There is +no separate "immediate" or "typed" form. == Security Model diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleDeserializer.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleDeserializer.java index c842d60aa6..f7ab29295b 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleDeserializer.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleDeserializer.java @@ -36,7 +36,6 @@ import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.tika.metadata.Metadata; import org.apache.tika.parser.ParseContext; @@ -50,7 +49,6 @@ public class FetchEmitTupleDeserializer extends JsonDeserializer<FetchEmitTuple> @Override public FetchEmitTuple deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JacksonException { JsonNode root = jsonParser.readValueAsTree(); - ObjectMapper mapper = (ObjectMapper) jsonParser.getCodec(); String id = readVal(ID, root, null, true); String fetcherId = readVal(FETCHER, root, null, true); @@ -61,7 +59,10 @@ public class FetchEmitTupleDeserializer extends JsonDeserializer<FetchEmitTuple> long fetchRangeEnd = readLong(FETCH_RANGE_END, root, -1l, false); Metadata metadata = readMetadata(root); JsonNode parseContextNode = root.get(PARSE_CONTEXT); - ParseContext parseContext = parseContextNode == null ? new ParseContext() : ParseContextDeserializer.readParseContext(parseContextNode, mapper); + // A FetchEmitTuple is always untrusted wire input (request body, pipes iterator): restrict + // its parseContext so it cannot introduce wire-blocked components (parsers, detectors, ...). + ParseContext parseContext = parseContextNode == null ? new ParseContext() + : ParseContextDeserializer.readParseContext(parseContextNode, true); FetchEmitTuple.ON_PARSE_EXCEPTION onParseException = readOnParseException(root); return new FetchEmitTuple(id, new FetchKey(fetcherId, fetchKey, fetchRangeStart, fetchRangeEnd), diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ConnectionHandler.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ConnectionHandler.java index c6f802e516..5628cbde4d 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ConnectionHandler.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ConnectionHandler.java @@ -163,9 +163,9 @@ public class ConnectionHandler implements Runnable, Closeable { } ParseContext mergedContext = null; try { - ServerProtocolIO.validateFetchEmitTuple(fetchEmitTuple); mergedContext = resources.createMergedParseContext(fetchEmitTuple.getParseContext()); ParseContextUtils.resolveAll(mergedContext, getClass().getClassLoader()); + ServerProtocolIO.validateParseContext(mergedContext); TikaProgressTracker tracker = new TikaProgressTracker(); mergedContext.set(TikaProgressTracker.class, tracker); diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java index 06f1f98ea2..29334b2fd2 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java @@ -367,12 +367,12 @@ public class PipesServer implements AutoCloseable { handleCrash(PipesMessageType.UNSPECIFIED_CRASH, "unknown", e); break; // unreachable after handleCrash/exit, but needed for compilation } - // Validate before merging with global config - ServerProtocolIO.validateFetchEmitTuple(fetchEmitTuple); // Create merged ParseContext: defaults from tika-config + request overrides ParseContext mergedContext = createMergedParseContext(fetchEmitTuple.getParseContext()); // Resolve friendly-named configs in ParseContext to actual objects ParseContextUtils.resolveAll(mergedContext, getClass().getClassLoader()); + // Validate the effective (merged + resolved) context + ServerProtocolIO.validateParseContext(mergedContext); TikaProgressTracker tracker = new TikaProgressTracker(); mergedContext.set(TikaProgressTracker.class, tracker); diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ServerProtocolIO.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ServerProtocolIO.java index 531db0036f..a4d1cf320f 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ServerProtocolIO.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ServerProtocolIO.java @@ -26,7 +26,6 @@ import org.slf4j.LoggerFactory; import org.apache.tika.exception.TikaConfigException; import org.apache.tika.metadata.Metadata; import org.apache.tika.parser.ParseContext; -import org.apache.tika.pipes.api.FetchEmitTuple; import org.apache.tika.pipes.api.ParseMode; import org.apache.tika.pipes.api.PipesResult; import org.apache.tika.pipes.core.extractor.UnpackConfig; @@ -113,19 +112,17 @@ public class ServerProtocolIO { } /** - * Validates that a FetchEmitTuple's configuration is consistent. - * <p> - * If the tuple has an UnpackConfig with an emitter but ParseMode is not UNPACK, - * that's a configuration error. + * Validates a (resolved) ParseContext's configuration. Must be called <em>after</em> + * {@link org.apache.tika.serialization.ParseContextUtils#resolveAll}, since configs are lazy + * and only populated once resolved. */ - public static void validateFetchEmitTuple(FetchEmitTuple fetchEmitTuple) + public static void validateParseContext(ParseContext context) throws TikaConfigException { - ParseContext requestContext = fetchEmitTuple.getParseContext(); - if (requestContext == null) { + if (context == null) { return; } - UnpackConfig unpackConfig = requestContext.get(UnpackConfig.class); - ParseMode parseMode = requestContext.get(ParseMode.class); + UnpackConfig unpackConfig = context.get(UnpackConfig.class); + ParseMode parseMode = context.get(ParseMode.class); // Warn (don't throw) when UnpackConfig has an emitter but ParseMode is not UNPACK. // The global parse-context may include UnpackConfig as a default for UNPACK pipe runs, diff --git a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/JsonFetchEmitTupleTest.java b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/JsonFetchEmitTupleTest.java index b34f65169c..f029c3e4a0 100644 --- a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/JsonFetchEmitTupleTest.java +++ b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/JsonFetchEmitTupleTest.java @@ -17,6 +17,7 @@ package org.apache.tika.pipes.core.serialization; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import java.io.Reader; import java.io.StringReader; @@ -33,6 +34,7 @@ import org.apache.tika.pipes.api.fetcher.FetchKey; import org.apache.tika.pipes.core.extractor.UnpackConfig; import org.apache.tika.sax.BasicContentHandlerFactory; import org.apache.tika.sax.ContentHandlerFactory; +import org.apache.tika.serialization.ParseContextUtils; public class JsonFetchEmitTupleTest { @@ -59,7 +61,16 @@ public class JsonFetchEmitTupleTest { JsonFetchEmitTuple.toJson(t, writer); Reader reader = new StringReader(writer.toString()); FetchEmitTuple deserialized = JsonFetchEmitTuple.fromJson(reader); - assertEquals(t, deserialized); + // Config deserializes lazily now; resolve before comparing the effective context. + ParseContextUtils.resolveAll(deserialized.getParseContext(), + Thread.currentThread().getContextClassLoader()); + assertEquals(t.getId(), deserialized.getId()); + assertEquals(t.getFetchKey(), deserialized.getFetchKey()); + assertEquals(t.getEmitKey(), deserialized.getEmitKey()); + assertEquals(m, deserialized.getMetadata()); + assertEquals(ParseMode.CONCATENATE, deserialized.getParseContext().get(ParseMode.class)); + assertInstanceOf(BasicContentHandlerFactory.class, + deserialized.getParseContext().get(ContentHandlerFactory.class)); } @Test @@ -128,6 +139,9 @@ public class JsonFetchEmitTupleTest { Reader reader = new StringReader(json); FetchEmitTuple deserialized = JsonFetchEmitTuple.fromJson(reader); + // Config deserializes lazily now; resolve before reading the bound values. + ParseContextUtils.resolveAll(deserialized.getParseContext(), + Thread.currentThread().getContextClassLoader()); // Verify ParseMode is preserved assertEquals(ParseMode.UNPACK, deserialized.getParseContext().get(ParseMode.class)); diff --git a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/WireRestrictedFetchEmitTupleTest.java b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/WireRestrictedFetchEmitTupleTest.java new file mode 100644 index 0000000000..1fc697bfb5 --- /dev/null +++ b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/WireRestrictedFetchEmitTupleTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.pipes.core.serialization; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.StringReader; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.pipes.api.FetchEmitTuple; +import org.apache.tika.pipes.api.emitter.EmitKey; +import org.apache.tika.pipes.api.fetcher.FetchKey; + +/** + * End-to-end checks at the actual wire entry points. A FetchEmitTuple is always untrusted request + * input, so its parseContext must not introduce a wire-blocked component. All three entry points + * (/pipes, /async, fork IPC) share {@code FetchEmitTupleDeserializer}, which enforces this. + */ +public class WireRestrictedFetchEmitTupleTest { + + private static final String EXPLOIT_PARSE_CONTEXT = + "\"parse-context\":{\"typed\":{\"external-parser\":{\"config\":{" + + "\"commandLine\":[\"/bin/sh\",\"-c\",\"echo pwned\"]," + + "\"supportedTypes\":[\"text/plain\"]}}}}"; + + private static String tuple(String parseContextField) { + return "{\"id\":\"t\",\"fetcher\":\"f\",\"fetchKey\":\"k\"," + + "\"emitter\":\"e\",\"emitKey\":\"ek\",\"onParseException\":\"skip\"" + + (parseContextField.isEmpty() ? "" : "," + parseContextField) + "}"; + } + + @Test + public void pipesEndpointRejectsParserInjection() { + Exception e = assertThrows(Exception.class, + () -> JsonFetchEmitTuple.fromJson(new StringReader(tuple(EXPLOIT_PARSE_CONTEXT)))); + assertTrue(root(e).contains("may not be supplied via a request parseContext"), + "expected wire-blocked rejection, got: " + root(e)); + } + + @Test + public void asyncEndpointRejectsParserInjection() { + Exception e = assertThrows(Exception.class, + () -> JsonFetchEmitTupleList.fromJson(new StringReader("[" + tuple(EXPLOIT_PARSE_CONTEXT) + "]"))); + assertTrue(root(e).contains("may not be supplied via a request parseContext"), + "expected wire-blocked rejection, got: " + root(e)); + } + + @Test + public void pipesEndpointAllowsSafeParseContext() throws Exception { + String safe = "\"parse-context\":{" + + "\"basic-content-handler-factory\":{\"type\":\"XML\",\"writeLimit\":1000}," + + "\"timeout-limits\":{\"progressTimeoutMillis\":5000,\"totalTaskTimeoutMillis\":60000}}"; + FetchEmitTuple t = JsonFetchEmitTuple.fromJson(new StringReader(tuple(safe))); + assertNotNull(t); + assertTrue(t.getParseContext().hasJsonConfig("timeout-limits")); + assertTrue(t.getParseContext().hasJsonConfig("basic-content-handler-factory")); + } + + @Test + public void ipcRoundTripsSafeTuple() throws Exception { + FetchEmitTuple t = new FetchEmitTuple("t", new FetchKey("f", "k"), + new EmitKey("e", "ek"), new Metadata(), new ParseContext(), + FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP); + byte[] bytes = JsonPipesIpc.toBytes(t); + FetchEmitTuple back = JsonPipesIpc.fromBytes(bytes, FetchEmitTuple.class); + assertEquals(t, back); + } + + private static String root(Throwable t) { + Throwable r = t; + while (r.getCause() != null && r.getCause() != r) { + r = r.getCause(); + } + return String.valueOf(r.getMessage()); + } +} diff --git a/tika-serialization/src/main/java/org/apache/tika/config/loader/AbstractSpiComponentLoader.java b/tika-serialization/src/main/java/org/apache/tika/config/loader/AbstractSpiComponentLoader.java index 191d5b164d..eb0787c711 100644 --- a/tika-serialization/src/main/java/org/apache/tika/config/loader/AbstractSpiComponentLoader.java +++ b/tika-serialization/src/main/java/org/apache/tika/config/loader/AbstractSpiComponentLoader.java @@ -143,8 +143,8 @@ public abstract class AbstractSpiComponentLoader<T> implements ComponentLoader<T // ==================== Abstract methods for subclasses ==================== /** - * Load a single component from config. - * Subclasses can apply decorations (e.g., mime filtering for parsers). + * Load a single component from config. The default instantiates the named component from its + * config; subclasses override to apply decorations (e.g., mime filtering for parsers). * * @param name the component name (friendly name or FQCN) * @param configNode the JSON configuration for this component @@ -152,8 +152,10 @@ public abstract class AbstractSpiComponentLoader<T> implements ComponentLoader<T * @return the loaded component * @throws TikaConfigException if loading fails */ - protected abstract T loadComponent(String name, JsonNode configNode, - LoaderContext context) throws TikaConfigException; + protected T loadComponent(String name, JsonNode configNode, + LoaderContext context) throws TikaConfigException { + return context.instantiate(name, configNode); + } /** * Create the SPI-backed default composite with exclusions. diff --git a/tika-serialization/src/main/java/org/apache/tika/config/loader/ComponentInstantiator.java b/tika-serialization/src/main/java/org/apache/tika/config/loader/ComponentInstantiator.java index 4dcc930353..8a8a3091d7 100644 --- a/tika-serialization/src/main/java/org/apache/tika/config/loader/ComponentInstantiator.java +++ b/tika-serialization/src/main/java/org/apache/tika/config/loader/ComponentInstantiator.java @@ -17,7 +17,6 @@ package org.apache.tika.config.loader; import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; import java.util.HashSet; import java.util.Set; @@ -35,7 +34,6 @@ import org.apache.tika.parser.DefaultParser; import org.apache.tika.parser.Parser; import org.apache.tika.parser.ParserDecorator; import org.apache.tika.serialization.ComponentNameResolver; -import org.apache.tika.utils.ServiceLoaderUtils; /** * Utility class for instantiating Tika components from JSON configuration. @@ -43,62 +41,6 @@ import org.apache.tika.utils.ServiceLoaderUtils; */ public class ComponentInstantiator { - /** - * Instantiates a component with JsonConfig constructor or falls back to zero-arg constructor. - * <p> - * Instantiation strategy: - * <ol> - * <li>Try constructor with JsonConfig parameter</li> - * <li>If not found and JSON config has actual configuration, throw error</li> - * <li>Otherwise fall back to zero-arg constructor via ServiceLoader</li> - * </ol> - * - * @param componentClass the component class to instantiate - * @param jsonConfig the JSON configuration for the component - * @param classLoader the class loader to use - * @param componentTypeName the component type name (e.g., "Detector", "Parser") for error messages - * @param objectMapper the Jackson ObjectMapper for parsing JSON - * @param <T> the component type - * @return the instantiated component - * @throws TikaConfigException if instantiation fails - */ - @SuppressWarnings("unchecked") - public static <T> T instantiate(Class<?> componentClass, - JsonConfig jsonConfig, - ClassLoader classLoader, - String componentTypeName, - ObjectMapper objectMapper) - throws TikaConfigException { - try { - T component; - - // Try constructor with JsonConfig parameter - try { - Constructor<?> constructor = componentClass.getConstructor(JsonConfig.class); - component = (T) constructor.newInstance(jsonConfig); - } catch (NoSuchMethodException e) { - // Check if JSON config has actual configuration - if (hasConfiguration(jsonConfig, objectMapper)) { - throw new TikaConfigException( - componentTypeName + " '" + componentClass.getName() + "' has configuration in JSON, " + - "but does not have a constructor that accepts JsonConfig. " + - "Please add a constructor: public " + componentClass.getSimpleName() + "(JsonConfig jsonConfig)"); - } - // Fall back to zero-arg constructor if no configuration provided - component = (T) ServiceLoaderUtils.newInstance(componentClass, - new org.apache.tika.config.ServiceLoader(classLoader)); - } - - // Call initialize() on Initializable components - initializeIfNeeded(component); - - return component; - } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) { - throw new TikaConfigException("Failed to instantiate " + componentTypeName + ": " + - componentClass.getName(), e); - } - } - /** * Instantiates a component from a JsonNode configuration. * <p> @@ -327,35 +269,6 @@ public class ComponentInstantiator { return cleaned; } - /** - * Checks if the JsonConfig contains actual configuration (non-empty JSON object with fields). - * - * @param jsonConfig the JSON configuration - * @param objectMapper the Jackson ObjectMapper for parsing JSON - * @return true if there's meaningful configuration, false if empty or just "{}" - */ - public static boolean hasConfiguration(JsonConfig jsonConfig, ObjectMapper objectMapper) { - if (jsonConfig == null) { - return false; - } - String json = jsonConfig.json(); - if (json == null || json.trim().isEmpty()) { - return false; - } - // Parse to check if it's an empty object or has actual fields - try { - JsonNode node = objectMapper.readTree(json); - // Check if it's an object and has at least one field - if (node.isObject() && node.size() > 0) { - return true; - } - return false; - } catch (Exception e) { - // If we can't parse it, assume it has configuration to be safe - return true; - } - } - /** * Calls initialize() on the component if it implements Initializable. * diff --git a/tika-serialization/src/main/java/org/apache/tika/config/loader/ComponentRegistry.java b/tika-serialization/src/main/java/org/apache/tika/config/loader/ComponentRegistry.java index 576dc2e88e..5bc17f4fd4 100644 --- a/tika-serialization/src/main/java/org/apache/tika/config/loader/ComponentRegistry.java +++ b/tika-serialization/src/main/java/org/apache/tika/config/loader/ComponentRegistry.java @@ -48,20 +48,6 @@ import org.apache.tika.exception.TikaConfigException; */ public class ComponentRegistry { - /** - * Built-in aliases for external dependencies. - * Maps component names to fully qualified class names. - */ - private static final Map<String, String> BUILTIN_ALIASES = createBuiltinAliases(); - - private static Map<String, String> createBuiltinAliases() { - Map<String, String> aliases = new HashMap<>(); - // UnpackConfig is in tika-pipes-core which can't depend on tika-core for @TikaComponent - aliases.put("unpack-config", - "org.apache.tika.pipes.core.extractor.UnpackConfig"); - return Collections.unmodifiableMap(aliases); - } - private final Map<String, ComponentInfo> components; private final Map<String, String> classNameToFriendlyName; // Reverse lookup by class name private final ClassLoader classLoader; diff --git a/tika-serialization/src/main/java/org/apache/tika/config/loader/DetectorLoader.java b/tika-serialization/src/main/java/org/apache/tika/config/loader/DetectorLoader.java index b8e8868c4c..f2725cc241 100644 --- a/tika-serialization/src/main/java/org/apache/tika/config/loader/DetectorLoader.java +++ b/tika-serialization/src/main/java/org/apache/tika/config/loader/DetectorLoader.java @@ -36,12 +36,6 @@ public class DetectorLoader extends AbstractSpiComponentLoader<Detector> { super("detectors", "default-detector", Detector.class); } - @Override - protected Detector loadComponent(String name, JsonNode configNode, - LoaderContext context) throws TikaConfigException { - return context.instantiate(name, configNode); - } - @Override protected Detector createDefaultComposite(Set<Class<? extends Detector>> exclusions, LoaderContext context) { diff --git a/tika-serialization/src/main/java/org/apache/tika/config/loader/EncodingDetectorLoader.java b/tika-serialization/src/main/java/org/apache/tika/config/loader/EncodingDetectorLoader.java index a5f60f8187..b6f5ab40c1 100644 --- a/tika-serialization/src/main/java/org/apache/tika/config/loader/EncodingDetectorLoader.java +++ b/tika-serialization/src/main/java/org/apache/tika/config/loader/EncodingDetectorLoader.java @@ -19,13 +19,10 @@ package org.apache.tika.config.loader; import java.util.List; import java.util.Set; -import com.fasterxml.jackson.databind.JsonNode; - import org.apache.tika.config.ServiceLoader; import org.apache.tika.detect.CompositeEncodingDetector; import org.apache.tika.detect.DefaultEncodingDetector; import org.apache.tika.detect.EncodingDetector; -import org.apache.tika.exception.TikaConfigException; /** * Loader for encoding detectors with support for SPI fallback via @@ -37,12 +34,6 @@ public class EncodingDetectorLoader extends AbstractSpiComponentLoader<EncodingD super("encoding-detectors", "default-encoding-detector", EncodingDetector.class); } - @Override - protected EncodingDetector loadComponent(String name, JsonNode configNode, - LoaderContext context) throws TikaConfigException { - return context.instantiate(name, configNode); - } - @Override protected EncodingDetector createDefaultComposite( Set<Class<? extends EncodingDetector>> exclusions, LoaderContext context) { diff --git a/tika-serialization/src/main/java/org/apache/tika/config/loader/TikaLoader.java b/tika-serialization/src/main/java/org/apache/tika/config/loader/TikaLoader.java index 7d6433afa5..06673f395f 100644 --- a/tika-serialization/src/main/java/org/apache/tika/config/loader/TikaLoader.java +++ b/tika-serialization/src/main/java/org/apache/tika/config/loader/TikaLoader.java @@ -148,12 +148,7 @@ public class TikaLoader { // Special cached instances that aren't standard components private Parser autoDetectParser; - private Detector detectors; - private EncodingDetector encodingDetectors; - private MetadataFilter metadataFilter; private ContentHandlerFactory contentHandlerFactory; - private Renderer renderers; - private Translator translator; private ConfigLoader configLoader; private GlobalSettings globalSettings; @@ -409,7 +404,7 @@ public class TikaLoader { } try { ParseContext context = - ParseContextDeserializer.readParseContext(parseContextNode, objectMapper); + ParseContextDeserializer.readParseContext(parseContextNode); ParseContextUtils.resolveAll(context, classLoader); return context; } catch (IOException e) { @@ -653,26 +648,6 @@ public class TikaLoader { return component; } - /** - * Gets a component by its JSON field name. - * Components are loaded lazily and cached. - * - * @param jsonField the JSON field name (e.g., "parsers", "detectors") - * @return the loaded component - * @throws TikaConfigException if loading fails - */ - @SuppressWarnings("unchecked") - public <T> T get(String jsonField) throws TikaConfigException { - // Get component config from registry by field name - ComponentConfig<?> componentConfig = ComponentNameResolver.getComponentConfig(jsonField); - if (componentConfig == null) { - throw new IllegalArgumentException("No component registered for field: " + jsonField); - } - - // Delegate to get by class (which handles caching) - return (T) get(componentConfig.getComponentClass()); - } - /** * Load a component using its configuration. * Delegates to custom loader if available, otherwise uses default list-based loading. diff --git a/tika-serialization/src/main/java/org/apache/tika/serialization/ComponentNameResolver.java b/tika-serialization/src/main/java/org/apache/tika/serialization/ComponentNameResolver.java index 0198815344..977dd895a4 100644 --- a/tika-serialization/src/main/java/org/apache/tika/serialization/ComponentNameResolver.java +++ b/tika-serialization/src/main/java/org/apache/tika/serialization/ComponentNameResolver.java @@ -74,10 +74,43 @@ public final class ComponentNameResolver { CONTEXT_KEY_INTERFACES.add(UnpackSelector.class); } + /** + * Subset of {@link #CONTEXT_KEY_INTERFACES} a request-supplied (wire) ParseContext may + * instantiate and bind. Default-deny: a type is listed only if every registered implementation + * is confined to transforming this request's metadata or shaping its output -- no exec, IO, + * network, or control over which other components run. Enforced only on untrusted wire input; + * trusted load-time config via TikaLoader is unrestricted. + */ + private static final Set<Class<?>> WIRE_INSTANTIABLE_CONTEXT_KEYS = new HashSet<>(); + + /** + * Complement of {@link #WIRE_INSTANTIABLE_CONTEXT_KEYS}: context keys a wire ParseContext may + * NOT instantiate (exec/IO/network or parse-graph control). Explicit so the exhaustiveness test + * can assert every context-key interface is classified as exactly one of allowed/blocked. + */ + private static final Set<Class<?>> WIRE_BLOCKED_CONTEXT_KEYS = new HashSet<>(); + + static { + // Allowed: bounded to this request's metadata/output; no exec or IO. + WIRE_INSTANTIABLE_CONTEXT_KEYS.add(MetadataFilter.class); + WIRE_INSTANTIABLE_CONTEXT_KEYS.add(ContentHandlerFactory.class); + WIRE_INSTANTIABLE_CONTEXT_KEYS.add(ContentHandlerDecoratorFactory.class); + WIRE_INSTANTIABLE_CONTEXT_KEYS.add(DigesterFactory.class); + WIRE_INSTANTIABLE_CONTEXT_KEYS.add(MetadataWriteLimiterFactory.class); + WIRE_INSTANTIABLE_CONTEXT_KEYS.add(UnpackSelector.class); + + // Blocked: exec / IO / network / parse-graph control. + WIRE_BLOCKED_CONTEXT_KEYS.add(Parser.class); + WIRE_BLOCKED_CONTEXT_KEYS.add(Detector.class); + WIRE_BLOCKED_CONTEXT_KEYS.add(EncodingDetector.class); + WIRE_BLOCKED_CONTEXT_KEYS.add(Renderer.class); + WIRE_BLOCKED_CONTEXT_KEYS.add(Translator.class); + WIRE_BLOCKED_CONTEXT_KEYS.add(EmbeddedDocumentExtractorFactory.class); + } + private static final Map<String, ComponentRegistry> REGISTRIES = new ConcurrentHashMap<>(); - // Component configuration storage (keyed by JSON field name and by component class) - private static final Map<String, ComponentConfig<?>> FIELD_TO_CONFIG = new ConcurrentHashMap<>(); + // Component configuration storage (keyed by component class) private static final Map<Class<?>, ComponentConfig<?>> CLASS_TO_CONFIG = new ConcurrentHashMap<>(); private ComponentNameResolver() { @@ -238,20 +271,9 @@ public final class ComponentNameResolver { * @param config the component configuration */ public static <T> void registerComponentConfig(ComponentConfig<T> config) { - FIELD_TO_CONFIG.put(config.getJsonField(), config); CLASS_TO_CONFIG.put(config.getComponentClass(), config); } - /** - * Gets component configuration by JSON field name. - * - * @param jsonField the JSON field name (e.g., "parsers") - * @return the component config, or null if not registered - */ - public static ComponentConfig<?> getComponentConfig(String jsonField) { - return FIELD_TO_CONFIG.get(jsonField); - } - /** * Gets component configuration by component class. * @@ -263,36 +285,40 @@ public final class ComponentNameResolver { return (ComponentConfig<T>) CLASS_TO_CONFIG.get(componentClass); } - /** - * Checks if a component config is registered for the given JSON field. - */ - public static boolean hasComponentConfig(String jsonField) { - return FIELD_TO_CONFIG.containsKey(jsonField); - } + // ==================== Context Key Resolution Methods ==================== /** - * Checks if a component config is registered for the given class. + * Returns the set of interfaces that use compact format serialization. + * + * @return unmodifiable set of context key interfaces */ - public static boolean hasComponentConfig(Class<?> componentClass) { - return CLASS_TO_CONFIG.containsKey(componentClass); + public static Set<Class<?>> getContextKeyInterfaces() { + return Collections.unmodifiableSet(CONTEXT_KEY_INTERFACES); } - /** - * Gets all registered component JSON field names. - */ - public static Set<String> getComponentFields() { - return Collections.unmodifiableSet(FIELD_TO_CONFIG.keySet()); + /** Wire-instantiable context-key interfaces; see {@link #WIRE_INSTANTIABLE_CONTEXT_KEYS}. */ + public static Set<Class<?>> getWireInstantiableContextKeys() { + return Collections.unmodifiableSet(WIRE_INSTANTIABLE_CONTEXT_KEYS); } - // ==================== Context Key Resolution Methods ==================== + /** True if a wire ParseContext may bind this context-key type (default-deny). */ + public static boolean isWireInstantiable(Class<?> contextKey) { + return WIRE_INSTANTIABLE_CONTEXT_KEYS.contains(contextKey); + } /** - * Returns the set of interfaces that use compact format serialization. - * - * @return unmodifiable set of context key interfaces + * True if a wire ParseContext must NOT instantiate this context-key type. Fail-closed: any + * context-key interface not on the wire allowlist is blocked, so a newly-added one is refused + * until consciously allow-listed. Non-component keys (plain config DTOs) are never blocked. */ - public static Set<Class<?>> getContextKeyInterfaces() { - return Collections.unmodifiableSet(CONTEXT_KEY_INTERFACES); + public static boolean isWireBlocked(Class<?> contextKey) { + return CONTEXT_KEY_INTERFACES.contains(contextKey) + && !WIRE_INSTANTIABLE_CONTEXT_KEYS.contains(contextKey); + } + + /** Explicitly wire-blocked context-key interfaces; complement of the wire allowlist. */ + public static Set<Class<?>> getWireBlockedContextKeys() { + return Collections.unmodifiableSet(WIRE_BLOCKED_CONTEXT_KEYS); } /** diff --git a/tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextUtils.java b/tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextUtils.java index 252af2396b..28514295ea 100644 --- a/tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextUtils.java +++ b/tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextUtils.java @@ -126,8 +126,10 @@ public class ParseContextUtils { // Try to find this friendly name in any registered component registry var optionalInfo = ComponentNameResolver.getComponentInfo(friendlyName); if (optionalInfo.isEmpty()) { - // Not a registered component - that's okay, might be used for something else - LOG.debug("'{}' not found in any component registry, skipping", friendlyName); + // Not a registered component -- ignored (not applied). WARN so a typo'd config key + // is visible rather than silently dropped. + LOG.warn("Ignoring unrecognized parse-context entry '{}' (not a registered " + + "component); check for a typo", friendlyName); continue; } diff --git a/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextDeserializer.java b/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextDeserializer.java index 3e526f7b88..ab8df0bd0d 100644 --- a/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextDeserializer.java +++ b/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextDeserializer.java @@ -17,7 +17,6 @@ package org.apache.tika.serialization.serdes; import static org.apache.tika.serialization.serdes.ParseContextSerializer.PARSE_CONTEXT; -import static org.apache.tika.serialization.serdes.ParseContextSerializer.TYPED; import java.io.IOException; import java.util.HashMap; @@ -39,28 +38,29 @@ import org.apache.tika.parser.ParseContext; import org.apache.tika.serialization.ComponentNameResolver; /** - * Deserializes ParseContext from JSON. - * <p> - * Handles two types of entries: - * <ul> - * <li>"typed" section: Deserialized directly to typed objects in the context map</li> - * <li>Other entries: Stored as JSON configs for lazy resolution</li> - * </ul> - * <p> - * Example input: - * <pre> - * { - * "typed": { - * "handler-config": {"type": "XML", "parseMode": "RMETA"} - * }, - * "metadata-filters": ["mock-upper-case-filter"] - * } - * </pre> + * Deserializes ParseContext from JSON. Every entry is stored as a JSON config and resolved lazily + * by {@link org.apache.tika.serialization.ParseContextUtils#resolveAll}; nothing is constructed + * here. Example: {@code {"basic-content-handler-factory":{"type":"XML"},"metadata-filters":[...]}} */ public class ParseContextDeserializer extends JsonDeserializer<ParseContext> { private static final Logger LOG = LoggerFactory.getLogger(ParseContextDeserializer.class); + /** + * When true, refuses context-key types blocked from untrusted wire input + * (see {@link ComponentNameResolver#isWireBlocked(Class)}). Use for request bodies (e.g. + * tika-server /pipes); the default (false) is for trusted load-time config via TikaLoader. + */ + private final boolean restricted; + + public ParseContextDeserializer() { + this(false); + } + + public ParseContextDeserializer(boolean restricted) { + this.restricted = restricted; + } + private static ObjectMapper plainMapper() { return TikaObjectMapperFactory.getPlainMapper(); } @@ -69,25 +69,25 @@ public class ParseContextDeserializer extends JsonDeserializer<ParseContext> { public ParseContext deserialize(JsonParser jsonParser, DeserializationContext ctxt) throws IOException { JsonNode root = jsonParser.readValueAsTree(); - return readParseContext(root, (ObjectMapper) jsonParser.getCodec()); + return readParseContext(root, restricted); } /** - * Deserializes a ParseContext from a JsonNode. - * <p> - * The "typed" section is deserialized directly to typed objects in the context map. - * All other fields are stored as JSON config strings for lazy resolution. - * <p> - * Duplicate detection is performed within a single document: if multiple entries - * resolve to the same context key (e.g., both "bouncy-castle-digester" and - * "commons-digester" resolve to DigesterFactory), an IOException is thrown. - * - * @param jsonNode the JSON node containing the ParseContext data - * @param mapper the ObjectMapper for deserializing typed objects - * @return the deserialized ParseContext - * @throws IOException if deserialization fails or duplicate context keys are detected + * Deserializes a ParseContext from a JsonNode. Every field is stored as a JSON config string + * for lazy resolution; nothing is constructed here. Throws if multiple entries resolve to the + * same context key (e.g. "bouncy-castle-digester" and "commons-digester" -> DigesterFactory). */ - public static ParseContext readParseContext(JsonNode jsonNode, ObjectMapper mapper) + public static ParseContext readParseContext(JsonNode jsonNode) + throws IOException { + return readParseContext(jsonNode, false); + } + + /** + * As {@link #readParseContext(JsonNode)}, but when {@code restricted} is true, refuses any + * reference to a context-key type that is blocked from untrusted wire input + * (see {@link ComponentNameResolver#isWireBlocked(Class)}). + */ + public static ParseContext readParseContext(JsonNode jsonNode, boolean restricted) throws IOException { // Handle optional wrapper: { "parse-context": {...} } JsonNode contextNode = jsonNode.get(PARSE_CONTEXT); @@ -101,27 +101,22 @@ public class ParseContextDeserializer extends JsonDeserializer<ParseContext> { return parseContext; } - // Track context keys to detect duplicates within this document - // Maps contextKey -> friendlyName for error messages + // Untrusted wire input: refuse any blocked component anywhere in the tree before anything + // is constructed. + if (restricted) { + assertNoBlockedComponents(contextNode); + } + + // contextKey -> friendlyName, to detect duplicates within this document. Map<Class<?>, String> seenContextKeys = new HashMap<>(); Iterator<String> fieldNames = contextNode.fieldNames(); while (fieldNames.hasNext()) { String name = fieldNames.next(); JsonNode value = contextNode.get(name); - - if (TYPED.equals(name)) { - // Deserialize typed objects directly to context map - deserializeTypedObjects(value, parseContext, mapper, seenContextKeys); - } else { - // Check for duplicate context key before storing - checkForDuplicateContextKey(name, seenContextKeys); - - // Store as JSON config for lazy resolution - // Use plain JSON mapper since the main mapper may be binary (Smile) - String json = plainMapper().writeValueAsString(value); - parseContext.setJsonConfig(name, json); - } + checkForDuplicateContextKey(name, seenContextKeys); + // Store as a lazy JSON config (plain mapper: the main one may be binary/Smile). + parseContext.setJsonConfig(name, plainMapper().writeValueAsString(value)); } return parseContext; @@ -167,73 +162,73 @@ public class ParseContextDeserializer extends JsonDeserializer<ParseContext> { } /** - * Deserializes the "typed" section into typed objects in the context map. - * - * @param typedNode the JSON node containing typed objects - * @param parseContext the ParseContext to add objects to - * @param mapper the ObjectMapper for deserializing - * @param seenContextKeys map tracking context keys to their friendly names (for duplicate detection) - * @throws IOException if deserialization fails or duplicate context keys are detected + * Refuses any wire-blocked component a request must not be able to bind. Deserialization is + * lazy, so this gate runs before {@code resolveAll} would instantiate anything. It is + * position-aware: a top-level flat key for a self-configuring component (e.g. + * {@code {"pdf-parser":{...}}}) is an inert per-request config that resolveAll skips, so it is + * allowed and its opaque config subtree is not scanned; every other key is checked and its + * subtree scanned, refusing any blocked object key at any depth. Bare-string references are not + * scanned -- they name a defaults instance with no config. Running before resolution also + * covers ExternalParser, which execs from getSupportedTypes(). */ - @SuppressWarnings("unchecked") - private static void deserializeTypedObjects(JsonNode typedNode, ParseContext parseContext, - ObjectMapper mapper, - Map<Class<?>, String> seenContextKeys) throws IOException { - if (!typedNode.isObject()) { + private static void assertNoBlockedComponents(JsonNode contextNode) throws IOException { + if (contextNode == null || !contextNode.isObject()) { return; } - - Iterator<String> fieldNames = typedNode.fieldNames(); - while (fieldNames.hasNext()) { - String componentName = fieldNames.next(); - JsonNode configNode = typedNode.get(componentName); - - Class<?> configClass = null; - Class<?> contextKeyClass = null; - - // First, try component registry lookup (for friendly names like "pdf-parser-config") - Optional<ComponentInfo> infoOpt = ComponentNameResolver.getComponentInfo(componentName); - if (infoOpt.isPresent()) { - ComponentInfo info = infoOpt.get(); - configClass = info.componentClass(); - contextKeyClass = info.contextKey(); - } - - // If not found in registry, reject — components must be registered - if (configClass == null) { - throw new IOException("Unknown typed component '" + componentName + "'. " + - "Components must be registered via @TikaComponent annotation or .idx file."); + Iterator<Map.Entry<String, JsonNode>> fields = contextNode.fields(); + while (fields.hasNext()) { + Map.Entry<String, JsonNode> e = fields.next(); + String name = e.getKey(); + if (isSelfConfiguring(name)) { + // Inert per-request config for a self-configuring component; resolveAll skips it. + continue; } + assertNameAllowed(name); + scanForBlocked(e.getValue()); + } + } - // Determine context key: explicit > interface detection > class itself - Class<?> parseContextKey = contextKeyClass; - if (parseContextKey == null) { - parseContextKey = ComponentNameResolver.findContextKeyInterface(configClass); + /** + * Recursively refuses blocked-component object keys within a scanned subtree. Only object keys + * are checked; bare strings are inert defaults (see {@link #assertNoBlockedComponents}). + */ + private static void scanForBlocked(JsonNode node) throws IOException { + if (node == null) { + return; + } + if (node.isObject()) { + Iterator<Map.Entry<String, JsonNode>> fields = node.fields(); + while (fields.hasNext()) { + Map.Entry<String, JsonNode> e = fields.next(); + assertNameAllowed(e.getKey()); + scanForBlocked(e.getValue()); } - if (parseContextKey == null) { - parseContextKey = configClass; + } else if (node.isArray()) { + for (JsonNode child : node) { + scanForBlocked(child); } + } + } - // Check for duplicate context key - String existingName = seenContextKeys.get(parseContextKey); - if (existingName != null) { - throw new IOException("Duplicate parse-context entries resolve to the same key " + - parseContextKey.getName() + ": '" + existingName + "' and '" + componentName + "'"); - } - seenContextKeys.put(parseContextKey, componentName); - - // Deserialize and add to context - try { - Object config = mapper.treeToValue(configNode, configClass); - parseContext.set((Class) parseContextKey, config); - LOG.debug("Deserialized typed object '{}' -> {} (contextKey={})", - componentName, configClass.getName(), parseContextKey.getName()); - } catch (Exception e) { - LOG.warn("Failed to deserialize typed component '{}' as {}, storing as JSON config", - componentName, configClass.getName(), e); - // Use plain JSON mapper since main mapper may be binary (Smile) - parseContext.setJsonConfig(componentName, plainMapper().writeValueAsString(configNode)); - } + private static boolean isSelfConfiguring(String name) { + Optional<ComponentInfo> infoOpt = ComponentNameResolver.getComponentInfo(name); + return infoOpt.isPresent() && infoOpt.get().selfConfiguring(); + } + + private static void assertNameAllowed(String name) throws IOException { + Optional<ComponentInfo> infoOpt = ComponentNameResolver.getComponentInfo(name); + if (infoOpt.isEmpty()) { + return; + } + Class<?> contextKey = ComponentNameResolver.determineContextKey(infoOpt.get()); + if (ComponentNameResolver.isWireBlocked(contextKey)) { + throw new IOException(wireBlockedMessage(name, contextKey)); } } + + private static String wireBlockedMessage(String friendlyName, Class<?> contextKey) { + return "Component '" + friendlyName + "' (context key " + contextKey.getName() + + ") may not be supplied via a request parseContext. Components of this type must " + + "be configured server-side in the Tika config, not at request time."; + } } diff --git a/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextSerializer.java b/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextSerializer.java index 3168b4834b..0048a3753c 100644 --- a/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextSerializer.java +++ b/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextSerializer.java @@ -32,25 +32,14 @@ import org.apache.tika.parser.ParseContext; import org.apache.tika.serialization.ComponentNameResolver; /** - * Serializes ParseContext to JSON. - * <p> - * Typed objects from the context map are serialized under a "typed" key. - * JSON configs are serialized at the top level. - * <p> - * Example output: - * <pre> - * { - * "typed": { - * "handler-config": {"type": "XML", "parseMode": "RMETA"} - * }, - * "metadata-filters": ["mock-upper-case-filter"] - * } - * </pre> + * Serializes ParseContext to JSON. Every entry is written flat, by friendly name -- both the live + * objects in the context map (serialized to their config) and the raw JSON configs -- and read back + * as lazy configs constructed by {@code resolveAll}. Example: + * {@code {"basic-content-handler-factory":{"type":"XML"},"metadata-filters":[...]}} */ public class ParseContextSerializer extends JsonSerializer<ParseContext> { public static final String PARSE_CONTEXT = "parse-context"; - public static final String TYPED = "typed"; private static ObjectMapper plainMapper() { return TikaObjectMapperFactory.getPlainMapper(); @@ -61,16 +50,12 @@ public class ParseContextSerializer extends JsonSerializer<ParseContext> { SerializerProvider serializers) throws IOException { gen.writeStartObject(); - // Track which friendly names have been serialized under "typed" - // so we can skip them when serializing jsonConfigs (avoid duplicates) + // Friendly names written from the context map, so we skip them when writing jsonConfigs. Set<String> serializedNames = new HashSet<>(); - // First, serialize typed objects from the context map under "typed" key + // Context-map objects are written flat, by friendly name; read back as lazy jsonConfigs. Map<String, Object> contextMap = parseContext.getContextMap(); - boolean hasTypedObjects = false; - for (Map.Entry<String, Object> entry : contextMap.entrySet()) { - String keyClassName = entry.getKey(); Object value = entry.getValue(); // Skip null values @@ -87,11 +72,6 @@ public class ParseContextSerializer extends JsonSerializer<ParseContext> { "@TikaComponent annotation or .idx file to be serializable."); } - if (!hasTypedObjects) { - gen.writeFieldName(TYPED); - gen.writeStartObject(); - hasTypedObjects = true; - } gen.writeFieldName(keyName); // Use writeTree instead of writeRawValue for binary format support (e.g., Smile) // and stricter validation (fails early if value can't be serialized) @@ -101,16 +81,10 @@ public class ParseContextSerializer extends JsonSerializer<ParseContext> { serializedNames.add(keyName); } - if (hasTypedObjects) { - gen.writeEndObject(); - } - - // Then, serialize JSON configs at the top level - // Skip entries that were already serialized under "typed" (they've been resolved) + // Then, serialize JSON configs at the top level (skip any already written above) Map<String, JsonConfig> jsonConfigs = parseContext.getJsonConfigs(); for (Map.Entry<String, JsonConfig> entry : jsonConfigs.entrySet()) { if (serializedNames.contains(entry.getKey())) { - // Already serialized under "typed", skip to avoid duplicate continue; } gen.writeFieldName(entry.getKey()); diff --git a/tika-serialization/src/test/java/org/apache/tika/serialization/WireRestrictedParseContextTest.java b/tika-serialization/src/test/java/org/apache/tika/serialization/WireRestrictedParseContextTest.java new file mode 100644 index 0000000000..fc0d2bef9a --- /dev/null +++ b/tika-serialization/src/test/java/org/apache/tika/serialization/WireRestrictedParseContextTest.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.serialization; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.HashSet; +import java.util.Set; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import org.junit.jupiter.api.Test; + +import org.apache.tika.config.loader.TikaObjectMapperFactory; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.serialization.serdes.ParseContextDeserializer; + +/** + * A request-supplied (wire) ParseContext may <em>configure</em> a parse but never + * <em>introduce</em> an executable/IO component (parser, detector, renderer, ...). These tests pin + * that boundary at the deserializer. + */ +public class WireRestrictedParseContextTest { + + /** Fresh mapper with the restricted ParseContext deserializer (untrusted/wire mode). */ + private ObjectMapper restrictedMapper() { + ObjectMapper mapper = TikaObjectMapperFactory.createMapper(); + SimpleModule module = new SimpleModule(); + module.addDeserializer(ParseContext.class, new ParseContextDeserializer(true)); + mapper.registerModule(module); + return mapper; + } + + @Test + public void everyContextKeyInterfaceIsClassifiedExactlyOnce() { + Set<Class<?>> all = ComponentNameResolver.getContextKeyInterfaces(); + Set<Class<?>> allowed = ComponentNameResolver.getWireInstantiableContextKeys(); + Set<Class<?>> blocked = ComponentNameResolver.getWireBlockedContextKeys(); + for (Class<?> c : all) { + assertTrue(allowed.contains(c) ^ blocked.contains(c), + c.getName() + " must be classified as exactly one of wire-allowed or wire-blocked"); + } + Set<Class<?>> union = new HashSet<>(allowed); + union.addAll(blocked); + assertEquals(all, union, "wire-allowed union wire-blocked must equal CONTEXT_KEY_INTERFACES"); + } + + @Test + public void restrictedRejectsTypedParserInjection() { + // The historical RCE payload (external-parser under a "typed" wrapper); the nested scan + // refuses it regardless of the wrapper key. + String json = "{\"typed\":{\"external-parser\":{\"config\":{" + + "\"commandLine\":[\"/bin/sh\",\"-c\",\"echo pwned\"]," + + "\"supportedTypes\":[\"text/plain\"]}}}}"; + Exception e = assertThrows(Exception.class, + () -> restrictedMapper().readValue(json, ParseContext.class)); + assertTrue(rootMessage(e).contains("may not be supplied via a request parseContext"), + "expected wire-blocked rejection, got: " + rootMessage(e)); + } + + @Test + public void restrictedAllowsFlatSelfConfiguringParserConfig() throws Exception { + // Per-request tuning of an already-loaded self-configuring parser is config, not + // instantiation: stored as an inert jsonConfig, never bound, so it must be allowed + // (mirrors the real pdf-parser sortByPosition / OCR-strategy use case). + String json = "{\"configurable-test-parser\":{\"maxItems\":5}}"; + ParseContext ctx = restrictedMapper().readValue(json, ParseContext.class); + assertTrue(ctx.hasJsonConfig("configurable-test-parser")); + } + + @Test + public void restrictedRejectsNestedParserObject() { + // A blocked parser nested inside an allowed component's config (compact object form). + String json = "{\"basic-content-handler-factory\":{\"nested\":{\"external-parser\":{" + + "\"config\":{\"commandLine\":[\"/bin/sh\",\"-c\",\"echo pwned\"]}}}}}"; + Exception e = assertThrows(Exception.class, + () -> restrictedMapper().readValue(json, ParseContext.class)); + assertTrue(rootMessage(e).contains("may not be supplied via a request parseContext"), + "expected nested wire-blocked rejection, got: " + rootMessage(e)); + } + + @Test + public void keysOnlyScanLetsInertBareStringReferencesThrough() { + // A bare-string reference (compact "defaults" form, no config) is deliberately NOT scanned: + // it cannot carry a commandLine and is never bound-and-invoked. Pins that decision so we + // don't regress to scanning arbitrary string values (false positives). + String json = "{\"metadata-filters\":[\"external-parser\"]}"; + assertDoesNotThrow(() -> restrictedMapper().readValue(json, ParseContext.class)); + } + + @Test + public void restrictedRejectsParserSmuggledInArrayConfig() { + // Nesting a parser inside an array config (e.g. metadata-filters) must also be refused. + String json = "{\"metadata-filters\":[{\"external-parser\":{}}]}"; + Exception e = assertThrows(Exception.class, + () -> restrictedMapper().readValue(json, ParseContext.class)); + assertTrue(rootMessage(e).contains("may not be supplied via a request parseContext"), + "expected array-nested wire-blocked rejection, got: " + rootMessage(e)); + } + + @Test + public void restrictedAllowsSafeConfig() throws Exception { + // Allowed component (metadata filter) + bounded config DTOs (handler, timeout) must pass. + String json = "{\"metadata-filters\":[\"mock-upper-case-filter\"]," + + "\"basic-content-handler-factory\":{\"type\":\"XML\",\"writeLimit\":1000}," + + "\"timeout-limits\":{\"progressTimeoutMillis\":5000,\"totalTaskTimeoutMillis\":60000}}"; + ParseContext ctx = restrictedMapper().readValue(json, ParseContext.class); + assertNotNull(ctx); + assertTrue(ctx.hasJsonConfig("metadata-filters")); + assertTrue(ctx.hasJsonConfig("timeout-limits")); + } + + private static String rootMessage(Throwable t) { + Throwable r = t; + while (r.getCause() != null && r.getCause() != r) { + r = r.getCause(); + } + return String.valueOf(r.getMessage()); + } +} diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java index 1d1cddfacb..6814cf8330 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java @@ -63,6 +63,7 @@ import org.apache.tika.config.ServiceLoader; import org.apache.tika.config.TikaExtras; import org.apache.tika.config.loader.TikaJsonConfig; import org.apache.tika.config.loader.TikaLoader; +import org.apache.tika.exception.TikaConfigException; import org.apache.tika.exception.TikaException; import org.apache.tika.pipes.core.EmitStrategy; import org.apache.tika.pipes.core.EmitStrategyConfig; @@ -393,6 +394,16 @@ public class TikaServerProcess { } } + // /pipes and /async fork processes and read/write via fetchers/emitters: never expose them + // unless the operator opted into unsecure features, even when explicitly listed. (The + // zero-endpoints branch only enables them when unsecure is on, so this won't false-fire.) + if ((addPipesResource || addAsyncResource) && !tikaServerConfig.isEnableUnsecureFeatures()) { + throw new TikaConfigException("The pipes/async endpoints require " + + "<enableUnsecureFeatures>true</enableUnsecureFeatures> in the server config: " + + "they fork processes and read/write via configured fetchers and emitters, so " + + "they are disabled by default even when explicitly listed as endpoints."); + } + if (addAsyncResource) { final AsyncResource localAsyncResource = new AsyncResource(tikaServerConfig.getConfigPath()); Runtime diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncResource.java index 4a2efcf909..4c0b24f353 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncResource.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncResource.java @@ -51,6 +51,7 @@ import org.apache.tika.pipes.core.emitter.EmitterManager; import org.apache.tika.pipes.core.extractor.UnpackConfig; import org.apache.tika.pipes.core.serialization.JsonFetchEmitTupleList; import org.apache.tika.plugins.TikaPluginManager; +import org.apache.tika.serialization.ParseContextUtils; @Path("/async") public class AsyncResource { @@ -113,6 +114,8 @@ public class AsyncResource { .getEmitterId()); } ParseContext parseContext = t.getParseContext(); + // Configs are lazy; resolve before reading UnpackConfig for the bytes-emitter check. + ParseContextUtils.resolveAll(parseContext, getClass().getClassLoader()); UnpackConfig unpackConfig = parseContext.get(UnpackConfig.class); if (unpackConfig != null && !StringUtils.isAllBlank(unpackConfig.getEmitter())) { String bytesEmitter = unpackConfig.getEmitter(); diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java index a0ea80b80e..b48cfcfadd 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java @@ -173,8 +173,8 @@ public class TikaResource { public static void mergeParseContextFromConfig(String configJson, ParseContext context) throws IOException, TikaConfigException { ObjectMapper mapper = new ObjectMapper(); JsonNode root = mapper.readTree(configJson); - // Use root directly - the JSON should contain parser configs at the top level - ParseContext configuredContext = ParseContextDeserializer.readParseContext(root, mapper); + // Request-supplied config: restrict so it cannot bind wire-blocked components. + ParseContext configuredContext = ParseContextDeserializer.readParseContext(root, true); ParseContextUtils.resolveAll(configuredContext, Thread.currentThread().getContextClassLoader()); // Copy resolved context entries for (Map.Entry<String, Object> entry : configuredContext.getContextMap().entrySet()) { diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerPipesIntegrationTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerPipesIntegrationTest.java index f4a4520d67..6234fe4803 100644 --- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerPipesIntegrationTest.java +++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerPipesIntegrationTest.java @@ -260,6 +260,34 @@ public class TikaServerPipesIntegrationTest extends IntegrationTestBase { return null; } + @Test + public void testPipesRejectsParserInjection() throws Exception { + startProcess(new String[]{ + "-config", ProcessUtils.escapeCommandLine(TIKA_CONFIG + .toAbsolutePath() + .toString())}); + awaitServerStartup(); + // A request whose parseContext binds a process-spawning ExternalParser must be refused at + // deserialization, before any fork/parse -- so the request fails and nothing is emitted. + String malicious = "{\"id\":\"hello_world.xml\"," + + "\"fetcher\":\"" + CXFTestBase.FETCHER_ID + "\"," + + "\"fetchKey\":\"hello_world.xml\"," + + "\"emitter\":\"" + CXFTestBase.EMITTER_JSON_ID + "\"," + + "\"emitKey\":\"\"," + + "\"onParseException\":\"emit\"," + + "\"parse-context\":{\"typed\":{\"external-parser\":{\"config\":{" + + "\"commandLine\":[\"/bin/sh\",\"-c\",\"echo pwned\"]," + + "\"supportedTypes\":[\"text/plain\"]}}}}}"; + Response response = WebClient + .create(endPoint + "/pipes") + .accept("application/json") + .post(malicious); + assertTrue(response.getStatus() != 200, + "parser injection must be rejected, got HTTP " + response.getStatus()); + assertFalse(Files.isRegularFile(TEMP_OUTPUT_DIR.resolve("hello_world.xml.json")), + "nothing should be emitted for a rejected request"); + } + private String getJsonString(String fileName, FetchEmitTuple.ON_PARSE_EXCEPTION onParseException) throws IOException { ParseContext parseContext = new ParseContext(); parseContext.set(ContentHandlerFactory.class, diff --git a/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-basic.json b/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-basic.json index a14eda47ae..56251d5459 100644 --- a/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-basic.json +++ b/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-basic.json @@ -1,11 +1,4 @@ { - "fetchers": { - "file-system-fetcher": { - "file-system-fetcher": { - "allowAbsolutePaths": true - } - } - }, "server": { "port": 9999, "endpoints": [ diff --git a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaPipesTest.java b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaPipesTest.java index 3c63d6703a..544382a2bd 100644 --- a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaPipesTest.java +++ b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaPipesTest.java @@ -66,6 +66,7 @@ import org.apache.tika.plugins.TikaPluginManager; import org.apache.tika.sax.BasicContentHandlerFactory; import org.apache.tika.sax.ContentHandlerFactory; import org.apache.tika.serialization.JsonMetadataList; +import org.apache.tika.serialization.ParseContextUtils; import org.apache.tika.server.core.CXFTestBase; import org.apache.tika.server.core.TikaServerParseExceptionMapper; import org.apache.tika.server.core.resource.PipesResource; @@ -282,7 +283,14 @@ public class TikaPipesTest extends CXFTestBase { JsonFetchEmitTuple.toJson(t, writer); FetchEmitTuple deserialized = JsonFetchEmitTuple.fromJson(new StringReader(writer.toString())); - assertEquals(t, deserialized); + // Config deserializes lazily now; resolve before comparing the effective context. + ParseContextUtils.resolveAll(deserialized.getParseContext(), + Thread.currentThread().getContextClassLoader()); + assertEquals(t.getId(), deserialized.getId()); + assertEquals(t.getFetchKey(), deserialized.getFetchKey()); + assertEquals(t.getEmitKey(), deserialized.getEmitKey()); + assertEquals(ParseMode.UNPACK, deserialized.getParseContext().get(ParseMode.class)); + assertEquals(config, deserialized.getParseContext().get(UnpackConfig.class)); String getUrl = endPoint + PIPES_PATH; Response response = WebClient .create(getUrl) diff --git a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaResourceTest.java b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaResourceTest.java index 58ef99ecc6..11b7998805 100644 --- a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaResourceTest.java +++ b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaResourceTest.java @@ -20,6 +20,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.cxf.helpers.HttpHeaderHelper.CONTENT_ENCODING; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.assumeTrue; @@ -45,9 +46,11 @@ import org.apache.cxf.jaxrs.lifecycle.SingletonResourceProvider; import org.junit.jupiter.api.Test; import org.apache.tika.TikaTest; +import org.apache.tika.config.loader.TikaObjectMapperFactory; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.OfficeOpenXMLExtended; import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.parser.ParseContext; import org.apache.tika.parser.ocr.TesseractOCRParser; import org.apache.tika.serialization.JsonMetadata; import org.apache.tika.server.core.CXFTestBase; @@ -432,6 +435,29 @@ public class TikaResourceTest extends CXFTestBase { assertEquals("Left column line 1 Right column line 1 Left colu mn line 2 Right column line 2", responseMsg); } + @Test + public void testConfigRejectsWireBlockedComponent() { + // A request "config" may tune already-loaded parsers but must not introduce a wire-blocked + // component (parser, detector, ...). mergeParseContextFromConfig uses the restricted + // deserializer, which refuses such references before anything is constructed. The + // deserializer is covered exhaustively by WireRestrictedParseContextTest; this pins the + // server-side wiring. Build a Tika mapper first so the component registries are loaded. + TikaObjectMapperFactory.getMapper(); + String configJson = "{\"metadata-filters\":[{\"external-parser\":{}}]}"; + Exception e = assertThrows(Exception.class, + () -> TikaResource.mergeParseContextFromConfig(configJson, new ParseContext())); + assertTrue(rootMessage(e).contains("may not be supplied via a request parseContext"), + "expected wire-blocked rejection, got: " + rootMessage(e)); + } + + private static String rootMessage(Throwable t) { + Throwable r = t; + while (r.getCause() != null && r.getCause() != r) { + r = r.getCause(); + } + return String.valueOf(r.getMessage()); + } + @Test public void testDataIntegrityCheck() throws Exception { // This test requires tesseract to be installed - the validation only happens
