This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 3d4262da49 TIKA-4897: serialize unpaired surrogates in metadata as
U+FFFD (#3184)
3d4262da49 is described below
commit 3d4262da49b0edcd871d59fb36169eb339799e2c
Author: Tim Allison <[email protected]>
AuthorDate: Wed Sep 16 20:35:13 2026 -0400
TIKA-4897: serialize unpaired surrogates in metadata as U+FFFD (#3184)
---
CHANGES.txt | 9 ++
.../java/org/apache/tika/metadata/Metadata.java | 20 +++-
.../java/org/apache/tika/utils/StringUtils.java | 53 +++++++++
.../tika/metadata/MetadataWellFormedTest.java | 96 ++++++++++++++++
.../core/serialization/PipesResultSerializer.java | 3 +-
.../apache/tika/pipes/core/server/PipesWorker.java | 24 ++--
.../tika/pipes/core/server/ServerProtocolIO.java | 47 +++++++-
.../pipes/core/server/ServerProtocolIOTest.java | 127 +++++++++++++++++++++
.../serialization/serdes/MetadataSerializer.java | 1 +
.../serdes/MetadataSerializerSurrogateTest.java | 58 ++++++++++
10 files changed, 416 insertions(+), 22 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 4a41f7d0a1..05fe851477 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,14 @@
Release 4.1.0 - unreleased
+ * Metadata never stores an unpaired UTF-16 surrogate: every write replaces
+ one with U+FFFD (StringUtils.wellFormed), so no reader meets one; the
+ Smile encoder on the pipes channel rejected them and the parent counted
+ a worker crash. tk:content is exempt, since SafeContentHandler already
+ guarantees it. A result the worker cannot encode is reported for its
+ document with its own status (a fetch failure stays a fetch failure)
+ instead of counting as a crash, and that report cannot itself overflow
+ the payload limit (TIKA-4897).
+
* openai-embedding-engine gains "requestParameters" to enable custom
parameters per vendor (TIKA-4896).
diff --git a/tika-core/src/main/java/org/apache/tika/metadata/Metadata.java
b/tika-core/src/main/java/org/apache/tika/metadata/Metadata.java
index d5a9467dbf..f995e42fea 100644
--- a/tika-core/src/main/java/org/apache/tika/metadata/Metadata.java
+++ b/tika-core/src/main/java/org/apache/tika/metadata/Metadata.java
@@ -41,6 +41,7 @@ import
org.apache.tika.metadata.writelimiter.MetadataWriteLimiter;
import org.apache.tika.metadata.writelimiter.MetadataWriteLimiterFactory;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.utils.DateUtils;
+import org.apache.tika.utils.StringUtils;
/**
* A multi-valued metadata container.
@@ -445,7 +446,18 @@ public class Metadata implements Serializable {
* reserved key by name rather than by its {@link Property}.
*/
public void addTrusted(final String name, final String value) {
- writeLimiter.add(name, value, metadata);
+ writeLimiter.add(name, wellFormed(name, value), metadata);
+ }
+
+ /**
+ * Every string value is stored well-formed: a lone surrogate becomes
U+FFFD at the write,
+ * so no reader (a Smile encoder, for one) meets one. {@code tk:content}
is exempt: it came
+ * through {@link org.apache.tika.sax.SafeContentHandler}, which already
did this, and it
+ * is the one value large enough for the scan to cost anything.
+ */
+ private static String wellFormed(String name, String value) {
+ return TikaCoreProperties.TIKA_CONTENT.getName().equals(name) ? value
+ : StringUtils.wellFormed(value);
}
/**
@@ -481,7 +493,9 @@ public class Metadata implements Serializable {
name.length(), MAX_PREFIX_ROUTE_NAME_LENGTH);
return;
}
- String key = prefix.key(name);
+ // a source-derived name can carry a lone surrogate as easily as a
value; a code
+ // constant never does, so only this route pays for the check on the
name
+ String key = prefix.key(StringUtils.wellFormed(name));
if (!metadata.containsKey(key)) {
if (prefixRouteNames >= MAX_PREFIX_ROUTE_NAMES) {
if (!prefixRouteFloodWarned) {
@@ -680,7 +694,7 @@ public class Metadata implements Serializable {
* ({@code tk:}) keys directly. See {@link #addTrusted}.
*/
public void setTrusted(String name, String value) {
- writeLimiter.set(name, value, metadata);
+ writeLimiter.set(name, wellFormed(name, value), metadata);
}
protected void set(String name, String[] values) {
diff --git a/tika-core/src/main/java/org/apache/tika/utils/StringUtils.java
b/tika-core/src/main/java/org/apache/tika/utils/StringUtils.java
index b09963d461..0a37e1a79f 100644
--- a/tika-core/src/main/java/org/apache/tika/utils/StringUtils.java
+++ b/tika-core/src/main/java/org/apache/tika/utils/StringUtils.java
@@ -221,4 +221,57 @@ public class StringUtils {
}
return sb.toString();
}
+
+ /**
+ * The value with every unpaired UTF-16 surrogate replaced by U+FFFD. A
parser can hand back
+ * a lone surrogate (an HTML numeric character reference for one, say); a
UTF-8 or Smile
+ * encoder rejects it, and one bad value must not fail a whole document.
Returns the same
+ * instance when nothing needs replacing.
+ */
+ public static String wellFormed(String value) {
+ if (value == null) {
+ return null;
+ }
+ int n = value.length();
+ StringBuilder out = null;
+ for (int i = 0; i < n; i++) {
+ char c = value.charAt(i);
+ if (!Character.isSurrogate(c)) {
+ if (out != null) {
+ out.append(c);
+ }
+ continue;
+ }
+ boolean paired = Character.isHighSurrogate(c) && i + 1 < n
+ && Character.isLowSurrogate(value.charAt(i + 1));
+ if (paired) {
+ if (out != null) {
+ out.append(c).append(value.charAt(i + 1));
+ }
+ i++;
+ continue;
+ }
+ if (out == null) {
+ out = new StringBuilder(n);
+ out.append(value, 0, i);
+ }
+ out.append('\uFFFD');
+ }
+ return out == null ? value : out.toString();
+ }
+
+ /** {@link #wellFormed(String)} over an array: the same array when nothing
needs replacing. */
+ public static String[] wellFormed(String[] values) {
+ String[] out = null;
+ for (int i = 0; i < values.length; i++) {
+ String w = wellFormed(values[i]);
+ if (w != values[i]) {
+ if (out == null) {
+ out = values.clone();
+ }
+ out[i] = w;
+ }
+ }
+ return out == null ? values : out;
+ }
}
diff --git
a/tika-core/src/test/java/org/apache/tika/metadata/MetadataWellFormedTest.java
b/tika-core/src/test/java/org/apache/tika/metadata/MetadataWellFormedTest.java
new file mode 100644
index 0000000000..e5cf0dbf10
--- /dev/null
+++
b/tika-core/src/test/java/org/apache/tika/metadata/MetadataWellFormedTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.metadata;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.metadata.writelimiter.StandardMetadataLimiterFactory;
+import org.apache.tika.utils.StringUtils;
+
+/**
+ * A lone UTF-16 surrogate never gets stored: every write replaces it with
U+FFFD, so no
+ * reader (a Smile encoder, for one) meets one. tk:content is exempt, since it
came through
+ * SafeContentHandler, which already did this.
+ */
+public class MetadataWellFormedTest {
+
+ private static final String LONE_HIGH = "abc\uDB2Cdef";
+ private static final String LONE_LOW = "\uDC00abc";
+ private static final String PAIR = "a😀b";
+
+ @Test
+ public void testWellFormed() {
+ assertEquals("abc�def", StringUtils.wellFormed(LONE_HIGH));
+ assertEquals("�abc", StringUtils.wellFormed(LONE_LOW));
+ assertEquals("a��b", StringUtils.wellFormed("a\uDB2C\uDB2Cb"));
+ assertEquals("�", StringUtils.wellFormed("\uDB2C"));
+ assertEquals("", StringUtils.wellFormed(""));
+ // a well-formed value comes back as is, not copied; so does an array
of them
+ assertSame(PAIR, StringUtils.wellFormed(PAIR));
+ String[] clean = {"plain", PAIR};
+ assertSame(clean, StringUtils.wellFormed(clean));
+ String[] dirty = {"plain", LONE_HIGH};
+ String[] fixed = StringUtils.wellFormed(dirty);
+ assertArrayEquals(new String[]{"plain", "abc�def"}, fixed);
+ assertEquals(LONE_HIGH, dirty[1], "the input array is not rewritten");
+ }
+
+ @Test
+ public void testEveryWriteStoresWellFormed() {
+ for (Metadata metadata : new Metadata[]{new Metadata(),
+ new Metadata(new
StandardMetadataLimiterFactory().newInstance())}) {
+ metadata.set("single", LONE_HIGH);
+ metadata.add("multi", LONE_LOW);
+ metadata.add("multi", PAIR);
+ metadata.set(TikaCoreProperties.TITLE, LONE_HIGH);
+ metadata.setTrusted("tk:trusted", LONE_LOW);
+ metadata.addTrusted("tk:trusted", LONE_HIGH);
+ metadata.set(TikaCoreProperties.SUBJECT, new String[]{LONE_LOW,
PAIR});
+
+ assertEquals("abc�def", metadata.get("single"));
+ assertArrayEquals(new String[]{"�abc", PAIR},
metadata.getValues("multi"));
+ assertEquals("abc�def", metadata.get(TikaCoreProperties.TITLE));
+ assertArrayEquals(new String[]{"�abc", "abc�def"},
+ metadata.getValues("tk:trusted"));
+ assertArrayEquals(new String[]{"�abc", PAIR},
+ metadata.getValues(TikaCoreProperties.SUBJECT));
+ }
+ }
+
+ /** A source-derived name on the prefix route is sanitized like a value;
the serializer writes it as a field name. */
+ @Test
+ public void testPrefixRouteNameIsWellFormed() {
+ Metadata metadata = new Metadata();
+ metadata.add(Office.USER_DEFINED, "prop" + LONE_HIGH, "v");
+ String[] names = metadata.names();
+ assertEquals(1, names.length);
+ assertEquals(Office.USER_DEFINED.key("propabc\uFFFDdef"), names[0]);
+ assertEquals("v", metadata.get(names[0]));
+ }
+
+ /** tk:content is SafeContentHandler's job and the one value big enough
for the scan to matter. */
+ @Test
+ public void testContentIsExempt() {
+ Metadata metadata = new Metadata();
+ metadata.add(TikaCoreProperties.TIKA_CONTENT, LONE_HIGH);
+ assertEquals(LONE_HIGH, metadata.get(TikaCoreProperties.TIKA_CONTENT));
+ }
+}
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/PipesResultSerializer.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/PipesResultSerializer.java
index ebf01a5a47..b735c47bda 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/PipesResultSerializer.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/PipesResultSerializer.java
@@ -39,7 +39,8 @@ public class PipesResultSerializer extends
JsonSerializer<PipesResult> {
jsonGenerator.writeObjectField(EMIT_DATA, pipesResult.emitData());
}
if (!StringUtils.isBlank(pipesResult.message())) {
- jsonGenerator.writeStringField(MESSAGE, pipesResult.message());
+ // an exception message may quote document text; Smile rejects a
lone surrogate
+ jsonGenerator.writeStringField(MESSAGE,
StringUtils.wellFormed(pipesResult.message()));
}
jsonGenerator.writeEndObject();
}
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java
index 331d70d74e..54122a811e 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java
@@ -440,16 +440,7 @@ class PipesWorker implements Callable<PipesResult> {
List<java.util.Map<String, Object>> metadataMapList = new
java.util.ArrayList<>();
for (Metadata metadata : metadataList) {
- java.util.Map<String, Object> metadataMap = new
java.util.LinkedHashMap<>();
- for (String name : metadata.names()) {
- String[] values = metadata.getValues(name);
- if (values.length == 1) {
- metadataMap.put(name, values[0]);
- } else {
- metadataMap.put(name, values);
- }
- }
- metadataMapList.add(metadataMap);
+ metadataMapList.add(metadataMap(metadata));
}
mapper.writeValue(os, metadataMapList);
}
@@ -498,17 +489,16 @@ class PipesWorker implements Callable<PipesResult> {
ObjectMapper mapper = new ObjectMapper();
// Disable auto-close so we don't close the zip output stream
mapper.configure(com.fasterxml.jackson.core.JsonGenerator.Feature.AUTO_CLOSE_TARGET,
false);
- // Convert metadata to a map for JSON serialization
+ mapper.writeValue(os, metadataMap(metadata));
+ }
+
+ private static java.util.Map<String, Object> metadataMap(Metadata
metadata) {
java.util.Map<String, Object> metadataMap = new
java.util.LinkedHashMap<>();
for (String name : metadata.names()) {
String[] values = metadata.getValues(name);
- if (values.length == 1) {
- metadataMap.put(name, values[0]);
- } else {
- metadataMap.put(name, values);
- }
+ metadataMap.put(name, values.length == 1 ? values[0] : values);
}
- mapper.writeValue(os, metadataMap);
+ return metadataMap;
}
protected ParseDataOrPipesResult parseFromTuple() throws TikaException,
InterruptedException {
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ServerProtocolIO.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ServerProtocolIO.java
index 1e8eb30ee6..dc1a60b108 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ServerProtocolIO.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ServerProtocolIO.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.io.OutputStream;
import java.util.Locale;
+import com.fasterxml.jackson.core.JsonProcessingException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -128,7 +129,16 @@ public class ServerProtocolIO {
} catch (IOException e) {
lastRespSerNanos = System.nanoTime() - serStart;
if (!bos.overflowed()) {
- throw e;
+ if (!(e instanceof JsonProcessingException)) {
+ throw e;
+ }
+ // the result itself cannot be encoded (the buffer is in
memory, so this is
+ // not the pipe): report it for this document rather than let
the parent
+ // count a worker crash and lose the document
+ LOG.warn("result could not be serialized; returning a
status-only result", e);
+ writeStatusOnly(unserializableStatus(pipesResult.status()),
+ "result could not be serialized: " + e.getMessage());
+ return;
}
LOG.warn("Payload exceeded maxIpcPayloadBytes {}; returning
PAYLOAD_LIMIT_EXCEEDED",
maxIpcPayloadBytes);
@@ -204,6 +214,41 @@ public class ServerProtocolIO {
lastIntermediateNanos = -1;
}
+ /**
+ * A success whose payload is lost is a parse failure for the parent; an
emitted status
+ * stays so the parent does not emit again, and a failure keeps its own
status and category.
+ */
+ static PipesResult.RESULT_STATUS
unserializableStatus(PipesResult.RESULT_STATUS status) {
+ if (alreadyEmitted(status) || status.getCategory() !=
PipesResult.CATEGORY.SUCCESS) {
+ return status;
+ }
+ return PipesResult.RESULT_STATUS.PARSE_EXCEPTION_NO_EMIT;
+ }
+
+ /**
+ * A status and message; the same status with a short fixed message when
that overflows
+ * the limit (the status is what the parent's retry and reporting need);
the guaranteed-fit
+ * frame when even that does not fit.
+ */
+ private void writeStatusOnly(PipesResult.RESULT_STATUS status, String
message)
+ throws IOException {
+ for (String m : new String[]{message, "result could not be
serialized"}) {
+ BoundedOutputStream fallbackBos = new
BoundedOutputStream(maxIpcPayloadBytes);
+ try {
+ JsonPipesIpc.toStream(new PipesResult(status, m), fallbackBos);
+ } catch (IOException e) {
+ if (!fallbackBos.overflowed()) {
+ throw e;
+ }
+ continue;
+ }
+ PipesMessage.finished(fallbackBos.toByteArray()).write(output);
+ awaitAck();
+ return;
+ }
+ doWritePayloadLimitExceeded();
+ }
+
/**
* True for statuses whose content the server already emitted. Replacing
one of these
* with a failure status makes the client treat an emitted document as
failed, so a
diff --git
a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/server/ServerProtocolIOTest.java
b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/server/ServerProtocolIOTest.java
index 2b91a2afb0..41458feffa 100644
---
a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/server/ServerProtocolIOTest.java
+++
b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/server/ServerProtocolIOTest.java
@@ -33,6 +33,7 @@ import org.junit.jupiter.api.Test;
import org.apache.tika.config.ExceptionReporting;
import org.apache.tika.config.TimeoutLimits;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.pipes.api.PipesResult;
import org.apache.tika.pipes.core.emitter.EmitDataImpl;
@@ -192,6 +193,132 @@ class ServerProtocolIOTest {
assertEquals(PipesResult.RESULT_STATUS.PAYLOAD_LIMIT_EXCEEDED,
returned.status());
}
+ /**
+ * A lone surrogate in a metadata value (an HTML numeric character
reference in a title,
+ * say) used to fail the Smile encoding of the whole result, which the
server treated as
+ * a crash; Metadata stores U+FFFD in its place, so the value crosses the
pipe. Extracted
+ * text is SafeContentHandler's job and never carries one.
+ */
+ @Test
+ void testLoneSurrogateInMetadataCrossesThePipe() throws Exception {
+ Metadata m = new Metadata();
+ m.set(TikaCoreProperties.TITLE, "Korean : \uDB2C\u0620 : more");
+ PipesResult result = new
PipesResult(PipesResult.RESULT_STATUS.PARSE_SUCCESS,
+ new EmitDataImpl("key", List.of(m)));
+
+ PipesResult returned = exchange(result,
PipesMessage.MAX_PAYLOAD_BYTES);
+
+ assertEquals(PipesResult.RESULT_STATUS.PARSE_SUCCESS,
returned.status());
+ assertEquals("Korean : \uFFFD\u0620 : more",
+
returned.emitData().getMetadataList().get(0).get(TikaCoreProperties.TITLE));
+ }
+
+ /**
+ * A result that cannot be encoded at all is reported for its document,
not as a worker
+ * crash: the parent gets a status-only result naming the failure, and an
already
+ * emitted status is kept so the parent does not emit the document again.
+ */
+ @Test
+ void testUnserializableResultBecomesAStatusOnlyResult() throws Exception {
+ Metadata poison = new Metadata() {
+ @Override
+ public String[] getValues(String name) {
+ throw new IllegalStateException("cannot read this value");
+ }
+ };
+ poison.set("k", "v");
+
+ PipesResult notEmitted = new
PipesResult(PipesResult.RESULT_STATUS.PARSE_SUCCESS,
+ new EmitDataImpl("key", List.of(poison)));
+ PipesResult returned = exchange(notEmitted,
PipesMessage.MAX_PAYLOAD_BYTES);
+ assertEquals(PipesResult.RESULT_STATUS.PARSE_EXCEPTION_NO_EMIT,
returned.status());
+ assertTrue(returned.message().contains("could not be serialized"),
returned.message());
+ assertTrue(returned.message().contains("cannot read this value"),
returned.message());
+
+ PipesResult emitted = new
PipesResult(PipesResult.RESULT_STATUS.EMIT_SUCCESS,
+ new EmitDataImpl("key", List.of(poison)));
+ returned = exchange(emitted, PipesMessage.MAX_PAYLOAD_BYTES);
+ assertEquals(PipesResult.RESULT_STATUS.EMIT_SUCCESS,
returned.status());
+ }
+
+ /**
+ * A failure whose payload cannot be encoded keeps its status and
category: the parent
+ * treats a fetch failure as a fetch failure, not as a parse it can report
on.
+ */
+ @Test
+ void testUnserializableFailureKeepsItsStatus() throws Exception {
+ Metadata poison = new Metadata() {
+ @Override
+ public String[] getValues(String name) {
+ throw new IllegalStateException("cannot read this value");
+ }
+ };
+ poison.set("k", "v");
+ PipesResult fetchFailed = new
PipesResult(PipesResult.RESULT_STATUS.FETCH_EXCEPTION,
+ new EmitDataImpl("key", List.of(poison)), "fetch failed");
+
+ PipesResult returned = exchange(fetchFailed,
PipesMessage.MAX_PAYLOAD_BYTES);
+
+ assertEquals(PipesResult.RESULT_STATUS.FETCH_EXCEPTION,
returned.status());
+ assertTrue(returned.message().contains("could not be serialized"),
returned.message());
+ }
+
+ /** The category rule behind the fallback, over every status. */
+ @Test
+ void testUnserializableStatusRule() {
+ for (PipesResult.RESULT_STATUS status :
PipesResult.RESULT_STATUS.values()) {
+ PipesResult.RESULT_STATUS fallback =
ServerProtocolIO.unserializableStatus(status);
+ boolean emitted = status == PipesResult.RESULT_STATUS.EMIT_SUCCESS
+ || status ==
PipesResult.RESULT_STATUS.EMIT_SUCCESS_PASSBACK
+ || status ==
PipesResult.RESULT_STATUS.EMIT_SUCCESS_PARSE_EXCEPTION;
+ if (emitted || status.getCategory() !=
PipesResult.CATEGORY.SUCCESS) {
+ assertEquals(status, fallback, status.name());
+ } else {
+
assertEquals(PipesResult.RESULT_STATUS.PARSE_EXCEPTION_NO_EMIT, fallback,
status.name());
+ }
+ }
+ }
+
+ /**
+ * When the status-only report does not fit the limit, the status is kept
with a short
+ * fixed message (a fetch failure stays a fetch failure); only when even
that does not
+ * fit does the guaranteed-fit frame go out, never an exception the parent
counts as a crash.
+ */
+ @Test
+ void testUnserializableFallbackThatOverflowsKeepsTheStatus() throws
Exception {
+ String longMessage = "x".repeat(4096);
+ Metadata poison = new Metadata() {
+ @Override
+ public String[] getValues(String name) {
+ throw new IllegalStateException(longMessage);
+ }
+ };
+ poison.set("k", "v");
+ PipesResult fetchFailed = new
PipesResult(PipesResult.RESULT_STATUS.FETCH_EXCEPTION,
+ new EmitDataImpl("key", List.of(poison)), "fetch failed");
+
+ PipesResult returned = exchange(fetchFailed, 256);
+ assertEquals(PipesResult.RESULT_STATUS.FETCH_EXCEPTION,
returned.status(),
+ "the long message is dropped, the status is not");
+ assertEquals("result could not be serialized", returned.message());
+
+ returned = exchange(fetchFailed,
ServerProtocolIO.MIN_FALLBACK_PAYLOAD_BYTES);
+ assertEquals(PipesResult.RESULT_STATUS.PAYLOAD_LIMIT_EXCEEDED,
returned.status(),
+ "nothing but the static frame fits the minimum limit");
+ }
+
+ /** A lone surrogate in an exception message crosses the Smile channel as
U+FFFD. */
+ @Test
+ void testLoneSurrogateInMessageCrossesThePipe() throws Exception {
+ PipesResult result = new
PipesResult(PipesResult.RESULT_STATUS.FETCH_EXCEPTION,
+ "no such file: \uD800.pdf");
+
+ PipesResult returned = exchange(result,
PipesMessage.MAX_PAYLOAD_BYTES);
+
+ assertEquals(PipesResult.RESULT_STATUS.FETCH_EXCEPTION,
returned.status());
+ assertEquals("no such file: \uFFFD.pdf", returned.message());
+ }
+
/**
* Status-only results (no emitData) always pass through unchanged.
*/
diff --git
a/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/MetadataSerializer.java
b/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/MetadataSerializer.java
index 0dd0607159..db67f6e51a 100644
---
a/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/MetadataSerializer.java
+++
b/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/MetadataSerializer.java
@@ -73,4 +73,5 @@ public class MetadataSerializer extends
JsonSerializer<Metadata> {
}
jsonGenerator.writeEndObject();
}
+
}
diff --git
a/tika-serialization/src/test/java/org/apache/tika/serialization/serdes/MetadataSerializerSurrogateTest.java
b/tika-serialization/src/test/java/org/apache/tika/serialization/serdes/MetadataSerializerSurrogateTest.java
new file mode 100644
index 0000000000..d5e3ef6cbf
--- /dev/null
+++
b/tika-serialization/src/test/java/org/apache/tika/serialization/serdes/MetadataSerializerSurrogateTest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.serialization.serdes;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.config.loader.TikaObjectMapperFactory;
+import org.apache.tika.metadata.Metadata;
+
+/**
+ * A parser can hand back a lone surrogate (an HTML numeric character
reference for one, for
+ * instance). The UTF-8 and Smile generators reject it, and before this a
single such value
+ * failed the whole document's serialization, which in the pipes worker was
fatal.
+ */
+public class MetadataSerializerSurrogateTest {
+
+ private static final String LONE_HIGH = "abc\uDB2Cdef";
+ private static final String LONE_LOW = "\uDC00abc";
+ private static final String PAIR = "a😀b";
+
+ @Test
+ public void testLoneSurrogatesSurviveUtf8AndSmile() throws Exception {
+ Metadata metadata = new Metadata();
+ metadata.set("single", LONE_HIGH);
+ metadata.add("multi", LONE_LOW);
+ metadata.add("multi", PAIR);
+ metadata.set("pair", PAIR);
+
+ for (ObjectMapper mapper : new ObjectMapper[] {
+ TikaObjectMapperFactory.createMapper(),
+ TikaObjectMapperFactory.createMapper(new SmileFactory())}) {
+ byte[] bytes = mapper.writeValueAsBytes(metadata);
+ Metadata back = mapper.readValue(bytes, Metadata.class);
+ assertEquals("abc\uFFFDdef", back.get("single"));
+ assertArrayEquals(new String[] {"\uFFFDabc", PAIR},
back.getValues("multi"));
+ assertEquals(PAIR, back.get("pair"));
+ }
+ }
+}
