This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch metadata-explicit-trusted-writes in repository https://gitbox.apache.org/repos/asf/tika.git
commit b0fe3b6c30ff3a8c022f88f15491bd7e0852eca4 Author: tallison <[email protected]> AuthorDate: Wed Jul 29 21:42:45 2026 -0400 clean up metadata safe --- .../java/org/apache/tika/metadata/Metadata.java | 48 ++++++++++------------ .../filter/CaptureGroupMetadataFilter.java | 3 +- .../metadata/filter/FieldNameMappingFilter.java | 5 ++- .../metadata/filter/GeoPointMetadataFilter.java | 3 +- .../tika/metadata/filter/MetadataFilter.java | 19 +++------ .../metadata/MetadataInternalKeyGuardTest.java | 16 ++++++-- .../org/apache/tika/inference/ChunkSerializer.java | 8 +--- .../apache/tika/parser/pdf/AbstractPDF2XHTML.java | 8 +--- .../apache/tika/pipes/core/server/EmitHandler.java | 16 +++----- .../filter/AttachmentCountingListFilter.java | 2 +- .../tika/metadata/filter/MockUpperCaseFilter.java | 2 +- .../apache/tika/pipes/emitter/es/ESClientTest.java | 4 +- .../filter/AttachmentCountingListFilter.java | 2 +- .../tika/metadata/filter/JsonConfigOnlyFilter.java | 2 +- .../tika/metadata/filter/MockUpperCaseFilter.java | 2 +- .../CustomClassSerializationTest.java | 2 +- 16 files changed, 59 insertions(+), 83 deletions(-) diff --git a/tika-core/src/main/java/org/apache/tika/metadata/Metadata.java b/tika-core/src/main/java/org/apache/tika/metadata/Metadata.java index 534ceaaee3..1f20a5ce09 100644 --- a/tika-core/src/main/java/org/apache/tika/metadata/Metadata.java +++ b/tika-core/src/main/java/org/apache/tika/metadata/Metadata.java @@ -100,7 +100,6 @@ public class Metadata private MetadataWriteLimiter writeLimiter = ACCEPT_ALL; - private transient boolean trusted; /** * Constructs a new, empty metadata. */ @@ -312,30 +311,22 @@ public class Metadata if (blockReservedKeyWrite(name)) { return; } - addUnchecked(name, value); - } - - /** Trusted add, bypassing the reserved-key guard. */ - private void addUnchecked(final String name, final String value) { - writeLimiter.add(name, value, metadata); + addTrusted(name, value); } /** - * Mark this Metadata as a trusted transformation target (e.g. a metadata filter), letting - * String writes reach reserved Tika-native ({@code tk:}) keys. Reset when the transformation - * is done. + * Trusted add, bypassing the reserved-key guard: writes reach reserved Tika-native + * ({@code tk:}) keys directly. For internal/known-trusted writers (metadata filters, + * clone/merge/deserialize, emit-time enrichment) that legitimately need to assert a + * reserved key by name rather than by its {@link Property}. */ - public void setTrusted(boolean trusted) { - this.trusted = trusted; - } - - public boolean isTrusted() { - return trusted; + public void addTrusted(final String name, final String value) { + writeLimiter.add(name, value, metadata); } - /** Drop String writes to reserved Tika-native keys unless trusted; use their Property. */ + /** Drop String writes to reserved Tika-native keys; use their Property or {@link #addTrusted}/{@link #setTrusted(String, String)}. */ private boolean blockReservedKeyWrite(String name) { - if (!trusted && ReservedNamespaces.isTikaNative(name)) { + if (ReservedNamespaces.isTikaNative(name)) { LOG.debug("Dropping String write to reserved metadata key '{}'; use its Property.", name); return true; } @@ -357,9 +348,9 @@ public class Metadata set(property, value); } } else if (append) { - addUnchecked(name, value); + addTrusted(name, value); } else { - setUnchecked(name, value); + setTrusted(name, value); } return; } @@ -383,7 +374,7 @@ public class Metadata set(name, newValues); } else { for (String val : newValues) { - addUnchecked(name, val); + addTrusted(name, val); } } } @@ -414,7 +405,7 @@ public class Metadata set(property, value); } else { if (property.isMultiValuePermitted()) { - addUnchecked(property.getName(), value); + addTrusted(property.getName(), value); } else { throw new PropertyTypeException( property.getName() + " : " + property.getPropertyType()); @@ -450,11 +441,14 @@ public class Metadata if (blockReservedKeyWrite(name)) { return; } - setUnchecked(name, value); + setTrusted(name, value); } - /** Trusted set, bypassing the reserved-key guard. */ - private void setUnchecked(String name, String value) { + /** + * Trusted set, bypassing the reserved-key guard: writes reach reserved Tika-native + * ({@code tk:}) keys directly. See {@link #addTrusted}. + */ + public void setTrusted(String name, String value) { writeLimiter.set(name, value, metadata); } @@ -464,7 +458,7 @@ public class Metadata if (values != null) { metadata.remove(name); for (String v : values) { - addUnchecked(name, v); + addTrusted(name, v); } } else { metadata.remove(name); @@ -490,7 +484,7 @@ public class Metadata } } } else { - setUnchecked(property.getName(), value); + setTrusted(property.getName(), value); } } diff --git a/tika-core/src/main/java/org/apache/tika/metadata/filter/CaptureGroupMetadataFilter.java b/tika-core/src/main/java/org/apache/tika/metadata/filter/CaptureGroupMetadataFilter.java index a83e074a01..8d1ce468bb 100644 --- a/tika-core/src/main/java/org/apache/tika/metadata/filter/CaptureGroupMetadataFilter.java +++ b/tika-core/src/main/java/org/apache/tika/metadata/filter/CaptureGroupMetadataFilter.java @@ -112,7 +112,8 @@ public class CaptureGroupMetadataFilter extends MetadataFilterBase implements In } Matcher m = regex.matcher(val); if (m.find()) { - metadata.set(targetField, m.group(1)); + // target field name is operator-configured; may legitimately be a reserved key + metadata.setTrusted(targetField, m.group(1)); } } diff --git a/tika-core/src/main/java/org/apache/tika/metadata/filter/FieldNameMappingFilter.java b/tika-core/src/main/java/org/apache/tika/metadata/filter/FieldNameMappingFilter.java index 982906ea84..4b5464d948 100644 --- a/tika-core/src/main/java/org/apache/tika/metadata/filter/FieldNameMappingFilter.java +++ b/tika-core/src/main/java/org/apache/tika/metadata/filter/FieldNameMappingFilter.java @@ -70,7 +70,8 @@ public class FieldNameMappingFilter extends MetadataFilterBase { String[] vals = metadata.getValues(n); metadata.remove(n); for (String val : vals) { - metadata.add(mappings.get(n), val); + // target field name is operator-configured; may legitimately be a reserved key + metadata.addTrusted(mappings.get(n), val); } } else { metadata.remove(n); @@ -82,7 +83,7 @@ public class FieldNameMappingFilter extends MetadataFilterBase { String[] vals = metadata.getValues(n); metadata.remove(n); for (String val : vals) { - metadata.add(mappings.get(n), val); + metadata.addTrusted(mappings.get(n), val); } } } diff --git a/tika-core/src/main/java/org/apache/tika/metadata/filter/GeoPointMetadataFilter.java b/tika-core/src/main/java/org/apache/tika/metadata/filter/GeoPointMetadataFilter.java index 86d5d67d28..a57ec2af6d 100644 --- a/tika-core/src/main/java/org/apache/tika/metadata/filter/GeoPointMetadataFilter.java +++ b/tika-core/src/main/java/org/apache/tika/metadata/filter/GeoPointMetadataFilter.java @@ -88,6 +88,7 @@ public class GeoPointMetadataFilter extends MetadataFilterBase { if (StringUtils.isEmpty(lng)) { return; } - metadata.set(geoPointFieldName, lat + "," + lng); + // target field name is operator-configured; may legitimately be a reserved key + metadata.setTrusted(geoPointFieldName, lat + "," + lng); } } diff --git a/tika-core/src/main/java/org/apache/tika/metadata/filter/MetadataFilter.java b/tika-core/src/main/java/org/apache/tika/metadata/filter/MetadataFilter.java index 7b66a3f815..a2e8efff40 100644 --- a/tika-core/src/main/java/org/apache/tika/metadata/filter/MetadataFilter.java +++ b/tika-core/src/main/java/org/apache/tika/metadata/filter/MetadataFilter.java @@ -19,9 +19,7 @@ package org.apache.tika.metadata.filter; import java.io.Closeable; import java.io.IOException; import java.io.Serializable; -import java.util.IdentityHashMap; import java.util.List; -import java.util.Map; import org.apache.tika.exception.TikaException; import org.apache.tika.metadata.Metadata; @@ -41,21 +39,14 @@ public abstract class MetadataFilter implements Serializable, Closeable { */ public final void filter(List<Metadata> metadataList, ParseContext parseContext) throws TikaException { - Map<Metadata, Boolean> previous = new IdentityHashMap<>(); - for (Metadata m : metadataList) { - previous.put(m, m.isTrusted()); - m.setTrusted(true); - } - try { - doFilter(metadataList, parseContext); - } finally { - previous.forEach(Metadata::setTrusted); - } + doFilter(metadataList, parseContext); } /** - * Applies the filter in place. Reserved-key String writes are permitted here: filters run - * on already-parsed, trusted metadata. + * Applies the filter in place. A filter that needs to write a reserved Tika-native + * ({@code tk:}) key by name (rather than via its {@link org.apache.tika.metadata.Property}) + * should call {@link Metadata#addTrusted} / {@link Metadata#setTrusted} explicitly at that + * write. * * @param metadataList the list to filter (must be mutable) * @param parseContext per-request context diff --git a/tika-core/src/test/java/org/apache/tika/metadata/MetadataInternalKeyGuardTest.java b/tika-core/src/test/java/org/apache/tika/metadata/MetadataInternalKeyGuardTest.java index f7df174c4c..8603c5c27f 100644 --- a/tika-core/src/test/java/org/apache/tika/metadata/MetadataInternalKeyGuardTest.java +++ b/tika-core/src/test/java/org/apache/tika/metadata/MetadataInternalKeyGuardTest.java @@ -121,14 +121,22 @@ public class MetadataInternalKeyGuardTest { } @Test - public void testTrustedModeAllowsReservedStringWrites() { + public void testTrustedWriteBypassesGuard() { Metadata metadata = new Metadata(); - metadata.setTrusted(true); - metadata.set(TikaCoreProperties.TIKA_CONTENT.getName(), "trusted"); + metadata.setTrusted(TikaCoreProperties.TIKA_CONTENT.getName(), "trusted"); assertEquals("trusted", metadata.get(TikaCoreProperties.TIKA_CONTENT)); - metadata.setTrusted(false); + // untrusted String-path attempt must not clobber metadata.set(TikaCoreProperties.TIKA_CONTENT.getName(), "blocked"); assertEquals("trusted", metadata.get(TikaCoreProperties.TIKA_CONTENT)); } + + @Test + public void testTrustedAddBypassesGuard() { + Metadata metadata = new Metadata(); + metadata.addTrusted(TikaCoreProperties.TIKA_PARSED_BY.getName(), "p1"); + metadata.addTrusted(TikaCoreProperties.TIKA_PARSED_BY.getName(), "p2"); + assertArrayEquals(new String[] {"p1", "p2"}, + metadata.getValues(TikaCoreProperties.TIKA_PARSED_BY)); + } } diff --git a/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/ChunkSerializer.java b/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/ChunkSerializer.java index e13147ff62..6a8f66bea0 100644 --- a/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/ChunkSerializer.java +++ b/tika-parsers/tika-parsers-ml/tika-inference/src/main/java/org/apache/tika/inference/ChunkSerializer.java @@ -92,13 +92,7 @@ public final class ChunkSerializer { } existing.addAll(newChunks); // Chunks are Tika-native output; the default field (tk:chunks) is reserved, so write trusted. - boolean wasTrusted = metadata.isTrusted(); - metadata.setTrusted(true); - try { - metadata.set(fieldName, toJson(existing)); - } finally { - metadata.setTrusted(wasTrusted); - } + metadata.setTrusted(fieldName, toJson(existing)); } /** diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java index 1e7b4c33c9..ac6e3b6b9e 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java @@ -590,13 +590,7 @@ class AbstractPDF2XHTML extends PDFTextStripper { String renderChunks = renderMetadata.get(TikaCoreProperties.TIKA_CHUNKS); if (renderChunks != null && metadata.get(TikaCoreProperties.TIKA_CHUNKS) == null) { // tk:chunks is reserved; this is Tika propagating its own native output - boolean wasTrusted = metadata.isTrusted(); - metadata.setTrusted(true); - try { - metadata.set(TikaCoreProperties.TIKA_CHUNKS, renderChunks); - } finally { - metadata.setTrusted(wasTrusted); - } + metadata.setTrusted(TikaCoreProperties.TIKA_CHUNKS.getName(), renderChunks); } } } catch (IOException e) { diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java index 4245b9e631..90ef9a6aeb 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java @@ -232,18 +232,12 @@ class EmitHandler { private void injectUserMetadata(Metadata userMetadata, List<Metadata> metadataList) { Metadata target = metadataList.get(0); - boolean prev = target.isTrusted(); - target.setTrusted(true); - try { - for (String n : userMetadata.names()) { - //overwrite whatever was there - target.set(n, null); - for (String val : userMetadata.getValues(n)) { - target.add(n, val); - } + for (String n : userMetadata.names()) { + //overwrite whatever was there + target.setTrusted(n, null); + for (String val : userMetadata.getValues(n)) { + target.addTrusted(n, val); } - } finally { - target.setTrusted(prev); } } diff --git a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/metadata/filter/AttachmentCountingListFilter.java b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/metadata/filter/AttachmentCountingListFilter.java index 94a54c6f14..1e01d43482 100644 --- a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/metadata/filter/AttachmentCountingListFilter.java +++ b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/metadata/filter/AttachmentCountingListFilter.java @@ -33,7 +33,7 @@ public class AttachmentCountingListFilter extends MetadataFilter { if (metadataList == null || metadataList.isEmpty()) { return; } - metadataList.get(0).set("tk:attachment-count", Integer.toString(metadataList.size() - 1)); + metadataList.get(0).setTrusted("tk:attachment-count", Integer.toString(metadataList.size() - 1)); count += metadataList.size(); } diff --git a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/metadata/filter/MockUpperCaseFilter.java b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/metadata/filter/MockUpperCaseFilter.java index 30c698fc7e..6d4c07192e 100644 --- a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/metadata/filter/MockUpperCaseFilter.java +++ b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/metadata/filter/MockUpperCaseFilter.java @@ -33,7 +33,7 @@ public class MockUpperCaseFilter extends MetadataFilterBase { String[] vals = metadata.getValues(n); metadata.remove(n); for (String val : vals) { - metadata.add(n, val.toUpperCase(Locale.US)); + metadata.addTrusted(n, val.toUpperCase(Locale.US)); } } } diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-es/src/test/java/org/apache/tika/pipes/emitter/es/ESClientTest.java b/tika-pipes/tika-pipes-plugins/tika-pipes-es/src/test/java/org/apache/tika/pipes/emitter/es/ESClientTest.java index 83d7ffb5e8..d79ba86f63 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-es/src/test/java/org/apache/tika/pipes/emitter/es/ESClientTest.java +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-es/src/test/java/org/apache/tika/pipes/emitter/es/ESClientTest.java @@ -94,9 +94,7 @@ public class ESClientTest extends TikaTest { /** tk:chunks is reserved; simulate a trusted producer having written it. */ private static void setChunks(Metadata metadata, String json) { - metadata.setTrusted(true); - metadata.set(TikaCoreProperties.TIKA_CHUNKS, json); - metadata.setTrusted(false); + metadata.setTrusted(TikaCoreProperties.TIKA_CHUNKS.getName(), json); } @Test diff --git a/tika-serialization/src/test/java/org/apache/tika/metadata/filter/AttachmentCountingListFilter.java b/tika-serialization/src/test/java/org/apache/tika/metadata/filter/AttachmentCountingListFilter.java index 48af709438..a2bddcffc8 100644 --- a/tika-serialization/src/test/java/org/apache/tika/metadata/filter/AttachmentCountingListFilter.java +++ b/tika-serialization/src/test/java/org/apache/tika/metadata/filter/AttachmentCountingListFilter.java @@ -32,7 +32,7 @@ public class AttachmentCountingListFilter extends MetadataFilter { if (metadataList == null || metadataList.isEmpty()) { return; } - metadataList.get(0).set("tk:attachment-count", Integer.toString(metadataList.size() - 1)); + metadataList.get(0).setTrusted("tk:attachment-count", Integer.toString(metadataList.size() - 1)); count += metadataList.size(); } diff --git a/tika-serialization/src/test/java/org/apache/tika/metadata/filter/JsonConfigOnlyFilter.java b/tika-serialization/src/test/java/org/apache/tika/metadata/filter/JsonConfigOnlyFilter.java index d3e70a4afb..480c648402 100644 --- a/tika-serialization/src/test/java/org/apache/tika/metadata/filter/JsonConfigOnlyFilter.java +++ b/tika-serialization/src/test/java/org/apache/tika/metadata/filter/JsonConfigOnlyFilter.java @@ -59,7 +59,7 @@ public class JsonConfigOnlyFilter extends MetadataFilterBase { String[] values = metadata.getValues(name); metadata.remove(name); for (String value : values) { - metadata.add(name, prefix + value); + metadata.addTrusted(name, prefix + value); } } } diff --git a/tika-serialization/src/test/java/org/apache/tika/metadata/filter/MockUpperCaseFilter.java b/tika-serialization/src/test/java/org/apache/tika/metadata/filter/MockUpperCaseFilter.java index 30c698fc7e..6d4c07192e 100644 --- a/tika-serialization/src/test/java/org/apache/tika/metadata/filter/MockUpperCaseFilter.java +++ b/tika-serialization/src/test/java/org/apache/tika/metadata/filter/MockUpperCaseFilter.java @@ -33,7 +33,7 @@ public class MockUpperCaseFilter extends MetadataFilterBase { String[] vals = metadata.getValues(n); metadata.remove(n); for (String val : vals) { - metadata.add(n, val.toUpperCase(Locale.US)); + metadata.addTrusted(n, val.toUpperCase(Locale.US)); } } } diff --git a/tika-serialization/src/test/java/org/apache/tika/serialization/CustomClassSerializationTest.java b/tika-serialization/src/test/java/org/apache/tika/serialization/CustomClassSerializationTest.java index e660cc2acc..fb7cd33fa1 100644 --- a/tika-serialization/src/test/java/org/apache/tika/serialization/CustomClassSerializationTest.java +++ b/tika-serialization/src/test/java/org/apache/tika/serialization/CustomClassSerializationTest.java @@ -70,7 +70,7 @@ public class CustomClassSerializationTest { String[] values = metadata.getValues(name); metadata.remove(name); for (String value : values) { - metadata.add(name, prefix + value.toUpperCase(Locale.ROOT)); + metadata.addTrusted(name, prefix + value.toUpperCase(Locale.ROOT)); } } }
