This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch shim-for-tika-eval-for-pre-4.0.0 in repository https://gitbox.apache.org/repos/asf/tika.git
commit 3ae11448842efad649b3baa895efb044caefdb82 Author: tallison <[email protected]> AuthorDate: Wed Jul 29 12:06:53 2026 -0400 add shim so that tika-eval works for 4.0.0 vs 4.0.0-beta-1 --- .../org/apache/tika/eval/app/io/ExtractReader.java | 76 ++++++++++++++++++++++ .../apache/tika/eval/app/io/ExtractReaderTest.java | 34 ++++++++++ .../test-dirs/legacy/beta1-style.doc.json | 17 +++++ .../resources/test-dirs/legacy/collision.doc.json | 7 ++ 4 files changed, 134 insertions(+) diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/ExtractReader.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/ExtractReader.java index 26ff7f22cf..bac92b48e4 100644 --- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/ExtractReader.java +++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/ExtractReader.java @@ -25,8 +25,10 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -159,6 +161,9 @@ public class ExtractReader { try { if (fileSuffixes.format == FileSuffixes.FORMAT.JSON) { metadataList = JsonMetadataList.fromJson(reader); + for (Metadata m : metadataList) { + normalizeLegacyKeys(m); + } if (alterMetadataList.equals(ALTER_METADATA_LIST.FIRST_ONLY) && metadataList.size() > 1) { while (metadataList.size() > 1) { metadataList.remove(metadataList.size() - 1); @@ -214,6 +219,77 @@ public class ExtractReader { } + // Pre-4.0 extract key -> 4.0 key, for the Tika-native fields tika-eval reads. Digest keys are + // handled by the prefix rule in normalizeLegacyKeys; Content-Type/Content-Length are standard + // names (unchanged) so they are not listed. New-side keys come from the live constants so this + // can't drift from the 4.0 declarations. + private static final Map<String, String> LEGACY_KEY_MAP = Map.ofEntries( + Map.entry("X-TIKA:content", TikaCoreProperties.TIKA_CONTENT.getName()), + Map.entry("X-TIKA:content_handler", TikaCoreProperties.TIKA_CONTENT_HANDLER.getName()), + Map.entry("X-TIKA:embedded_depth", TikaCoreProperties.EMBEDDED_DEPTH.getName()), + Map.entry("X-TIKA:embedded_resource_path", TikaCoreProperties.EMBEDDED_RESOURCE_PATH.getName()), + Map.entry("X-TIKA:final_embedded_resource_path", TikaCoreProperties.FINAL_EMBEDDED_RESOURCE_PATH.getName()), + Map.entry("X-TIKA:parse_time_millis", TikaCoreProperties.PARSE_TIME_MILLIS.getName()), + Map.entry("X-TIKA:resourceName", TikaCoreProperties.RESOURCE_NAME_KEY.getName()), + Map.entry("X-TIKA:detectedEncoding", TikaCoreProperties.DETECTED_ENCODING.getName()), + Map.entry("X-TIKA:encodingDetector", TikaCoreProperties.ENCODING_DETECTOR.getName()), + Map.entry("Content-Type-Hint", TikaCoreProperties.CONTENT_TYPE_HINT.getName()), + Map.entry("embeddedResourceType", TikaCoreProperties.EMBEDDED_RESOURCE_TYPE.getName()), + Map.entry("X-TIKA:EXCEPTION:container_exception", TikaCoreProperties.CONTAINER_EXCEPTION.getName()), + Map.entry("X-TIKA:EXCEPTION:embedded_exception", TikaCoreProperties.EMBEDDED_EXCEPTION.getName())); + + private static final String LEGACY_DIGEST_PREFIX = TikaCoreProperties.LEGACY_TIKA_META_PREFIX + + "digest" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER; + + /** + * Pre-4.0 extracts (e.g. 4.0.0-beta-1) key Tika-native fields under X-TIKA:/camelCase names. + * Normalize the fields tika-eval reads to their 4.0 tk: keys so a cross-version compare reflects + * real diffs, not the rename. Harmless on 4.0 extracts: the legacy keys are simply absent. + */ + private static void normalizeLegacyKeys(Metadata m) { + m.setTrusted(true); // sanctioned trusted transformation: may write reserved tk: keys + try { + for (Map.Entry<String, String> e : LEGACY_KEY_MAP.entrySet()) { + remapLegacyKey(m, e.getKey(), e.getValue()); + } + // digest keys: X-TIKA:digest:<alg> -> tk:digest:<alg> (algorithm unchanged; MD5 drives + // embedded-doc matching). names() is a snapshot, so remapping while iterating is safe. + for (String name : m.names()) { + if (name.startsWith(LEGACY_DIGEST_PREFIX)) { + remapLegacyKey(m, name, TikaCoreProperties.TIKA_META_PREFIX + + name.substring(TikaCoreProperties.LEGACY_TIKA_META_PREFIX.length())); + } + } + } finally { + m.setTrusted(false); + } + } + + private static void remapLegacyKey(Metadata m, String legacyKey, String modernKey) { + String[] legacyVals = m.getValues(legacyKey); + if (legacyVals.length == 0) { + return; + } + String[] modernVals = m.getValues(modernKey); + if (modernVals.length > 0) { + // Both present: safe only if identical. Fail loud rather than silently clobber a value. + if (!Arrays.equals(legacyVals, modernVals)) { + throw new IllegalStateException("Extract has both legacy key '" + legacyKey + + "' and modern key '" + modernKey + "' with different values; legacy-key " + + "normalization would clobber. Extract is inconsistent."); + } + } else { + for (int i = 0; i < legacyVals.length; i++) { + if (i == 0) { + m.set(modernKey, legacyVals[i]); + } else { + m.add(modernKey, legacyVals[i]); + } + } + } + m.remove(legacyKey); + } + public enum ALTER_METADATA_LIST { AS_IS, //leave the metadata list as is FIRST_ONLY, //take only the metadata list for the "container" document diff --git a/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/io/ExtractReaderTest.java b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/io/ExtractReaderTest.java index 8c33d42ffc..e164bfe4fa 100644 --- a/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/io/ExtractReaderTest.java +++ b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/io/ExtractReaderTest.java @@ -18,6 +18,8 @@ package org.apache.tika.eval.app.io; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.io.IOException; import java.nio.file.Path; @@ -88,6 +90,38 @@ public class ExtractReaderTest extends TikaTest { .get(TikaCoreProperties.TIKA_CONTENT)); } + @Test + public void testLegacyKeyNormalization() throws Exception { + // a pre-4.0 (4.0.0-beta-1 style) extract: X-TIKA:/camelCase Tika-native keys + Path f = getResourceAsFile("/test-dirs/legacy/beta1-style.doc.json").toPath(); + List<Metadata> list = new ExtractReader().loadExtract(f); + assertEquals(2, list.size()); + + Metadata container = list.get(0); + // readable via the 4.0 Property constants that tika-eval uses + assertEquals("the quick brown fox", container.get(TikaCoreProperties.TIKA_CONTENT)); + assertEquals("12", container.get(TikaCoreProperties.PARSE_TIME_MILLIS)); + assertEquals("boom", container.get(TikaCoreProperties.CONTAINER_EXCEPTION)); + assertEquals("abc123", container.get("tk:digest:MD5")); + // the legacy keys are gone + assertNull(container.get("X-TIKA:content")); + assertNull(container.get("X-TIKA:digest:MD5")); + + Metadata embedded = list.get(1); + assertEquals("inner.txt", embedded.get(TikaCoreProperties.EMBEDDED_RESOURCE_PATH)); + assertEquals("1", embedded.get(TikaCoreProperties.EMBEDDED_DEPTH)); + assertEquals("inner.txt", embedded.get(TikaCoreProperties.RESOURCE_NAME_KEY)); + assertEquals("ATTACHMENT", embedded.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE)); + assertNull(embedded.get("X-TIKA:embedded_resource_path")); + } + + @Test + public void testLegacyNormalizationCollisionFailsLoud() throws Exception { + // both the legacy and modern content key present with different values -> must not clobber + Path f = getResourceAsFile("/test-dirs/legacy/collision.doc.json").toPath(); + assertThrows(IllegalStateException.class, () -> new ExtractReader().loadExtract(f)); + } + @Test public void testTextBasic() throws IOException { ExtractReader extractReader = new ExtractReader(); diff --git a/tika-eval/tika-eval-app/src/test/resources/test-dirs/legacy/beta1-style.doc.json b/tika-eval/tika-eval-app/src/test/resources/test-dirs/legacy/beta1-style.doc.json new file mode 100644 index 0000000000..d8938f4a06 --- /dev/null +++ b/tika-eval/tika-eval-app/src/test/resources/test-dirs/legacy/beta1-style.doc.json @@ -0,0 +1,17 @@ +[ + { + "Content-Type" : "text/plain", + "X-TIKA:content" : "the quick brown fox", + "X-TIKA:parse_time_millis" : "12", + "X-TIKA:digest:MD5" : "abc123", + "X-TIKA:EXCEPTION:container_exception" : "boom" + }, + { + "Content-Type" : "text/plain", + "X-TIKA:embedded_resource_path" : "inner.txt", + "X-TIKA:content" : "attachment contents", + "X-TIKA:embedded_depth" : "1", + "X-TIKA:resourceName" : "inner.txt", + "embeddedResourceType" : "ATTACHMENT" + } +] diff --git a/tika-eval/tika-eval-app/src/test/resources/test-dirs/legacy/collision.doc.json b/tika-eval/tika-eval-app/src/test/resources/test-dirs/legacy/collision.doc.json new file mode 100644 index 0000000000..47c2b342e1 --- /dev/null +++ b/tika-eval/tika-eval-app/src/test/resources/test-dirs/legacy/collision.doc.json @@ -0,0 +1,7 @@ +[ + { + "Content-Type" : "text/plain", + "X-TIKA:content" : "legacy value", + "tk:content" : "modern value" + } +]
