This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch metadata-key-passthrough
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/metadata-key-passthrough by
this push:
new 82a558484d add validation
82a558484d is described below
commit 82a558484d9beebcc20f6ed110514a64b2cda4c1
Author: tallison <[email protected]>
AuthorDate: Mon Jul 27 16:20:39 2026 -0400
add validation
---
tika-metadata-schema/README.md | 29 +++-
.../tika/metadata/schema/MetadataKeyValidator.java | 146 +++++++++++++++++++++
.../apache/tika/metadata/metadata-string-keys.json | 17 +++
.../metadata/schema/MetadataKeyValidatorTest.java | 90 +++++++++++++
.../metadata/schema/MetadataStringKeysTest.java | 65 +++++++++
5 files changed, 342 insertions(+), 5 deletions(-)
diff --git a/tika-metadata-schema/README.md b/tika-metadata-schema/README.md
index 105b4de77a..84c6d0f5ff 100644
--- a/tika-metadata-schema/README.md
+++ b/tika-metadata-schema/README.md
@@ -16,8 +16,8 @@
-->
# tika-metadata-schema
-A machine-readable schema of Apache Tika's metadata keys. Two registries,
because Tika has two
-kinds of keys.
+A machine-readable schema of Apache Tika's metadata keys, plus a
registry-driven validator
+(`MetadataKeyValidator`) that classifies any key as CLOSED / OPEN / TEMPLATE /
UNKNOWN.
**Scope: `tika-core` + the standard parser bundle.** The heavier/optional
parser families
(scientific, sqlite3, nlp, vlm) are *not* scanned — pulling their runtime deps
(netcdf, grib,
@@ -59,7 +59,26 @@ Not covered here: **templates** — parameterized key families
like XMP `rdf:Alt
`<base-key>:<lang>` (`dc:title:fr`), where the *suffix* rather than the prefix
is open. These are
documented by rule, not enumerated.
+## `metadata-string-keys.json` — legacy bare-String closed keys (curated +
gated)
+A handful of closed keys predate `Property` and are still declared as bare
`String` constants
+(`HttpHeaders.CONTENT_TYPE` = `Content-Type`, the `Content-*`/`Location`
family, `Message-*` /
+`Multipart-*`, `tika:chunks`). They self-register nowhere, so the `Property`
scan can't see them —
+yet Tika emits them constantly. Each record: `{ key, source }`.
+
+**Curated, but gated against the code:** `MetadataStringKeysTest` reflects
each `source` constant
+(e.g. `HttpHeaders.CONTENT_TYPE`) and asserts its live value equals `key`, so
a rename/retype/value
+change fails the build. The right long-term fix is to make these `Property`
constants (then they'd
+move to `metadata-keys.json` automatically); that's a large,
`Content-Type`-blast-radius change left
+for a future major release.
+
+## `MetadataKeyValidator` — the registry-driven lint
+Classifies any key by reading the three registries above (no parser classes
needed):
+`CLOSED` (in `metadata-keys.json` or `metadata-string-keys.json`), `OPEN`
(under a registered
+passthrough prefix), `TEMPLATE` (a `<closed-key>:<lang>` lang-alt instance),
or `UNKNOWN` — a typo,
+an unregistered namespace, or a key nobody declared. This is the payoff the
registries exist for: a
+data-driven legitimacy check instead of a hand-coded regex.
+
Together the files describe the key space of the scanned bundle: closed keys
are enumerated and
-gated; open namespaces are enumerated by prefix and gated; templates are
described by rule; and
-`MetadataCoverageTest` guarantees no scanned-bundle module is silently missed.
Keys from the
-out-of-scope families above are excluded by design.
+gated (Property-backed and legacy-String alike); open namespaces are
enumerated by prefix and gated;
+templates are described by rule; and `MetadataCoverageTest` guarantees no
scanned-bundle module is
+silently missed. Keys from the out-of-scope families above are excluded by
design.
diff --git
a/tika-metadata-schema/src/main/java/org/apache/tika/metadata/schema/MetadataKeyValidator.java
b/tika-metadata-schema/src/main/java/org/apache/tika/metadata/schema/MetadataKeyValidator.java
new file mode 100644
index 0000000000..4af312e9f6
--- /dev/null
+++
b/tika-metadata-schema/src/main/java/org/apache/tika/metadata/schema/MetadataKeyValidator.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.metadata.schema;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+/**
+ * Registry-driven lint: classifies a metadata key against the committed
registries rather than a
+ * hand-coded rule. A key is legitimate iff it is an enumerated closed key
+ * ({@code metadata-keys.json}), sits under a registered passthrough prefix
+ * ({@code metadata-open-namespaces.json}), or is a documented template
instance. Anything else is
+ * {@link Classification#UNKNOWN} — a typo, an unregistered namespace, or a
key someone forgot to
+ * declare.
+ *
+ * <p>Reads the JSON snapshots (which {@code MetadataSchemaTest} gates against
the live declarations),
+ * so it needs no parser classes loaded and stays dependency-free.
+ */
+public final class MetadataKeyValidator {
+
+ public enum Classification { CLOSED, OPEN, TEMPLATE, UNKNOWN }
+
+ private static final String KEYS =
"/org/apache/tika/metadata/metadata-keys.json";
+ private static final String OPEN =
"/org/apache/tika/metadata/metadata-open-namespaces.json";
+ // Legacy closed keys declared as bare String constants (not Property), so
absent from KEYS.
+ private static final String STRING_KEYS =
"/org/apache/tika/metadata/metadata-string-keys.json";
+
+ // Conservative BCP-47 subset for the XMP lang-alt template suffix
(<closed-key>:<lang>).
+ private static final Pattern LANG_TAG =
+ Pattern.compile("x-default|[A-Za-z]{2,3}(-[A-Za-z0-9]{1,8})*");
+
+ private final Set<String> closedKeys;
+ private final List<String> openPrefixes; // longest-first: most specific
prefix wins
+
+ MetadataKeyValidator(Set<String> closedKeys, List<String> openPrefixes) {
+ this.closedKeys = Set.copyOf(closedKeys);
+ List<String> sorted = new ArrayList<>(openPrefixes);
+ sorted.sort(Comparator.comparingInt(String::length).reversed());
+ this.openPrefixes = List.copyOf(sorted);
+ }
+
+ /** Loads the validator from the committed registries on the classpath. */
+ public static MetadataKeyValidator fromClasspath() {
+ Set<String> closed = new HashSet<>(readValues(KEYS, "key"));
+ closed.addAll(readValues(STRING_KEYS, "key"));
+ return new MetadataKeyValidator(closed, readValues(OPEN, "prefix"));
+ }
+
+ public Classification classify(String key) {
+ if (key == null || key.isEmpty()) {
+ return Classification.UNKNOWN;
+ }
+ if (closedKeys.contains(key)) {
+ return Classification.CLOSED;
+ }
+ for (String prefix : openPrefixes) {
+ if (key.length() > prefix.length() && key.startsWith(prefix)) {
+ return Classification.OPEN;
+ }
+ }
+ if (isLangAltInstance(key)) {
+ return Classification.TEMPLATE;
+ }
+ return Classification.UNKNOWN;
+ }
+
+ public boolean isLegitimate(String key) {
+ return classify(key) != Classification.UNKNOWN;
+ }
+
+ /** The unregistered keys among {@code names}, in encounter order. */
+ public List<String> findUnknown(Iterable<String> names) {
+ List<String> unknown = new ArrayList<>();
+ for (String name : names) {
+ if (classify(name) == Classification.UNKNOWN) {
+ unknown.add(name);
+ }
+ }
+ return unknown;
+ }
+
+ /** {@code <closed-key>:<lang>} — the XMP rdf:Alt language variants
(dc:title:fr, dc:title:x-default). */
+ private boolean isLangAltInstance(String key) {
+ int i = key.lastIndexOf(':');
+ if (i <= 0 || i == key.length() - 1) {
+ return false;
+ }
+ return closedKeys.contains(key.substring(0, i)) &&
LANG_TAG.matcher(key.substring(i + 1)).matches();
+ }
+
+ private static List<String> readValues(String resource, String field) {
+ String json = readResource(resource);
+ List<String> out = new ArrayList<>();
+ String marker = '"' + field + "\":\"";
+ int i = 0;
+ while ((i = json.indexOf(marker, i)) >= 0) {
+ i += marker.length();
+ StringBuilder sb = new StringBuilder();
+ while (i < json.length()) {
+ char c = json.charAt(i++);
+ if (c == '\\' && i < json.length()) {
+ sb.append(json.charAt(i++)); // unescape \" and \\
+ } else if (c == '"') {
+ break;
+ } else {
+ sb.append(c);
+ }
+ }
+ out.add(sb.toString());
+ }
+ return out;
+ }
+
+ private static String readResource(String resource) {
+ try (InputStream in =
MetadataKeyValidator.class.getResourceAsStream(resource)) {
+ if (in == null) {
+ throw new IllegalStateException("missing registry resource " +
resource);
+ }
+ return new String(in.readAllBytes(), StandardCharsets.UTF_8);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+}
diff --git
a/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-string-keys.json
b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-string-keys.json
new file mode 100644
index 0000000000..beaa0d0cff
--- /dev/null
+++
b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-string-keys.json
@@ -0,0 +1,17 @@
+[
+ {"key":"Content-Disposition","source":"HttpHeaders.CONTENT_DISPOSITION"},
+ {"key":"Content-Encoding","source":"HttpHeaders.CONTENT_ENCODING"},
+ {"key":"Content-Language","source":"HttpHeaders.CONTENT_LANGUAGE"},
+ {"key":"Content-Length","source":"HttpHeaders.CONTENT_LENGTH"},
+ {"key":"Content-Location","source":"HttpHeaders.CONTENT_LOCATION"},
+ {"key":"Content-Type","source":"HttpHeaders.CONTENT_TYPE"},
+ {"key":"Location","source":"HttpHeaders.LOCATION"},
+ {"key":"Message-Bcc","source":"Message.MESSAGE_BCC"},
+ {"key":"Message-Cc","source":"Message.MESSAGE_CC"},
+ {"key":"Message-From","source":"Message.MESSAGE_FROM"},
+
{"key":"Message-Recipient-Address","source":"Message.MESSAGE_RECIPIENT_ADDRESS"},
+ {"key":"Message-To","source":"Message.MESSAGE_TO"},
+ {"key":"Multipart-Boundary","source":"Message.MULTIPART_BOUNDARY"},
+ {"key":"Multipart-Subtype","source":"Message.MULTIPART_SUBTYPE"},
+ {"key":"tika:chunks","source":"TikaCoreProperties.TIKA_CHUNKS"}
+]
diff --git
a/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataKeyValidatorTest.java
b/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataKeyValidatorTest.java
new file mode 100644
index 0000000000..700b97be14
--- /dev/null
+++
b/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataKeyValidatorTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.metadata.schema;
+
+import static
org.apache.tika.metadata.schema.MetadataKeyValidator.Classification.CLOSED;
+import static
org.apache.tika.metadata.schema.MetadataKeyValidator.Classification.OPEN;
+import static
org.apache.tika.metadata.schema.MetadataKeyValidator.Classification.TEMPLATE;
+import static
org.apache.tika.metadata.schema.MetadataKeyValidator.Classification.UNKNOWN;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+public class MetadataKeyValidatorTest {
+
+ // Controlled fixture: two closed keys and two open prefixes, so
classification is unambiguous.
+ private final MetadataKeyValidator v = new MetadataKeyValidator(
+ java.util.Set.of("dc:title", "X-TIKA:Parsed-By"),
+ List.of("html:", "pdf:docinfo:custom:"));
+
+ @Test
+ public void closedKeyIsClosed() {
+ assertEquals(CLOSED, v.classify("dc:title"));
+ assertEquals(CLOSED, v.classify("X-TIKA:Parsed-By"));
+ }
+
+ @Test
+ public void passthroughPrefixIsOpen() {
+ assertEquals(OPEN, v.classify("html:og:image"));
+ assertEquals(OPEN, v.classify("pdf:docinfo:custom:MyField"));
+ }
+
+ @Test
+ public void langAltVariantIsTemplate() {
+ assertEquals(TEMPLATE, v.classify("dc:title:fr"));
+ assertEquals(TEMPLATE, v.classify("dc:title:x-default"));
+ assertEquals(TEMPLATE, v.classify("dc:title:en-US"));
+ }
+
+ @Test
+ public void unregisteredKeyIsUnknown() {
+ assertEquals(UNKNOWN, v.classify("bogus:notakey"));
+ assertEquals(UNKNOWN, v.classify("Content-Type")); // bare String
constant, not a Property
+ assertEquals(UNKNOWN, v.classify("dc:title:notalangtag_but_long"));
+ }
+
+ @Test
+ public void edgeCases() {
+ assertEquals(UNKNOWN, v.classify(null));
+ assertEquals(UNKNOWN, v.classify(""));
+ assertEquals(UNKNOWN, v.classify("html:")); // bare
prefix, no suffix
+ }
+
+ @Test
+ public void findUnknownFiltersLegit() {
+ List<String> unknown = v.findUnknown(
+ List.of("dc:title", "html:x", "dc:title:fr", "Content-Type",
"bogus:k"));
+ assertEquals(List.of("Content-Type", "bogus:k"), unknown);
+ }
+
+ @Test
+ public void loadsRealRegistriesAndAgreesWithThem() {
+ MetadataKeyValidator real = MetadataKeyValidator.fromClasspath();
+ // Every real closed key classifies CLOSED; a real passthrough suffix
classifies OPEN.
+ assertEquals(CLOSED, real.classify("dc:title"));
+ assertEquals(CLOSED, real.classify("X-TIKA:digest:MD5")); //
synthesized digest key
+ assertEquals(CLOSED, real.classify("Content-Type")); //
bare-String key registry
+ assertEquals(CLOSED, real.classify("Message-From"));
+ assertEquals(OPEN, real.classify("html:twitter:card"));
+ assertTrue(real.isLegitimate("dc:title:fr"));
+ assertFalse(real.isLegitimate("totally:made:up:key"));
+ }
+}
diff --git
a/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataStringKeysTest.java
b/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataStringKeysTest.java
new file mode 100644
index 0000000000..fc84fb33fd
--- /dev/null
+++
b/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataStringKeysTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.metadata.schema;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import java.io.InputStream;
+import java.lang.reflect.Field;
+import java.nio.charset.StandardCharsets;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.junit.jupiter.api.Test;
+
+/**
+ * Gate for the curated {@code metadata-string-keys.json}: each entry's key
must equal the live value
+ * of the {@code source} constant it names (e.g. {@code
HttpHeaders.CONTENT_TYPE}). If a constant is
+ * renamed, retyped, or its value changes, this fails — the hand-maintained
list can't silently rot.
+ */
+public class MetadataStringKeysTest {
+
+ private static final String RESOURCE =
"/org/apache/tika/metadata/metadata-string-keys.json";
+ private static final String PKG = "org.apache.tika.metadata.";
+ private static final Pattern ENTRY =
+ Pattern.compile("\\{\"key\":\"(.*?)\",\"source\":\"(.*?)\"}");
+
+ @Test
+ public void everyStringKeyMatchesItsSourceConstant() throws Exception {
+ String json;
+ try (InputStream in =
MetadataStringKeysTest.class.getResourceAsStream(RESOURCE)) {
+ assertNotNull(in, "committed " + RESOURCE + " is missing");
+ json = new String(in.readAllBytes(), StandardCharsets.UTF_8);
+ }
+ Matcher m = ENTRY.matcher(json);
+ int count = 0;
+ while (m.find()) {
+ String key = m.group(1);
+ String source = m.group(2); // e.g.
HttpHeaders.CONTENT_TYPE
+ int dot = source.lastIndexOf('.');
+ Class<?> cls = Class.forName(PKG + source.substring(0, dot));
+ Field field = cls.getField(source.substring(dot + 1));
+ assertEquals(String.class, field.getType(),
+ source + " must be a String constant (a Property belongs
in metadata-keys.json)");
+ assertEquals(key, field.get(null),
+ source + " value drifted from metadata-string-keys.json");
+ count++;
+ }
+ assertEquals(15, count, "unexpected metadata-string-keys.json entry
count");
+ }
+}
