This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4807 in repository https://gitbox.apache.org/repos/asf/tika.git
commit 2dc9f26ef26fe6177f5ac522655df6732d6af659 Author: tallison <[email protected]> AuthorDate: Fri Aug 7 12:10:38 2026 -0400 TIKA-4807: consolidate metadata registry regeneration into one script --- .skills/metadata-schema.md | 15 +++- docs/modules/ROOT/nav.adoc | 1 + docs/modules/ROOT/pages/developers/index.adoc | 2 + .../ROOT/pages/developers/metadata-keys.adoc | 67 +++++++++++++++ tika-metadata-schema/README.md | 10 +-- tika-metadata-schema/regen.sh | 95 ++++++++++++++++++++++ .../metadata/schema/MetadataFieldTableTest.java | 2 +- .../tika/metadata/schema/MetadataSchemaTest.java | 7 +- 8 files changed, 188 insertions(+), 11 deletions(-) diff --git a/.skills/metadata-schema.md b/.skills/metadata-schema.md index 627827eef6..da51bf1fdb 100644 --- a/.skills/metadata-schema.md +++ b/.skills/metadata-schema.md @@ -17,6 +17,19 @@ shows up as a diff). Don't switch to build-time-only generation — that loses t ## Regenerate (after adding/changing a Property or PassthroughPrefix) +```bash +tika-metadata-schema/regen.sh +``` + +This does the full sequence in one shot: `-am install` so newly added Property/PassthroughPrefix +classes are on the scan classpath, regenerate all three registries via the forked-exec profile, print +a before/after key-count check (catches an incomplete classpath scan), `git diff --stat` the +registries, then run the gate tests. Flags: `--skip-install` (only safe if nothing outside +`tika-metadata-schema` itself changed since the last install) and `--skip-tests` for a faster inner +loop. Then review the diff and commit the Property change and the regenerated JSON together. + +The manual sequence the script replaces, for reference or if you need to run a step in isolation: + ```bash # if parser Property classes changed, install them first so the scan sees them: ./mvnw -Pfast -DskipTests -pl tika-metadata-schema -am install -Dmaven.repo.local=$(pwd)/.local_m2_repo @@ -24,8 +37,6 @@ shows up as a diff). Don't switch to build-time-only generation — that loses t ./mvnw -pl tika-metadata-schema -Pregen-metadata-schema process-classes -Dmaven.repo.local=$(pwd)/.local_m2_repo ``` -Then `git diff` the JSONs, run the gate tests, commit. - **Trap — never use `exec:java`.** `SchemaGenerator` scans `java.class.path`, force-loads Property classes, and swallows load failures. `exec:java` runs in-process on Maven's classpath → finds zero keys → emits a near-empty registry that exits 0 and *passes* `MetadataNoUnderscoreTest`. The diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index 3c7ae7a011..a61ad78ee0 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -76,6 +76,7 @@ ** xref:advanced/integration-testing/run-uat-script.adoc[Tika-Server REST UAT Script] * xref:developers/index.adoc[Developers] ** xref:developers/serialization.adoc[Serialization and Configuration] +** xref:developers/metadata-keys.adoc[Adding a Metadata Key] * xref:faq.adoc[FAQ] * xref:security.adoc[Security] * xref:roadmap.adoc[Roadmap] diff --git a/docs/modules/ROOT/pages/developers/index.adoc b/docs/modules/ROOT/pages/developers/index.adoc index e72c12747b..ac1ef574be 100644 --- a/docs/modules/ROOT/pages/developers/index.adoc +++ b/docs/modules/ROOT/pages/developers/index.adoc @@ -22,6 +22,8 @@ with custom parsers, detectors, and other components. * xref:developers/serialization.adoc[Serialization and Configuration] - JSON configuration, @TikaComponent annotation, and creating custom components +* xref:developers/metadata-keys.adoc[Adding a Metadata Key] - the Property/PassthroughPrefix + registry, naming conventions, and regenerating the schema == Coming Soon diff --git a/docs/modules/ROOT/pages/developers/metadata-keys.adoc b/docs/modules/ROOT/pages/developers/metadata-keys.adoc new file mode 100644 index 0000000000..7d5deeb4f7 --- /dev/null +++ b/docs/modules/ROOT/pages/developers/metadata-keys.adoc @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + += Adding a Metadata Key + +Every metadata key Tika can emit is a `Property` constant (or, for runtime-minted names like scraped +HTML `<meta>` tags, a `PassthroughPrefix`) — there are no bare `String` keys. That closed/open key +space is tracked in a generated, build-gated registry, so adding a key involves one extra step beyond +writing the Java. + +== Add the constant + +Add the `Property` to its interface as usual: + +[source,java] +---- +Property MY_NEW_KEY = Property.internalText(TIKA_META_PREFIX + "my-new-key"); +---- + +Naming conventions (frozen for 4.0): + +* Tika-coined keys use the `tk:` namespace, kebab-case, no underscores. +* External-standard names are used verbatim, including the standard's own prefix (`dc:`, `xmp:`, + `cp:`, `extended-properties:`). +* HTTP headers stay bare — no `http:` namespace (`Content-Type`, `Content-Encoding`, `Location`). + +== Regenerate the registry + +The registry — three JSON files under `tika-metadata-schema/src/main/resources/`, listing every +declared key, every open-namespace prefix, and a field-provenance table — is generated from the live +`Property`/`PassthroughPrefix` declarations, never hand-edited. A committed copy is the reviewable +audit trail (a rename or dropped key shows up as a diff), and CI fails if it's stale. + +Run this after adding, renaming, or removing a `Property` or `PassthroughPrefix`: + +[source,bash] +---- +tika-metadata-schema/regen.sh +---- + +It installs the modules the change touched, regenerates all three registry files, sanity-checks the +diff, and runs the gate tests. Commit the Java change and the regenerated JSON together. + +Details, flags, and the traps this script exists to avoid (classpath scanning quirks, `exec:java` +vs. a forked classpath) are documented in `tika-metadata-schema/README.md`. + +== After a rename + +The compiler won't catch a stale string literal like `metadata.get("Message-From")`. Grep the repo +for the old key and replace it with the constant: + +[source,bash] +---- +grep -rn '"Message-' --include=*.java . | grep -v /target/ +---- diff --git a/tika-metadata-schema/README.md b/tika-metadata-schema/README.md index d2fbc96a17..94968e36fa 100644 --- a/tika-metadata-schema/README.md +++ b/tika-metadata-schema/README.md @@ -36,13 +36,13 @@ declare a `Property` field, force-loads them, reads the global `Property` table, sorted JSON. `MetadataSchemaTest` regenerates in-memory and asserts it matches the committed file, so the registry can never drift from the declarations. -Regenerate after adding/changing a `Property` **or** a `PassthroughPrefix` (writes both files): +Regenerate after adding/changing a `Property` **or** a `PassthroughPrefix` (writes all three files): ``` -java -cp <tika-metadata-schema + deps classpath> \ - org.apache.tika.metadata.schema.SchemaGenerator \ - src/main/resources/org/apache/tika/metadata/metadata-keys.json \ - src/main/resources/org/apache/tika/metadata/metadata-open-namespaces.json +tika-metadata-schema/regen.sh ``` +Installs the dependency modules, regenerates the registries via the forked-exec profile, sanity-checks +the key-count diff, and runs the gate tests — see `.skills/metadata-schema.md` for flags and the +manual steps this replaces. ## `metadata-open-namespaces.json` — the open sets (generated + gated) The **prefixes** under which parsers mint file-controlled key names at runtime — names that are not diff --git a/tika-metadata-schema/regen.sh b/tika-metadata-schema/regen.sh new file mode 100755 index 0000000000..4d20fa7f45 --- /dev/null +++ b/tika-metadata-schema/regen.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# +# Regenerates and validates the metadata key registry (tika-metadata-schema). +# +# Run this after adding, renaming, or removing a Property or PassthroughPrefix +# constant anywhere in tika-core or the standard parser bundle. It replaces the +# multi-step manual sequence in .skills/metadata-schema.md with one command: +# install the dependency modules, regenerate the three registry files, sanity +# check the diff, then run the gate tests. +# +# Usage: +# tika-metadata-schema/regen.sh [--skip-install] [--skip-tests] +# +# --skip-install skip the -am install step (only safe if no Property/ +# PassthroughPrefix classes outside tika-metadata-schema +# changed since the last install) +# --skip-tests skip the final gate-test run, for a faster inner loop +# +# See tika-metadata-schema/README.md and .skills/metadata-schema.md for the +# design and the traps this script exists to route around. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$REPO_ROOT" + +MVN_REPO_OPT="-Dmaven.repo.local=$REPO_ROOT/.local_m2_repo" + +SKIP_INSTALL=0 +SKIP_TESTS=0 +for arg in "$@"; do + case "$arg" in + --skip-install) SKIP_INSTALL=1 ;; + --skip-tests) SKIP_TESTS=1 ;; + -h|--help) + sed -n '2,20p' "$0" | sed 's/^# \{0,1\}//' + exit 0 + ;; + *) + echo "Unknown argument: $arg" >&2 + exit 1 + ;; + esac +done + +REGISTRY_DIR="tika-metadata-schema/src/main/resources/org/apache/tika/metadata" +REGISTRY_FILES=( + "$REGISTRY_DIR/metadata-keys.json" + "$REGISTRY_DIR/metadata-open-namespaces.json" + "$REGISTRY_DIR/metadata-key-fields.json" +) + +if [ "$SKIP_INSTALL" -eq 0 ]; then + echo "==> Installing tika-metadata-schema + its dependency modules (tika-core, standard parsers)" + echo " so newly added Property/PassthroughPrefix classes are on the scan classpath." + echo " (skip with --skip-install if you already did this)" + ./mvnw -Pfast -DskipTests -pl tika-metadata-schema -am install "$MVN_REPO_OPT" +fi + +echo "==> Recording committed key counts, to catch an incomplete classpath scan later" +BEFORE_COUNTS=() +for f in "${REGISTRY_FILES[@]}"; do + if git cat-file -e "HEAD:$f" 2>/dev/null; then + BEFORE_COUNTS+=("$(git show "HEAD:$f" | wc -l)") + else + BEFORE_COUNTS+=("0") + fi +done + +echo "==> Regenerating the registry (forked exec — see .skills/metadata-schema.md for why exec:java is unsafe)" +./mvnw -pl tika-metadata-schema -Pregen-metadata-schema process-classes "$MVN_REPO_OPT" + +echo "==> Comparing key counts before/after (a large drop usually means classes failed to load):" +for i in "${!REGISTRY_FILES[@]}"; do + f="${REGISTRY_FILES[$i]}" + before="${BEFORE_COUNTS[$i]}" + after=$(wc -l < "$f") + line=" $f: $before -> $after lines" + if [ "$before" -gt 0 ] && [ "$after" -lt $((before * 90 / 100)) ]; then + echo "$line *** WARNING: >10% drop, check --skip-install and module installs ***" + else + echo "$line" + fi +done + +echo "==> git diff of the registries (review before committing):" +git --no-pager diff --stat -- "${REGISTRY_FILES[@]}" + +if [ "$SKIP_TESTS" -eq 0 ]; then + echo "==> Running gate tests (MetadataSchemaTest, MetadataFieldTableTest, MetadataNoUnderscoreTest, MetadataCoverageTest, ...)" + ./mvnw -pl tika-metadata-schema test "$MVN_REPO_OPT" +fi + +echo "==> Done. Review the diff above, then commit the Property/PassthroughPrefix change and the regenerated JSON together." diff --git a/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataFieldTableTest.java b/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataFieldTableTest.java index 38ca3578f3..9f9c4297e4 100644 --- a/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataFieldTableTest.java +++ b/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataFieldTableTest.java @@ -41,6 +41,6 @@ public class MetadataFieldTableTest { committed = new String(in.readAllBytes(), StandardCharsets.UTF_8); } assertEquals(committed, SchemaGenerator.fieldTable(), - "metadata-key-fields.json is stale. Run SchemaGenerator.main (3rd arg) and commit it."); + "metadata-key-fields.json is stale. Run tika-metadata-schema/regen.sh and commit it."); } } diff --git a/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataSchemaTest.java b/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataSchemaTest.java index 25f26c5fbd..5530b82b10 100644 --- a/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataSchemaTest.java +++ b/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataSchemaTest.java @@ -38,7 +38,8 @@ public class MetadataSchemaTest { @Test public void committedKeysMatchDeclarations() throws Exception { assertEquals(committed(KEYS), SchemaGenerator.generate(), - "metadata-keys.json is stale. Run SchemaGenerator.main and commit the result."); + "metadata-keys.json is stale. Run tika-metadata-schema/regen.sh and commit the " + + "result."); } @Test @@ -46,8 +47,8 @@ public class MetadataSchemaTest { // generate() first, so the classpath scan force-loads the PassthroughPrefix declarations. SchemaGenerator.generate(); assertEquals(committed(OPEN), SchemaGenerator.passthroughJson(), - "metadata-open-namespaces.json is stale. Run SchemaGenerator.main and commit the " - + "result."); + "metadata-open-namespaces.json is stale. Run tika-metadata-schema/regen.sh and " + + "commit the result."); } private static String committed(String resource) throws Exception {
