This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch metadata-key-passthrough in repository https://gitbox.apache.org/repos/asf/tika.git
commit e8f366f867d68df002e89cbdf37521af2b3f0ae9 Author: tallison <[email protected]> AuthorDate: Mon Jul 27 16:00:36 2026 -0400 property prefix --- .../main/java/org/apache/tika/metadata/DWG.java | 3 + .../main/java/org/apache/tika/metadata/HTML.java | 3 + .../main/java/org/apache/tika/metadata/MAPI.java | 1 + .../java/org/apache/tika/metadata/Message.java | 3 + .../main/java/org/apache/tika/metadata/Office.java | 3 + .../main/java/org/apache/tika/metadata/PDF.java | 3 + .../apache/tika/metadata/PassthroughPrefix.java | 79 +++++++++++ tika-metadata-schema/README.md | 41 ++++-- .../tika/metadata/schema/SchemaGenerator.java | 33 ++++- .../tika/metadata/metadata-open-namespaces.json | 60 +++------ .../tika/metadata/schema/MetadataCoverageTest.java | 150 +++++++++++++++++++++ .../tika/metadata/schema/MetadataSchemaTest.java | 37 +++-- .../apache/tika/parser/envi/EnviHeaderParser.java | 11 +- .../org/apache/tika/parser/grib/GribParser.java | 5 +- .../apache/tika/parser/netcdf/NetCDFParser.java | 8 +- .../tika/parser/ctakes/CTAKESContentHandler.java | 7 +- .../tika/parser/journal/GrobidRESTParser.java | 6 +- .../apache/tika/parser/ner/NamedEntityParser.java | 5 +- .../tika/parser/iwork/NumbersContentHandler.java | 6 +- .../parser/iwork/iwana/IWork13PackageParser.java | 5 +- .../org/apache/tika/parser/ogg/OggAudioParser.java | 6 +- .../org/apache/tika/parser/video/FLVParser.java | 5 +- .../java/org/apache/tika/parser/dwg/DWGParser.java | 9 +- .../org/apache/tika/parser/dwg/DWGReadParser.java | 4 +- .../tika/parser/image/ImageMetadataExtractor.java | 11 +- .../tika/parser/mail/MailContentHandler.java | 4 +- .../org/apache/tika/parser/mbox/MboxParser.java | 5 +- .../tika/parser/microsoft/JackcessExtractor.java | 6 +- .../tika/parser/microsoft/JackcessParser.java | 7 + .../tika/parser/microsoft/SummaryExtractor.java | 3 +- .../microsoft/msg/ExtendedMetadataExtractor.java | 2 +- .../tika/parser/odf/OpenDocumentMetaParser.java | 2 +- .../java/org/apache/tika/parser/pdf/PDFParser.java | 2 +- .../org/apache/tika/parser/warc/WARCParser.java | 10 +- .../org/apache/tika/parser/xmp/XmpExtractor.java | 4 +- 35 files changed, 443 insertions(+), 106 deletions(-) diff --git a/tika-core/src/main/java/org/apache/tika/metadata/DWG.java b/tika-core/src/main/java/org/apache/tika/metadata/DWG.java index 99f5946c70..6d53912dd8 100644 --- a/tika-core/src/main/java/org/apache/tika/metadata/DWG.java +++ b/tika-core/src/main/java/org/apache/tika/metadata/DWG.java @@ -23,6 +23,9 @@ public interface DWG { String DWG_PREFIX = "dwg" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER; + PassthroughPrefix RAW_FIELD = + PassthroughPrefix.file(DWG_PREFIX, "DWGRead JSON header/summary field names"); + Property APPLICATION_NAME = Property.externalText(DWG_PREFIX + "applicationName"); Property APPLICATION_VERSION = Property.externalText(DWG_PREFIX + "applicationVersion"); diff --git a/tika-core/src/main/java/org/apache/tika/metadata/HTML.java b/tika-core/src/main/java/org/apache/tika/metadata/HTML.java index a25b949302..a26c979479 100644 --- a/tika-core/src/main/java/org/apache/tika/metadata/HTML.java +++ b/tika-core/src/main/java/org/apache/tika/metadata/HTML.java @@ -19,6 +19,9 @@ package org.apache.tika.metadata; public interface HTML { String PREFIX_HTML_META = "html" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER; + PassthroughPrefix SCRAPED_META = PassthroughPrefix.file(PREFIX_HTML_META, + "scraped <meta>/http-equiv/OpenGraph names not mapped to a Property"); + /** * If a script element contains a src value, this value diff --git a/tika-core/src/main/java/org/apache/tika/metadata/MAPI.java b/tika-core/src/main/java/org/apache/tika/metadata/MAPI.java index c8f81a980d..92722afdbd 100644 --- a/tika-core/src/main/java/org/apache/tika/metadata/MAPI.java +++ b/tika-core/src/main/java/org/apache/tika/metadata/MAPI.java @@ -27,6 +27,7 @@ public interface MAPI { String PREFIX_MAPI_META = "mapi" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER; String PREFIX_MAPI_ATTACH_META = "mapi:attach" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER; String PREFIX_MAPI_PROPERTY = PREFIX_MAPI_META + "property" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER; + PassthroughPrefix PROPERTY = PassthroughPrefix.file(PREFIX_MAPI_PROPERTY, "MAPI named properties"); /** * MAPI message class. What type of .msg/MAPI file is it? diff --git a/tika-core/src/main/java/org/apache/tika/metadata/Message.java b/tika-core/src/main/java/org/apache/tika/metadata/Message.java index fcb1421f3c..bee626e0f6 100644 --- a/tika-core/src/main/java/org/apache/tika/metadata/Message.java +++ b/tika-core/src/main/java/org/apache/tika/metadata/Message.java @@ -27,6 +27,9 @@ public interface Message { String MESSAGE_RAW_HEADER_PREFIX = MESSAGE_PREFIX + "Raw-Header" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER; + PassthroughPrefix RAW_HEADER = PassthroughPrefix.file(MESSAGE_RAW_HEADER_PREFIX, + "RFC822 / Outlook raw email header names"); + String MESSAGE_RECIPIENT_ADDRESS = "Message-Recipient-Address"; String MESSAGE_FROM = "Message-From"; diff --git a/tika-core/src/main/java/org/apache/tika/metadata/Office.java b/tika-core/src/main/java/org/apache/tika/metadata/Office.java index 51e4ebb0d9..7240e241dc 100644 --- a/tika-core/src/main/java/org/apache/tika/metadata/Office.java +++ b/tika-core/src/main/java/org/apache/tika/metadata/Office.java @@ -37,6 +37,9 @@ public interface Office { */ String USER_DEFINED_METADATA_NAME_PREFIX = "custom:"; + PassthroughPrefix USER_DEFINED = PassthroughPrefix.file(USER_DEFINED_METADATA_NAME_PREFIX, + "OOXML/OLE2/ODF user-defined document properties"); + /** * Keywords pertaining to a document. Also populates {@link DublinCore#SUBJECT}. diff --git a/tika-core/src/main/java/org/apache/tika/metadata/PDF.java b/tika-core/src/main/java/org/apache/tika/metadata/PDF.java index 1f6d9c3284..34df110e43 100644 --- a/tika-core/src/main/java/org/apache/tika/metadata/PDF.java +++ b/tika-core/src/main/java/org/apache/tika/metadata/PDF.java @@ -51,6 +51,9 @@ public interface PDF { String PDF_DOC_INFO_CUSTOM_PREFIX = PDF_DOC_INFO_PREFIX + "custom" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER; + PassthroughPrefix DOC_INFO_CUSTOM = + PassthroughPrefix.file(PDF_DOC_INFO_CUSTOM_PREFIX, "PDF Info-dict custom keys"); + Property DOC_INFO_CREATED = Property.internalDate(PDF_DOC_INFO_PREFIX + "created"); Property DOC_INFO_CREATOR = Property.internalText(PDF_DOC_INFO_PREFIX + "creator"); diff --git a/tika-core/src/main/java/org/apache/tika/metadata/PassthroughPrefix.java b/tika-core/src/main/java/org/apache/tika/metadata/PassthroughPrefix.java new file mode 100644 index 0000000000..ea17d3b555 --- /dev/null +++ b/tika-core/src/main/java/org/apache/tika/metadata/PassthroughPrefix.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.metadata; + +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * A Tika-owned prefix under which keys are passed through from the source: the prefix is fixed, but + * each key name comes verbatim from the document or tool, so keys are unbounded and can't be + * {@link Property} constants. Declaring one self-registers it, so the open set is enumerable (as + * {@link Property} makes the closed set) and lintable: a String write is legitimate iff its key is a + * registered {@link Property} or its prefix is a registered {@code PassthroughPrefix}. + * + * @since Apache Tika 4.0.0 + */ +public final class PassthroughPrefix { + + public enum Provenance { FILE, TOOL } + + private static final Map<String, PassthroughPrefix> REGISTRY = new ConcurrentHashMap<>(); + + private final String prefix; + private final Provenance provenance; + private final String description; + + private PassthroughPrefix(String prefix, Provenance provenance, String description) { + this.prefix = prefix; + this.provenance = provenance; + this.description = description; + REGISTRY.put(prefix, this); + } + + public static PassthroughPrefix file(String prefix, String description) { + return new PassthroughPrefix(prefix, Provenance.FILE, description); + } + + public static PassthroughPrefix tool(String prefix, String description) { + return new PassthroughPrefix(prefix, Provenance.TOOL, description); + } + + /** The full key for a source-derived {@code suffix}. */ + public String key(String suffix) { + return prefix + suffix; + } + + public String prefix() { + return prefix; + } + + public Provenance provenance() { + return provenance; + } + + public String description() { + return description; + } + + /** Declared prefixes, from loaded classes only. */ + public static Collection<PassthroughPrefix> registered() { + return Collections.unmodifiableCollection(REGISTRY.values()); + } +} diff --git a/tika-metadata-schema/README.md b/tika-metadata-schema/README.md index e58f2a7083..105b4de77a 100644 --- a/tika-metadata-schema/README.md +++ b/tika-metadata-schema/README.md @@ -17,7 +17,14 @@ # tika-metadata-schema A machine-readable schema of Apache Tika's metadata keys. Two registries, because Tika has two -kinds of keys: +kinds of keys. + +**Scope: `tika-core` + the standard parser bundle.** The heavier/optional parser families +(scientific, sqlite3, nlp, vlm) are *not* scanned — pulling their runtime deps (netcdf, grib, +opennlp, DL4J, sqlite-jdbc) into a build-time schema module isn't worth it. Their keys are the only +ones absent (e.g. `sqlite3:`, `vlm:`, `grib:`, `netcdf:`, `ctakes:`, `NER_`). `MetadataCoverageTest` +enforces this: any module declaring keys that is neither scanned nor on its explicit out-of-scope +list fails the build, so nothing escapes *silently*. ## `metadata-keys.json` — the closed set (generated + gated) Every key Tika declares as a `Property` constant, plus the bounded digest cross-product @@ -29,22 +36,30 @@ declare a `Property` field, force-loads them, reads the global `Property` table, sorted JSON. `MetadataSchemaTest` regenerates in-memory and asserts it matches the committed file, so the registry can never drift from the declarations. -Regenerate after adding/changing a `Property`: +Regenerate after adding/changing a `Property` **or** a `PassthroughPrefix` (writes both files): ``` java -cp <tika-metadata-schema + deps classpath> \ org.apache.tika.metadata.schema.SchemaGenerator \ - src/main/resources/org/apache/tika/metadata/metadata-keys.json + src/main/resources/org/apache/tika/metadata/metadata-keys.json \ + src/main/resources/org/apache/tika/metadata/metadata-open-namespaces.json ``` -## `metadata-open-namespaces.json` — the open sets (curated) -Keys minted at **runtime** whose names are not `Property` constants, so they cannot be generated: -- **open/passthrough namespaces** — file-controlled key names (scraped HTML `<meta>` under `html:`, - OOXML `custom:`, email `Message:Raw-Header:`, Access `MDB_PROP:`, Vorbis comments, GRIB/NetCDF/FLV - attributes, …); -- **templates** — e.g. XMP `rdf:Alt` language variants `<base-key>:<lang>` (`dc:title:fr`). +## `metadata-open-namespaces.json` — the open sets (generated + gated) +The **prefixes** under which parsers mint file-controlled key names at runtime — names that are not +`Property` constants, so the individual keys cannot be enumerated (scraped HTML `<meta>` under +`html:`, OOXML `custom:`, email `Message:Raw-Header:`, Access `MDB_PROP:`, Vorbis comments, FLV +attributes, unmapped image/XMP tags, …). Each record: `{ prefix, provenance, description }`. + +**Generated from the `PassthroughPrefix` declarations, never hand-edited.** Every such prefix is a +registered `PassthroughPrefix` constant; `SchemaGenerator` reads that registry the same way it reads +the `Property` table, and `MetadataSchemaTest` gates it identically. Adding a passthrough prefix in a +parser and forgetting to regenerate fails the build. -**Curated (hand-maintained, reviewed), not generated**, and possibly not exhaustive; the -closed-namespace lint (a follow-up) is the intended completeness backstop. +Not covered here: **templates** — parameterized key families like XMP `rdf:Alt` language variants +`<base-key>:<lang>` (`dc:title:fr`), where the *suffix* rather than the prefix is open. These are +documented by rule, not enumerated. -Together the two files describe the whole key space: closed keys are enumerated and gated; open keys -are described by rule. +Together the files describe the key space of the scanned bundle: closed keys are enumerated and +gated; open namespaces are enumerated by prefix and gated; templates are described by rule; and +`MetadataCoverageTest` guarantees no scanned-bundle module is silently missed. Keys from the +out-of-scope families above are excluded by design. diff --git a/tika-metadata-schema/src/main/java/org/apache/tika/metadata/schema/SchemaGenerator.java b/tika-metadata-schema/src/main/java/org/apache/tika/metadata/schema/SchemaGenerator.java index 4b4d189491..cbfd598c7c 100644 --- a/tika-metadata-schema/src/main/java/org/apache/tika/metadata/schema/SchemaGenerator.java +++ b/tika-metadata-schema/src/main/java/org/apache/tika/metadata/schema/SchemaGenerator.java @@ -29,6 +29,7 @@ import java.util.jar.JarEntry; import java.util.jar.JarFile; import org.apache.tika.digest.DigestDef; +import org.apache.tika.metadata.PassthroughPrefix; import org.apache.tika.metadata.Property; /** @@ -42,9 +43,11 @@ import org.apache.tika.metadata.Property; */ public final class SchemaGenerator { - // Field/parameter descriptor for org.apache.tika.metadata.Property in a .class constant pool. + // Field/parameter descriptors in a .class constant pool: a class referencing one is force-loaded. private static final byte[] PROP_DESC = "Lorg/apache/tika/metadata/Property;".getBytes(StandardCharsets.ISO_8859_1); + private static final byte[] PASSTHROUGH_DESC = + "Lorg/apache/tika/metadata/PassthroughPrefix;".getBytes(StandardCharsets.ISO_8859_1); private SchemaGenerator() { } @@ -105,7 +108,8 @@ public final class SchemaGenerator { } private static void maybeLoad(String classPath, byte[] bytes, ClassLoader cl) { - if (!classPath.startsWith("org/apache/tika/") || !contains(bytes, PROP_DESC)) { + if (!classPath.startsWith("org/apache/tika/") + || (!contains(bytes, PROP_DESC) && !contains(bytes, PASSTHROUGH_DESC))) { return; } String cn = classPath.substring(0, classPath.length() - 6).replace('/', '.'); @@ -149,9 +153,28 @@ public final class SchemaGenerator { return '"' + s.replace("\\", "\\\\").replace("\"", "\\\"") + '"'; } + /** Declared passthrough prefixes as stable JSON. Call after {@link #generate()} has loaded classes. */ + public static String passthroughJson() { + TreeMap<String, String[]> m = new TreeMap<>(); + for (PassthroughPrefix p : PassthroughPrefix.registered()) { + m.put(p.prefix(), new String[]{p.provenance().name(), p.description()}); + } + StringBuilder sb = new StringBuilder("[\n"); + int i = 0; + int n = m.size(); + for (Map.Entry<String, String[]> e : m.entrySet()) { + sb.append(" {\"prefix\":").append(quote(e.getKey())) + .append(",\"provenance\":\"").append(e.getValue()[0]) + .append("\",\"description\":").append(quote(e.getValue()[1])) + .append("}").append(++i < n ? "," : "").append('\n'); + } + return sb.append("]\n").toString(); + } + public static void main(String[] args) throws Exception { - Path out = Path.of(args[0]); - Files.writeString(out, generate()); - System.out.println("wrote " + out); + Files.writeString(Path.of(args[0]), generate()); // triggers the classpath scan + if (args.length > 1) { + Files.writeString(Path.of(args[1]), passthroughJson()); + } } } diff --git a/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-open-namespaces.json b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-open-namespaces.json index 40eae02925..e2360ae6af 100644 --- a/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-open-namespaces.json +++ b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-open-namespaces.json @@ -1,38 +1,22 @@ -{ - "_note": "Open / passthrough namespaces and templates. These keys are minted at RUNTIME (their names come from the document, an external tool, or a per-item qualifier), so they are NOT registered Property constants and do NOT appear in metadata-keys.json. This file is CURATED (hand-maintained, reviewed), not generated, and may not be exhaustive; the closed-namespace lint is the intended completeness backstop. provenance: 'file' = key name taken from document content; 'tool' = key name [...] - - "open-namespaces": [ - {"prefix": "html:", "provenance": "file", "note": "scraped HTML <meta>/http-equiv/OpenGraph names"}, - {"prefix": "custom:", "provenance": "file", "note": "OOXML/OLE2/ODF user-defined document properties"}, - {"prefix": "pdf:docinfo:custom:", "provenance": "file", "note": "PDF Info-dictionary custom keys"}, - {"prefix": "Message:Raw-Header:", "provenance": "file", "note": "RFC822 / Outlook raw email header names"}, - {"prefix": "MboxParser-", "provenance": "file", "note": "mbox header names"}, - {"prefix": "warc:", "provenance": "file", "note": "WARC record + HTTP response header names (also warc:http:)"}, - {"prefix": "mapi:property:", "provenance": "file", "note": "MSG MAPI named properties"}, - {"prefix": "MDB_PROP:", "provenance": "file", "note": "MS Access document properties (also MDB_SUMMARY_PROP:, MDB_USER_PROP:)"}, - {"prefix": "img:", "provenance": "file", "note": "ImageIO/drewnoakes unrecognized image tag names (also ICC:)"}, - {"prefix": "dwg-custom:", "provenance": "file", "note": "DWG custom properties (DWGParser)"}, - {"prefix": "dwg:", "provenance": "file", "note": "DWGRead JSON field passthrough"}, - {"prefix": "iworks:", "provenance": "file", "note": "iWork '13 package plist keys"}, - {"prefix": "numbers:", "provenance": "file", "note": "iWork Numbers element names"}, - {"prefix": "flv:", "provenance": "file", "note": "FLV onMetaData ECMA-array keys"}, - {"prefix": "vorbis:", "provenance": "file", "note": "Vorbis/FLAC/Opus/Speex comment field names"}, - {"prefix": "streams-", "provenance": "file", "note": "Ogg per-stream metadata"}, - {"prefix": "grib:", "provenance": "file", "note": "GRIB global attribute names"}, - {"prefix": "netcdf:", "provenance": "file", "note": "NetCDF global attribute names"}, - {"prefix": "envi.", "provenance": "file", "note": "ENVI header field names"}, - {"prefix": "rtf_pict:", "provenance": "file", "note": "RTF embedded-object \\sn/\\sv keys"}, - {"prefix": "grobid:header_", "provenance": "tool", "note": "GROBID TEI header fields"}, - {"prefix": "NER_", "provenance": "tool", "note": "named-entity types (NamedEntityParser)"}, - {"prefix": "ctakes:", "provenance": "tool", "note": "cTAKES/UIMA type names"} - ], - - "templates": [ - { - "pattern": "<base-key>:<BCP-47-language-tag>", - "provenance": "file", - "note": "XMP rdf:Alt language variants, e.g. dc:title:x-default, dc:title:fr. The unqualified base key holds the x-default value; to type a variant, resolve to its base key. NOTE: the language qualifier reuses ':' so 'dc:title:fr' is only disambiguated from a sub-namespace by this rule.", - "example-base-keys": ["dc:title", "dc:description", "dc:rights", "xmpRights:UsageTerms"] - } - ] -} +[ + {"prefix":"ICC:","provenance":"FILE","description":"unrecognized ICC profile tag names"}, + {"prefix":"MDB_PROP:","provenance":"FILE","description":"MS Access database properties"}, + {"prefix":"MDB_SUMMARY_PROP:","provenance":"FILE","description":"MS Access summary properties"}, + {"prefix":"MDB_USER_PROP:","provenance":"FILE","description":"MS Access user-defined properties"}, + {"prefix":"MboxParser-","provenance":"FILE","description":"mbox email header names"}, + {"prefix":"Message:Raw-Header:","provenance":"FILE","description":"RFC822 / Outlook raw email header names"}, + {"prefix":"custom:","provenance":"FILE","description":"OOXML/OLE2/ODF user-defined document properties"}, + {"prefix":"dwg-custom:","provenance":"FILE","description":"DWG custom document properties"}, + {"prefix":"dwg:","provenance":"FILE","description":"DWGRead JSON header/summary field names"}, + {"prefix":"flv:","provenance":"FILE","description":"FLV onMetaData tag keys"}, + {"prefix":"html:","provenance":"FILE","description":"scraped <meta>/http-equiv/OpenGraph names not mapped to a Property"}, + {"prefix":"img:","provenance":"FILE","description":"unrecognized image tag names"}, + {"prefix":"iworks:","provenance":"FILE","description":"iWork '13 plist document properties"}, + {"prefix":"mapi:property:","provenance":"FILE","description":"MAPI named properties"}, + {"prefix":"numbers:","provenance":"FILE","description":"iWork Numbers document metadata element names"}, + {"prefix":"pdf:docinfo:custom:","provenance":"FILE","description":"PDF Info-dict custom keys"}, + {"prefix":"vorbis:","provenance":"FILE","description":"Vorbis comment field names"}, + {"prefix":"warc:","provenance":"FILE","description":"WARC record header names"}, + {"prefix":"warc:http:","provenance":"FILE","description":"WARC HTTP response header names"}, + {"prefix":"xmp-raw:","provenance":"FILE","description":"unmapped XMP keys"} +] diff --git a/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataCoverageTest.java b/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataCoverageTest.java new file mode 100644 index 0000000000..4980d770d1 --- /dev/null +++ b/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataCoverageTest.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.metadata.schema; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; + +/** + * Completeness backstop for the standard-scoped registry. {@link MetadataSchemaTest} proves the + * registry matches the declarations it <em>scans</em>; this proves nothing <em>escapes</em> the + * scan. It greps the source tree for every module declaring a {@code Property} or + * {@code PassthroughPrefix} and fails if one is neither on the registry classpath nor an explicit + * {@link #OUT_OF_SCOPE} family — so a new key-bearing parser cannot slip in silently. + * + * <p>Reads source (always present) for the declaring set and the runtime classpath for the scanned + * set, so it is correct in a targeted {@code -pl tika-metadata-schema} run — no full build needed. + */ +public class MetadataCoverageTest { + + /** Parser families intentionally excluded from the standard-scoped registry (heavy/optional + * deps). Adding a module here is a conscious scope decision; see README. */ + private static final Set<String> OUT_OF_SCOPE = Set.of( + "tika-parser-scientific-module", // envi. grib: netcdf: + "tika-parser-sqlite3-module", // sqlite3: + "tika-parser-nlp-module", // ctakes: grobid:header_ NER_ + "tika-vlm"); // vlm: + + // Source markers for an actual key declaration (a factory call), not a mere type reference. + private static final String[] DECL_MARKERS = { + "Property.internal", "Property.external", "Property.composite", + "PassthroughPrefix.file(", "PassthroughPrefix.tool("}; + + private static final String SRC_MAIN = sep("src", "main") + File.separator + "java"; + private static final String SRC_TEST = sep("src", "test"); + + @Test + public void everyKeyDeclaringModuleIsScannedOrExcluded() throws Exception { + Path repoRoot = repoRoot(); + assumeTrue(repoRoot != null, "repo root (tika-core + tika-parsers) not found"); + + Set<String> declaring = new TreeSet<>(); + for (String sub : new String[]{"tika-core", "tika-parsers"}) { + Path base = repoRoot.resolve(sub); + if (!Files.isDirectory(base)) { + continue; + } + try (Stream<Path> walk = Files.walk(base)) { + for (Path p : (Iterable<Path>) walk::iterator) { + String module = keyDeclaringModule(p); + if (module != null) { + declaring.add(module); + } + } + } + } + assumeTrue(!declaring.isEmpty(), "no key-declaring source found (unexpected checkout layout)"); + + Set<String> scanned = classpathModules(); + Set<String> escaped = new TreeSet<>(declaring); + escaped.removeAll(scanned); + escaped.removeAll(OUT_OF_SCOPE); + + assertTrue(escaped.isEmpty(), "Module(s) declare metadata keys but are neither scanned by " + + "the registry nor listed OUT_OF_SCOPE: " + escaped + ". Either add the module to " + + "the registry's scan classpath, or add it to OUT_OF_SCOPE (and document it)."); + } + + /** Module name if {@code p} is a main-source file declaring a key, else null. */ + private static String keyDeclaringModule(Path p) throws Exception { + String s = p.toString(); + int i = s.indexOf(SRC_MAIN); + if (!s.endsWith(".java") || i < 0 || s.contains(SRC_TEST)) { + return null; + } + String src = Files.readString(p, StandardCharsets.UTF_8); + boolean declares = false; + for (String marker : DECL_MARKERS) { + if (src.contains(marker)) { + declares = true; + break; + } + } + if (!declares) { + return null; + } + // s.substring(0, i) is the module dir itself (the part before ".../src/main/java/...") + Path moduleDir = Paths.get(s.substring(0, i)); + return moduleDir.getFileName() == null ? null : moduleDir.getFileName().toString(); + } + + /** Artifacts the registry scan sees: reactor {@code target/classes} dirs plus dependency jars + * (Tika dir name == artifactId, so both keys line up with {@link #keyDeclaringModule}). */ + private static Set<String> classpathModules() { + Set<String> mods = new TreeSet<>(); + String classes = sep("target", "classes"); + for (String e : System.getProperty("java.class.path").split(File.pathSeparator)) { + Path p = Paths.get(e); + Path mod = null; + if (e.endsWith(classes)) { + mod = p.getParent().getParent(); // <mod>/target/classes -> <mod> + } else if (e.endsWith(".jar") && p.getParent() != null) { + mod = p.getParent().getParent(); // <artifactId>/<version>/x.jar -> <artifactId> + } + if (mod != null && mod.getFileName() != null) { + mods.add(mod.getFileName().toString()); + } + } + return mods; + } + + private static Path repoRoot() { + Path p = Paths.get("").toAbsolutePath(); + while (p != null) { + if (Files.isDirectory(p.resolve("tika-core")) && Files.isDirectory(p.resolve("tika-parsers"))) { + return p; + } + p = p.getParent(); + } + return null; + } + + private static String sep(String a, String b) { + return File.separator + a + File.separator + b; + } +} diff --git a/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataSchemaTest.java b/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataSchemaTest.java index eff7626df5..25f26c5fbd 100644 --- a/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataSchemaTest.java +++ b/tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataSchemaTest.java @@ -25,24 +25,35 @@ import java.nio.charset.StandardCharsets; import org.junit.jupiter.api.Test; /** - * The CI gate: regenerate the registry from the live {@link org.apache.tika.metadata.Property} - * declarations and assert it matches the committed {@code metadata-keys.json}. If a Property is - * added/changed/renamed without regenerating, this fails — the registry cannot silently drift. + * The CI gate: regenerate both registries from the live {@link org.apache.tika.metadata.Property} + * and {@link org.apache.tika.metadata.PassthroughPrefix} declarations and assert each matches its + * committed file. If a declaration is added/changed/renamed without regenerating, this fails — the + * registries cannot silently drift. */ public class MetadataSchemaTest { - private static final String RESOURCE = "/org/apache/tika/metadata/metadata-keys.json"; + private static final String KEYS = "/org/apache/tika/metadata/metadata-keys.json"; + private static final String OPEN = "/org/apache/tika/metadata/metadata-open-namespaces.json"; @Test - public void committedRegistryMatchesDeclarations() throws Exception { - String committed; - try (InputStream in = MetadataSchemaTest.class.getResourceAsStream(RESOURCE)) { - assertNotNull(in, "committed " + RESOURCE + " is missing"); - committed = new String(in.readAllBytes(), StandardCharsets.UTF_8); + public void committedKeysMatchDeclarations() throws Exception { + assertEquals(committed(KEYS), SchemaGenerator.generate(), + "metadata-keys.json is stale. Run SchemaGenerator.main and commit the result."); + } + + @Test + public void committedOpenNamespacesMatchDeclarations() throws Exception { + // generate() first, so the classpath scan force-loads the PassthroughPrefix declarations. + SchemaGenerator.generate(); + assertEquals(committed(OPEN), SchemaGenerator.passthroughJson(), + "metadata-open-namespaces.json is stale. Run SchemaGenerator.main and commit the " + + "result."); + } + + private static String committed(String resource) throws Exception { + try (InputStream in = MetadataSchemaTest.class.getResourceAsStream(resource)) { + assertNotNull(in, "committed " + resource + " is missing"); + return new String(in.readAllBytes(), StandardCharsets.UTF_8); } - String regenerated = SchemaGenerator.generate(); - assertEquals(committed, regenerated, - "metadata-keys.json is stale. Run `mvn -pl tika-metadata-schema exec:java` " - + "(or SchemaGenerator.main) and commit the regenerated file."); } } diff --git a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/envi/EnviHeaderParser.java b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/envi/EnviHeaderParser.java index 22565e5fd0..a4d902c882 100644 --- a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/envi/EnviHeaderParser.java +++ b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/envi/EnviHeaderParser.java @@ -34,6 +34,7 @@ import org.apache.tika.detect.EncodingDetector; import org.apache.tika.exception.TikaException; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.PassthroughPrefix; import org.apache.tika.mime.MediaType; import org.apache.tika.parser.AbstractEncodingDetectorParser; import org.apache.tika.parser.ParseContext; @@ -43,6 +44,8 @@ import org.apache.tika.sax.XHTMLContentHandler; public class EnviHeaderParser extends AbstractEncodingDetectorParser { public static final String ENVI_MIME_TYPE = "application/envi.hdr"; + public static final PassthroughPrefix ENVI = + PassthroughPrefix.file("envi.", "ENVI header field names"); private static final long serialVersionUID = -1479368523072408091L; private static final Logger LOG = LoggerFactory.getLogger(EnviHeaderParser.class); private static final Set<MediaType> SUPPORTED_TYPES = @@ -118,7 +121,7 @@ public class EnviHeaderParser extends AbstractEncodingDetectorParser { if (keyValue[0].trim().equals("map info")) { String[] mapInfoValues = parseMapInfoContents(keyValue[1]); if (mapInfoValues[0].equals("UTM")) { - metadata.set("envi." + keyValue[0].trim().replace(" ", "."), + metadata.set(ENVI.key(keyValue[0].trim().replace(" ", ".")), keyValue[1].trim()); String[] latLonStringArray = convertMapInfoValuesToLatLngAndSetMetadata(mapInfoValues, metadata); @@ -128,11 +131,11 @@ public class EnviHeaderParser extends AbstractEncodingDetectorParser { xhtml.characters(xhtmlLatLongLine); xhtml.endElement("p"); } else { - metadata.set("envi." + keyValue[0].trim().replace(" ", "."), + metadata.set(ENVI.key(keyValue[0].trim().replace(" ", ".")), keyValue[1].trim()); } } else { - metadata.set("envi." + keyValue[0].trim().replace(" ", "."), + metadata.set(ENVI.key(keyValue[0].trim().replace(" ", ".")), keyValue[1].trim()); } } @@ -211,7 +214,7 @@ public class EnviHeaderParser extends AbstractEncodingDetectorParser { double zoneCM = (zone > 0) ? 6 * zone - 183.0 : 3.0; double latitude = 180.0 * (phi1 - fact1 * (fact2 + fact3 + fact4)) / Math.PI; double longitude = zoneCM - _a3; - metadata.set("envi.lat/lon", latitude + ", " + longitude); + metadata.set(ENVI.key("lat/lon"), latitude + ", " + longitude); return new String[]{Double.toString(latitude), Double.toString(longitude)}; } diff --git a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/grib/GribParser.java b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/grib/GribParser.java index 1f98243e06..073eb7bf25 100644 --- a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/grib/GribParser.java +++ b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/grib/GribParser.java @@ -37,6 +37,7 @@ import org.apache.tika.exception.TikaException; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.ClimateForcast; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.PassthroughPrefix; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.mime.MediaType; import org.apache.tika.parser.ParseContext; @@ -47,6 +48,8 @@ import org.apache.tika.sax.XHTMLContentHandler; public class GribParser implements Parser { public static final String GRIB_MIME_TYPE = "application/x-grib2"; + public static final PassthroughPrefix GRIB = + PassthroughPrefix.file("grib:", "GRIB global attribute names"); private static final long serialVersionUID = 7855458954474247655L; private final Set<MediaType> SUPPORTED_TYPES = Collections.singleton(MediaType.application("x-grib2")); @@ -134,7 +137,7 @@ public class GribParser implements Parser { } else if (CF_GLOBAL_ATTRIBUTES.contains(name)) { metadata.add(name, value); } else { - metadata.add("grib:" + name, value); + metadata.add(GRIB.key(name), value); } } diff --git a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/netcdf/NetCDFParser.java b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/netcdf/NetCDFParser.java index f40f31215c..fe2300a976 100644 --- a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/netcdf/NetCDFParser.java +++ b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/netcdf/NetCDFParser.java @@ -35,6 +35,7 @@ import org.apache.tika.exception.TikaException; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.ClimateForcast; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.PassthroughPrefix; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.mime.MediaType; import org.apache.tika.parser.ParseContext; @@ -56,6 +57,9 @@ public class NetCDFParser implements Parser { */ private static final long serialVersionUID = -5940938274907708665L; + public static final PassthroughPrefix NETCDF = + PassthroughPrefix.file("netcdf:", "NetCDF global attribute names"); + private final Set<MediaType> SUPPORTED_TYPES = Collections.singleton(MediaType.application("x-netcdf")); @@ -81,7 +85,7 @@ public class NetCDFParser implements Parser { ParseContext context) throws IOException, SAXException, TikaException { try (NetcdfFile ncFile = NetcdfFile.open(tis.getFile().getAbsolutePath())) { - metadata.set("netcdf:File-Type-Description", ncFile.getFileTypeDescription()); + metadata.set(NETCDF.key("File-Type-Description"), ncFile.getFileTypeDescription()); // first parse out the set of global attributes for (Attribute attr : ncFile.getGlobalAttributes()) { if (attr.getDataType().isString()) { @@ -142,7 +146,7 @@ public class NetCDFParser implements Parser { } else if (CF_GLOBAL_ATTRIBUTES.contains(name)) { metadata.add(name, value); } else { - metadata.add("netcdf:" + name, value); + metadata.add(NETCDF.key(name), value); } } } diff --git a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/main/java/org/apache/tika/parser/ctakes/CTAKESContentHandler.java b/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/main/java/org/apache/tika/parser/ctakes/CTAKESContentHandler.java index 1ee5476060..bb6ae8cdcf 100644 --- a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/main/java/org/apache/tika/parser/ctakes/CTAKESContentHandler.java +++ b/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/main/java/org/apache/tika/parser/ctakes/CTAKESContentHandler.java @@ -27,6 +27,7 @@ import org.xml.sax.SAXException; import org.xml.sax.helpers.DefaultHandler; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.PassthroughPrefix; import org.apache.tika.sax.ContentHandlerDecorator; /** @@ -41,6 +42,8 @@ import org.apache.tika.sax.ContentHandlerDecorator; public class CTAKESContentHandler extends ContentHandlerDecorator { // Prefix used for metadata including cTAKES annotations public static String CTAKES_META_PREFIX = "ctakes:"; + public static final PassthroughPrefix CTAKES = + PassthroughPrefix.tool(CTAKES_META_PREFIX, "cTAKES annotation type names"); // Configuration object for CTAKESContentHandler private CTAKESConfig config = null; @@ -132,7 +135,7 @@ public class CTAKESContentHandler extends ContentHandlerDecorator { ae.process(jcas); // add annotations to metadata - metadata.add(CTAKES_META_PREFIX + "schema", config.getAnnotationPropsAsString()); + metadata.add(CTAKES.key("schema"), config.getAnnotationPropsAsString()); CTAKESAnnotationProperty[] annotationPros = config.getAnnotationProps(); Collection<IdentifiedAnnotation> collection = JCasUtil.select(jcas, IdentifiedAnnotation.class); @@ -146,7 +149,7 @@ public class CTAKESContentHandler extends ContentHandlerDecorator { .append(CTAKESUtils.getAnnotationProperty(annotation, property)); } } - metadata.add(CTAKES_META_PREFIX + annotation.getType().getShortName(), + metadata.add(CTAKES.key(annotation.getType().getShortName()), annotationBuilder.toString()); } diff --git a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/main/java/org/apache/tika/parser/journal/GrobidRESTParser.java b/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/main/java/org/apache/tika/parser/journal/GrobidRESTParser.java index f6c761ac40..3eab5ee4de 100644 --- a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/main/java/org/apache/tika/parser/journal/GrobidRESTParser.java +++ b/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/main/java/org/apache/tika/parser/journal/GrobidRESTParser.java @@ -35,6 +35,7 @@ import org.xml.sax.ContentHandler; import org.apache.tika.annotation.TikaComponent; import org.apache.tika.exception.TikaException; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.PassthroughPrefix; import org.apache.tika.parser.ParseContext; /** @@ -46,6 +47,9 @@ public class GrobidRESTParser { private static final Logger LOG = LoggerFactory.getLogger(GrobidRESTParser.class); + public static final PassthroughPrefix GROBID_HEADER = + PassthroughPrefix.tool("grobid:header_", "GROBID-inferred header fields"); + private static final String GROBID_REST_HOST = "http://localhost:8070";; private static final String GROBID_ISALIVE_PATH = "/api/isalive"; private static final String GROBID_PROCESSHEADER_PATH = "/api/processHeaderDocument"; @@ -110,7 +114,7 @@ public class GrobidRESTParser { String resp = response.readEntity(String.class); Metadata teiMet = new TEIDOMParser().parse(resp, context); for (String key : teiMet.names()) { - metadata.add("grobid:header_" + key, teiMet.get(key)); + metadata.add(GROBID_HEADER.key(key), teiMet.get(key)); } } catch (Exception e) { LOG.warn("Couldn't read response", e); diff --git a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/main/java/org/apache/tika/parser/ner/NamedEntityParser.java b/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/main/java/org/apache/tika/parser/ner/NamedEntityParser.java index 31ca528f5e..52df871ded 100644 --- a/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/main/java/org/apache/tika/parser/ner/NamedEntityParser.java +++ b/tika-parsers/tika-parsers-ml/tika-parser-nlp-module/src/main/java/org/apache/tika/parser/ner/NamedEntityParser.java @@ -38,6 +38,7 @@ import org.apache.tika.config.loader.TikaLoader; import org.apache.tika.exception.TikaException; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.PassthroughPrefix; import org.apache.tika.mime.MediaType; import org.apache.tika.parser.ParseContext; import org.apache.tika.parser.Parser; @@ -61,6 +62,8 @@ public class NamedEntityParser implements Parser { public static final Logger LOG = LoggerFactory.getLogger(NamedEntityParser.class); public static final Set<MediaType> MEDIA_TYPES = new HashSet<>(); public static final String MD_KEY_PREFIX = "NER_"; + public static final PassthroughPrefix NER = + PassthroughPrefix.tool(MD_KEY_PREFIX, "named-entity types"); public static final String DEFAULT_NER_IMPL = OpenNLPNERecogniser.class.getName() + "," + RegexNERecogniser.class.getName(); public static final String SYS_PROP_NER_IMPL = "ner.impl.class"; @@ -137,7 +140,7 @@ public class NamedEntityParser implements Parser { if (names != null) { for (Map.Entry<String, Set<String>> entry : names.entrySet()) { if (entry.getValue() != null) { - String mdKey = MD_KEY_PREFIX + entry.getKey(); + String mdKey = NER.key(entry.getKey()); for (String name : entry.getValue()) { metadata.add(mdKey, name); } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/NumbersContentHandler.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/NumbersContentHandler.java index bb5ffa52a0..87237ffca9 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/NumbersContentHandler.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/NumbersContentHandler.java @@ -25,11 +25,15 @@ import org.xml.sax.helpers.DefaultHandler; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.Office; +import org.apache.tika.metadata.PassthroughPrefix; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.sax.XHTMLContentHandler; class NumbersContentHandler extends DefaultHandler { + private static final PassthroughPrefix NUMBERS_METADATA = + PassthroughPrefix.file("numbers:", "iWork Numbers document metadata element names"); + private final XHTMLContentHandler xhtml; private final Metadata metadata; @@ -226,7 +230,7 @@ class NumbersContentHandler extends DefaultHandler { } else if ("comment".equals(localName)) { metadata.add(TikaCoreProperties.COMMENTS, value); } else { - metadata.add("numbers:" + localName, value); + metadata.add(NUMBERS_METADATA.key(localName), value); } } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/iwana/IWork13PackageParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/iwana/IWork13PackageParser.java index 4be10adc95..1ffb5fa5ec 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/iwana/IWork13PackageParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/iwana/IWork13PackageParser.java @@ -45,6 +45,7 @@ import org.apache.tika.extractor.EmbeddedDocumentUtil; import org.apache.tika.io.TemporaryResources; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.PassthroughPrefix; import org.apache.tika.metadata.Property; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.mime.MediaType; @@ -66,6 +67,8 @@ public class IWork13PackageParser implements Parser { Property.externalText(IWORKS_PREFIX + "document-id"); public static final Property IWORKS_BUILD_VERSION_HISTORY = Property.externalTextBag(IWORKS_PREFIX + "build-version-history"); + public static final PassthroughPrefix IWORKS_PROPERTIES = + PassthroughPrefix.file(IWORKS_PREFIX, "iWork '13 plist document properties"); private final static Set<MediaType> supportedTypes = Collections.unmodifiableSet( @@ -236,7 +239,7 @@ public class IWork13PackageParser implements Parser { NSDictionary dict = (NSDictionary)rootObj; for (String k : dict.keySet()) { String v = dict.get(k).toString(); - metadata.set(IWORKS_PREFIX + k, v); + metadata.set(IWORKS_PROPERTIES.key(k), v); } } } catch (SecurityException e) { diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/ogg/OggAudioParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/ogg/OggAudioParser.java index a10238c933..c590e43400 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/ogg/OggAudioParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/ogg/OggAudioParser.java @@ -34,6 +34,7 @@ import org.xml.sax.SAXException; import org.apache.tika.exception.TikaException; import org.apache.tika.metadata.Audio; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.PassthroughPrefix; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.metadata.XMP; import org.apache.tika.metadata.XMPDM; @@ -48,6 +49,9 @@ import org.apache.tika.sax.XHTMLContentHandler; public abstract class OggAudioParser extends AbstractParser { private static final long serialVersionUID = 5168743829615945633L; + private static final PassthroughPrefix VORBIS = + PassthroughPrefix.file("vorbis:", "Vorbis comment field names"); + /** * Returns the first positive integer found under the given comment keys, @@ -118,7 +122,7 @@ public abstract class OggAudioParser extends AbstractParser { for (String key : comments.getAllComments().keySet()) { if (!done.contains(key)) { for (String value : comments.getAllComments().get(key)) { - metadata.add("vorbis:" + key, value); + metadata.add(VORBIS.key(key), value); } } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/video/FLVParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/video/FLVParser.java index a3c36ebeea..2feb07cad1 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/video/FLVParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/video/FLVParser.java @@ -36,6 +36,7 @@ import org.apache.tika.annotation.TikaComponent; import org.apache.tika.exception.TikaException; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.PassthroughPrefix; import org.apache.tika.mime.MediaType; import org.apache.tika.parser.ParseContext; import org.apache.tika.parser.Parser; @@ -76,6 +77,8 @@ public class FLVParser implements Parser { private static int TYPE_METADATA = 0x12; private static byte MASK_AUDIO = 1; private static byte MASK_VIDEO = 4; + private static final PassthroughPrefix FLV = + PassthroughPrefix.file("flv:", "FLV onMetaData tag keys"); public Set<MediaType> getSupportedTypes(ParseContext context) { return SUPPORTED_TYPES; @@ -245,7 +248,7 @@ public class FLVParser implements Parser { if (entry.getValue() == null) { continue; } - metadata.set("flv:" + entry.getKey(), entry.getValue().toString()); + metadata.set(FLV.key(entry.getKey()), entry.getValue().toString()); } } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-cad-module/src/main/java/org/apache/tika/parser/dwg/DWGParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-cad-module/src/main/java/org/apache/tika/parser/dwg/DWGParser.java index b5db58cd32..a03578d0eb 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-cad-module/src/main/java/org/apache/tika/parser/dwg/DWGParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-cad-module/src/main/java/org/apache/tika/parser/dwg/DWGParser.java @@ -32,6 +32,7 @@ import org.apache.tika.exception.TikaException; import org.apache.tika.io.EndianUtils; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.PassthroughPrefix; import org.apache.tika.metadata.Property; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.mime.MediaType; @@ -60,6 +61,8 @@ public class DWGParser extends AbstractDWGParser { } public static String DWG_CUSTOM_META_PREFIX = "dwg-custom:"; + public static final PassthroughPrefix DWG_CUSTOM = + PassthroughPrefix.file(DWG_CUSTOM_META_PREFIX, "DWG custom document properties"); /** * Serial version UID */ @@ -169,7 +172,7 @@ public class DWGParser extends AbstractDWGParser { String propName = read2004String(tis); String propValue = read2004String(tis); if (propName.length() > 0 && propValue.length() > 0) { - metadata.add(DWG_CUSTOM_META_PREFIX + propName, propValue); + metadata.add(DWG_CUSTOM.key(propName), propValue); } } } @@ -204,7 +207,7 @@ public class DWGParser extends AbstractDWGParser { String propName = read2007and2010String(tis); String propValue = read2007and2010String(tis); if (propName.length() > 0 && propValue.length() > 0) { - metadata.add(DWG_CUSTOM_META_PREFIX + propName, propValue); + metadata.add(DWG_CUSTOM.key(propName), propValue); } } } @@ -255,7 +258,7 @@ public class DWGParser extends AbstractDWGParser { if (splitAt > -1) { String propName = val.substring(0, splitAt); String propVal = val.substring(splitAt + 1); - metadata.add(DWGParser.DWG_CUSTOM_META_PREFIX + propName, propVal); + metadata.add(DWG_CUSTOM.key(propName), propVal); } } } else { diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-cad-module/src/main/java/org/apache/tika/parser/dwg/DWGReadParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-cad-module/src/main/java/org/apache/tika/parser/dwg/DWGReadParser.java index f774723486..3ef16fe2cf 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-cad-module/src/main/java/org/apache/tika/parser/dwg/DWGReadParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-cad-module/src/main/java/org/apache/tika/parser/dwg/DWGReadParser.java @@ -294,7 +294,7 @@ public class DWGReadParser extends AbstractDWGParser { if (nextToken.isStructStart()) { jsonParser.skipChildren(); } else if (nextToken.isScalarValue()) { - metadata.set("dwg:" + nextFieldName, jsonParser.getText()); + metadata.set(DWG.RAW_FIELD.key(nextFieldName), jsonParser.getText()); } } } @@ -347,7 +347,7 @@ public class DWGReadParser extends AbstractDWGParser { } else if ("HYPERLINKBASE".equals(nextFieldName)) { metadata.set(TikaCoreProperties.RELATION, textVal); } else if (!Strings.CI.startsWith(nextFieldName, "unknown")) { - metadata.set("dwg:" + nextFieldName, textVal); + metadata.set(DWG.RAW_FIELD.key(nextFieldName), textVal); } } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java index 12527c4734..4008c39910 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java @@ -73,6 +73,7 @@ import org.apache.tika.exception.TikaException; import org.apache.tika.metadata.Geographic; import org.apache.tika.metadata.IPTC; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.PassthroughPrefix; import org.apache.tika.metadata.Property; import org.apache.tika.metadata.TIFF; import org.apache.tika.metadata.TikaCoreProperties; @@ -92,6 +93,10 @@ public class ImageMetadataExtractor { public static final String UNKNOWN_IMG_NS = "img" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER; public static final String ICC_NS = "ICC" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER; + public static final PassthroughPrefix UNKNOWN_IMG = + PassthroughPrefix.file(UNKNOWN_IMG_NS, "unrecognized image tag names"); + public static final PassthroughPrefix ICC = + PassthroughPrefix.file(ICC_NS, "unrecognized ICC profile tag names"); private final Metadata metadata; private DirectoryHandler[] handlers; @@ -292,11 +297,11 @@ public class ImageMetadataExtractor { value = Boolean.FALSE.toString(); } if (directory instanceof ExifDirectoryBase) { - metadata.set(UNKNOWN_IMG_NS + directory.getName() + ":" + name, value); + metadata.set(UNKNOWN_IMG.key(directory.getName() + ":" + name), value); } else if (directory instanceof IccDirectory) { - metadata.set(ICC_NS + name, value); + metadata.set(ICC.key(name), value); } else { - metadata.set(UNKNOWN_IMG_NS + name, value); + metadata.set(UNKNOWN_IMG.key(name), value); } } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java index 2f32aa5ec2..2dbf170060 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java @@ -370,7 +370,7 @@ class MailContentHandler implements ContentHandler { metadata.set(Message.MULTIPART_BOUNDARY, contentType.getParameters().get("boundary")); } else { - metadata.add(Metadata.MESSAGE_RAW_HEADER_PREFIX + parsedField.getName(), + metadata.add(Message.RAW_HEADER.key(parsedField.getName()), field.getBody()); } } else if (fieldname.equalsIgnoreCase("Date")) { @@ -385,7 +385,7 @@ class MailContentHandler implements ContentHandler { //swallow } } else { - metadata.add(Metadata.MESSAGE_RAW_HEADER_PREFIX + parsedField.getName(), + metadata.add(Message.RAW_HEADER.key(parsedField.getName()), field.getBody()); } } catch (RuntimeException me) { diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mbox/MboxParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mbox/MboxParser.java index 7e97ac764e..c5f898da29 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mbox/MboxParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mbox/MboxParser.java @@ -43,6 +43,7 @@ import org.apache.tika.extractor.EmbeddedDocumentUtil; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Message; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.PassthroughPrefix; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.mime.MediaType; import org.apache.tika.parser.ParseContext; @@ -71,6 +72,8 @@ public class MboxParser implements Parser { private static final Pattern EMAIL_ADDRESS_PATTERN = Pattern.compile("<(.*@.*)>"); private static final String EMAIL_HEADER_METADATA_PREFIX = "MboxParser-"; + private static final PassthroughPrefix EMAIL_HEADER = + PassthroughPrefix.file(EMAIL_HEADER_METADATA_PREFIX, "mbox email header names"); private static final String EMAIL_FROMLINE_METADATA = EMAIL_HEADER_METADATA_PREFIX + "from"; private final Map<Integer, Metadata> trackingMetadata = new HashMap<>(); private boolean tracking = false; @@ -224,7 +227,7 @@ public class MboxParser implements Parser { metadata.add(Metadata.CONTENT_TYPE, headerContent); metadata.set(TikaCoreProperties.FORMAT, headerContent); } else { - metadata.add(EMAIL_HEADER_METADATA_PREFIX + headerTag, headerContent); + metadata.add(EMAIL_HEADER.key(headerTag), headerContent); } } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/JackcessExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/JackcessExtractor.java index c302185b56..0a339a21c3 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/JackcessExtractor.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/JackcessExtractor.java @@ -102,13 +102,13 @@ class JackcessExtractor extends AbstractPOIFSExtractor { db.setDateTimeType(DateTimeType.DATE); PropertyMap dbp = db.getDatabaseProperties(); for (PropertyMap.Property p : dbp) { - parentMetadata.add(JackcessParser.MDB_PROPERTY_PREFIX + p.getName(), + parentMetadata.add(JackcessParser.MDB_PROPERTY.key(p.getName()), toString(p.getValue(), p.getType())); } PropertyMap up = db.getUserDefinedProperties(); for (PropertyMap.Property p : up) { - parentMetadata.add(JackcessParser.USER_DEFINED_PROPERTY_PREFIX + p.getName(), + parentMetadata.add(JackcessParser.MDB_USER_PROPERTY.key(p.getName()), toString(p.getValue(), p.getType())); } @@ -137,7 +137,7 @@ class JackcessExtractor extends AbstractPOIFSExtractor { for (PropertyMap.Property p : db.getSummaryProperties()) { if (!found.contains(p.getName())) { - parentMetadata.add(JackcessParser.SUMMARY_PROPERTY_PREFIX + p.getName(), + parentMetadata.add(JackcessParser.MDB_SUMMARY_PROPERTY.key(p.getName()), toString(p.getValue(), p.getType())); } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/JackcessParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/JackcessParser.java index 3110640549..99a8e4e911 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/JackcessParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/JackcessParser.java @@ -35,6 +35,7 @@ import org.apache.tika.exception.TikaException; import org.apache.tika.exception.UnsupportedFormatException; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.PassthroughPrefix; import org.apache.tika.metadata.Property; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.mime.MediaType; @@ -55,6 +56,8 @@ public class JackcessParser implements Parser { public static final String SUMMARY_PROPERTY_PREFIX = "MDB_SUMMARY_PROP" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER; + public static final PassthroughPrefix MDB_SUMMARY_PROPERTY = + PassthroughPrefix.file(SUMMARY_PROPERTY_PREFIX, "MS Access summary properties"); private final static LinkResolver IGNORE_LINK_RESOLVER = new IgnoreLinkResolver(); private static final long serialVersionUID = -752276948656079347L; private static final MediaType MEDIA_TYPE = MediaType.application("x-msaccess"); @@ -64,8 +67,12 @@ public class JackcessParser implements Parser { // public static Property LINKED_DATABASES = Property.externalTextBag("LinkedDatabases"); public static String MDB_PROPERTY_PREFIX = "MDB_PROP" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER; + public static final PassthroughPrefix MDB_PROPERTY = + PassthroughPrefix.file(MDB_PROPERTY_PREFIX, "MS Access database properties"); public static String USER_DEFINED_PROPERTY_PREFIX = "MDB_USER_PROP" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER; + public static final PassthroughPrefix MDB_USER_PROPERTY = + PassthroughPrefix.file(USER_DEFINED_PROPERTY_PREFIX, "MS Access user-defined properties"); public static Property MDB_PW = Property.externalText("Password"); private Locale locale = Locale.ROOT; diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/SummaryExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/SummaryExtractor.java index 89f1913fad..20c5676051 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/SummaryExtractor.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/SummaryExtractor.java @@ -192,8 +192,7 @@ public class SummaryExtractor { private void parse(CustomProperties customProperties) { if (customProperties != null) { for (String name : customProperties.nameSet()) { - // Apply the custom prefix - String key = Office.USER_DEFINED_METADATA_NAME_PREFIX + name; + String key = Office.USER_DEFINED.key(name); // Get, convert and save property value Object value = customProperties.get(name); diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/msg/ExtendedMetadataExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/msg/ExtendedMetadataExtractor.java index a9f35695b4..1e39990fd9 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/msg/ExtendedMetadataExtractor.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/msg/ExtendedMetadataExtractor.java @@ -148,7 +148,7 @@ public class ExtendedMetadataExtractor { if (!includeType(propertyValue)) { return; } - String key = MAPI.PREFIX_MAPI_PROPERTY + pair.tikaMapiProperty.name; + String key = MAPI.PROPERTY.key(pair.tikaMapiProperty.name); Types.MAPIType type = propertyValue.getActualType(); if (type == Types.TIME || type == Types.MV_TIME || type == Types.APP_TIME || type == Types.MV_APP_TIME) { Calendar calendar = (Calendar) propertyValue.getValue(); diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentMetaParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentMetaParser.java index 765a58a3ac..a40b06e09d 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentMetaParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentMetaParser.java @@ -84,7 +84,7 @@ public class OpenDocumentMetaParser extends XMLParser { // custom:Info1=Text1 ContentHandler branch = new MatchingContentHandler( new AttributeDependantMetadataHandler(md, "meta:name", - Office.USER_DEFINED_METADATA_NAME_PREFIX), matcher); + Office.USER_DEFINED.prefix()), matcher); return new TeeContentHandler(ch, branch); } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java index a2ccb629e6..04916b5567 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java @@ -665,7 +665,7 @@ public class PDFParser implements Parser, RenderingParser { for (COSName key : info.getCOSObject().keySet()) { String name = key.getName(); if (!handledMetadata.contains(name)) { - PDMetadataExtractor.addMetadata(metadata, PDF.PDF_DOC_INFO_CUSTOM_PREFIX + name, + PDMetadataExtractor.addMetadata(metadata, PDF.DOC_INFO_CUSTOM.key(name), info.getCOSObject().getDictionaryObject(key)); } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-webarchive-module/src/main/java/org/apache/tika/parser/warc/WARCParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-webarchive-module/src/main/java/org/apache/tika/parser/warc/WARCParser.java index 40ac9a1beb..7206d6c7ca 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-webarchive-module/src/main/java/org/apache/tika/parser/warc/WARCParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-webarchive-module/src/main/java/org/apache/tika/parser/warc/WARCParser.java @@ -40,6 +40,7 @@ import org.apache.tika.extractor.EmbeddedDocumentExtractor; import org.apache.tika.extractor.EmbeddedDocumentUtil; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.PassthroughPrefix; import org.apache.tika.metadata.Property; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.metadata.WARC; @@ -64,6 +65,11 @@ public class WARCParser implements Parser { public static String WARC_PREFIX = "warc:"; public static String WARC_HTTP_PREFIX = WARC_PREFIX + "http:"; + public static final PassthroughPrefix WARC_HEADER = + PassthroughPrefix.file(WARC_PREFIX, "WARC record header names"); + public static final PassthroughPrefix WARC_HTTP_HEADER = + PassthroughPrefix.file(WARC_HTTP_PREFIX, "WARC HTTP response header names"); + public static String WARC_HTTP_STATUS = WARC_HTTP_PREFIX + "status"; public static String WARC_HTTP_STATUS_REASON = WARC_HTTP_PREFIX + "status:reason"; @@ -159,7 +165,7 @@ public class WARCParser implements Parser { private void processWarcMetadata(WarcResponse warcResponse, Metadata metadata) { for (Map.Entry<String, List<String>> e : warcResponse.headers().map().entrySet()) { for (String val : e.getValue()) { - metadata.add(WARC_PREFIX + e.getKey(), val); + metadata.add(WARC_HEADER.key(e.getKey()), val); } } } @@ -171,7 +177,7 @@ public class WARCParser implements Parser { } for (Map.Entry<String, List<String>> e : http.headers().map().entrySet()) { for (String val : e.getValue()) { - metadata.add(WARC_HTTP_PREFIX + e.getKey(), val); + metadata.add(WARC_HTTP_HEADER.key(e.getKey()), val); } } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/src/main/java/org/apache/tika/parser/xmp/XmpExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/src/main/java/org/apache/tika/parser/xmp/XmpExtractor.java index 9ae8b4b325..b0db5d99cf 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/src/main/java/org/apache/tika/parser/xmp/XmpExtractor.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/src/main/java/org/apache/tika/parser/xmp/XmpExtractor.java @@ -39,6 +39,7 @@ import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.Office; import org.apache.tika.metadata.PDF; import org.apache.tika.metadata.PagedText; +import org.apache.tika.metadata.PassthroughPrefix; import org.apache.tika.metadata.Photoshop; import org.apache.tika.metadata.Property; import org.apache.tika.metadata.TIFF; @@ -76,6 +77,7 @@ public class XmpExtractor { // X-TIKA: field. Best-effort discovery surface: keys use the document's prefix (not the URI) // and are non-contractual -- promote a field into TABLE when it needs a stable key. static final String RAW_PREFIX = "xmp-raw:"; + static final PassthroughPrefix RAW = PassthroughPrefix.file(RAW_PREFIX, "unmapped XMP keys"); // strip a trailing array index so a raw bag/seq is one multi-valued key, not foo:Bag[1], foo:Bag[2] private static final Pattern TRAILING_INDEX = Pattern.compile("\\[\\d+\\]$"); @@ -425,7 +427,7 @@ public class XmpExtractor { return; } // unmapped: namespaced raw passthrough; a trailing array index collapses to a multi-valued key - metadata.add(RAW_PREFIX + TRAILING_INDEX.matcher(p.path).replaceFirst(""), value); + metadata.add(RAW.key(TRAILING_INDEX.matcher(p.path).replaceFirst("")), value); } private void emit(Metadata metadata, Property prop, String value) {
