This is an automated email from the ASF dual-hosted git repository.
tballison pushed a change to branch shim-for-tika-eval-for-pre-4.0.0
in repository https://gitbox.apache.org/repos/asf/tika.git
from 3ae1144884 add shim so that tika-eval works for 4.0.0 vs 4.0.0-beta-1
add 9b9e53ac62 TIKA-4794-option-3 (#2964)
add 0e33ca299b Merge remote-tracking branch 'origin/main' into
shim-for-tika-eval-for-pre-4.0.0
No new revisions were added by this update.
Summary of changes:
.skills/dev.md | 11 +
.skills/metadata-schema.md | 77 ++++
.skills/tika-eval-encoding-regression.md | 8 +-
.skills/tika-eval-h2-query.md | 2 +-
.../ROOT/pages/advanced/embedded-documents.adoc | 36 +-
.../advanced/integration-testing/tika-app.adoc | 4 +-
.../integration-testing/tika-eval-regression.adoc | 6 +-
.../advanced/integration-testing/tika-server.adoc | 2 +-
.../ROOT/pages/advanced/setting-limits.adoc | 18 +-
.../modules/ROOT/pages/advanced/zip-detection.adoc | 4 +-
.../ROOT/pages/configuration/digesters.adoc | 4 +-
.../pages/migration-to-4x/chunk-strategies.adoc | 24 +-
.../pages/migration-to-4x/metadata-changes-4x.adoc | 232 +++++++++++-
docs/modules/ROOT/pages/pipes/parse-modes.adoc | 18 +-
.../ROOT/pages/using-tika/server/index.adoc | 2 +-
.../java/org/apache/tika/cli/TikaCLIAsyncTest.java | 2 +-
.../test/java/org/apache/tika/cli/TikaCLITest.java | 20 +-
.../java/org/apache/tika/digest/DigestDef.java | 10 +-
.../apache/tika/digest/InputStreamDigester.java | 2 +-
.../apache/tika/metadata/AccessPermissions.java | 18 +-
.../main/java/org/apache/tika/metadata/DWG.java | 8 +-
.../java/org/apache/tika/metadata/Database.java | 8 +-
.../java/org/apache/tika/metadata/HttpHeaders.java | 16 +-
.../java/org/apache/tika/metadata/Message.java | 43 ++-
.../java/org/apache/tika/metadata/Metadata.java | 18 +-
.../main/java/org/apache/tika/metadata/Office.java | 6 +-
.../tika/metadata/OfficeOpenXMLExtended.java | 4 +-
.../main/java/org/apache/tika/metadata/PDF.java | 66 ++--
.../main/java/org/apache/tika/metadata/PST.java | 4 +-
.../java/org/apache/tika/metadata/QuattroPro.java | 10 +-
.../java/org/apache/tika/metadata/RTFMetadata.java | 29 +-
.../java/org/apache/tika/metadata/Rendering.java | 4 +-
.../apache/tika/metadata/ReservedNamespaces.java | 45 +++
.../apache/tika/metadata/TikaCoreProperties.java | 125 ++++---
.../org/apache/tika/metadata/TikaPagedText.java | 8 +-
.../java/org/apache/tika/metadata/WordPerfect.java | 14 +-
.../main/java/org/apache/tika/metadata/Zip.java | 22 +-
.../writefilter/StandardMetadataLimiter.java | 8 +-
.../sax/AbstractRecursiveParserWrapperHandler.java | 9 +-
.../java/org/apache/tika/utils/ParserUtils.java | 2 +-
.../tika/digest/InputStreamDigesterTest.java | 2 +-
.../metadata/MetadataInternalKeyGuardTest.java | 11 +-
.../tika/pipes/filesystem/HandlerTypeTest.java | 4 +-
.../org/apache/tika/eval/app/ExtractComparer.java | 4 +-
.../org/apache/tika/eval/app/ExtractProfiler.java | 2 +-
.../org/apache/tika/eval/app/ProfilerBase.java | 4 +-
.../java/org/apache/tika/eval/app/db/Cols.java | 2 +-
.../src/main/resources/comparison-reports-pg.xml | 2 +-
.../src/main/resources/comparison-reports.xml | 2 +-
.../src/test/resources/extracts/TIKA-4446-a.json | 2 +-
.../src/test/resources/extracts/TIKA-4446-b.json | 2 +-
.../resources/test-dirs/extractsA/file1.pdf.json | 2 +-
.../test-dirs/extractsA/file12_es.txt.json | 2 +-
.../extractsA/file13_attachANotB.doc.json | 6 +-
.../extractsA/file14_diffAttachOrder.json | 20 +-
.../resources/test-dirs/extractsA/file15_tags.json | 26 +-
.../test-dirs/extractsA/file16_badTags.json | 26 +-
.../test-dirs/extractsA/file17_tagsOutOfOrder.json | 26 +-
.../test-dirs/extractsA/file2_attachANotB.doc.json | 8 +-
.../test-dirs/extractsA/file3_attachBNotA.doc.json | 2 +-
.../test-dirs/extractsA/file4_emptyB.pdf.json | 2 +-
.../test-dirs/extractsA/file6_accessEx.pdf.json | 18 +-
.../test-dirs/extractsA/file7_badJson.pdf.json | 2 +-
.../test-dirs/extractsA/file8_IOEx.pdf.json | 18 +-
.../resources/test-dirs/extractsB/file1.pdf.json | 2 +-
.../test-dirs/extractsB/file12_es.txt.json | 2 +-
.../extractsB/file14_diffAttachOrder.json | 20 +-
.../test-dirs/extractsB/file2_attachANotB.doc.json | 2 +-
.../test-dirs/extractsB/file3_attachBNotA.doc.json | 8 +-
.../test-dirs/extractsB/file5_emptyA.pdf.json | 2 +-
.../test-dirs/extractsB/file6_accessEx.pdf.json | 18 +-
.../test-dirs/extractsB/file8_IOEx.pdf.json | 18 +-
.../resources/test-dirs/raw_input/file12_es.txt | 2 +-
.../test-dirs/raw_input/file13_attachANotB.doc | 6 +-
.../test-dirs/raw_input/file14_diffAttachOrder | 20 +-
.../test/resources/test-dirs/raw_input/file15_tags | 26 +-
.../resources/test-dirs/raw_input/file16_badTags | 26 +-
.../test-dirs/raw_input/file17_tagsOutOfOrder | 26 +-
.../apache/tika/example/TestParsingExample.java | 4 +-
.../resources/elasticsearch/plugins-template.json | 4 +-
.../src/test/resources/kafka/plugins-template.json | 4 +-
.../resources/opensearch/plugins-template.json | 4 +-
.../opensearch/tika-config-opensearch.json | 4 +-
.../src/test/resources/s3/tika-config-s3.json | 4 +-
.../src/test/resources/solr/plugins-template.json | 4 +-
.../src/test/resources/tika-config-solr-urls.json | 4 +-
tika-metadata-schema/README.md | 18 +-
tika-metadata-schema/pom.xml | 44 ++-
.../tika/metadata/schema/MetadataKeyValidator.java | 7 +-
.../apache/tika/metadata/metadata-key-fields.json | 356 +++++++++---------
.../org/apache/tika/metadata/metadata-keys.json | 406 +++++++++++----------
.../tika/metadata/metadata-open-namespaces.json | 9 +-
.../apache/tika/metadata/metadata-string-keys.json | 17 -
.../metadata/schema/MetadataKeyValidatorTest.java | 8 +-
.../metadata/schema/MetadataNoUnderscoreTest.java | 79 ++++
.../metadata/schema/MetadataStringKeysTest.java | 65 ----
.../tools/BuildJunkAugmentationData.java | 4 +-
.../org/apache/tika/inference/ChunkSerializer.java | 11 +-
.../tika/inference/ImageEmbeddingConfig.java | 2 +-
.../org/apache/tika/inference/InferenceConfig.java | 2 +-
.../tika/inference/JinaEmbeddingFilterTest.java | 4 +-
.../tika/inference/OpenAIEmbeddingFilterTest.java | 14 +-
.../inference/OpenAIImageEmbeddingParserTest.java | 6 +-
.../apache/tika/parser/vlm/AbstractVLMParser.java | 4 +-
.../tika/parser/AutoDetectParserConfigTest.java | 24 +-
.../tika/parser/RecursiveParserWrapperTest.java | 2 +-
.../tika/parser/digest/DigestConfigTest.java | 28 +-
.../apache/tika/parser/mail/RFC822ParserTest.java | 3 +-
.../parser/microsoft/ooxml/OOXMLParserTest.java | 2 +-
.../microsoft/ooxml/OleObjectRegressionTest.java | 6 +-
.../parser/microsoft/ooxml/TruncatedOOXMLTest.java | 4 +-
.../org/apache/tika/parser/pdf/PDFParserTest.java | 2 +-
.../org/apache/tika/parser/pkg/ZipParserTest.java | 6 +-
.../test/resources/configs/TIKA-3137-include.json | 2 +-
.../configs/tika-config-write-filter.json | 2 +-
.../tika/parser/iwork/IWorkPackageParser.java | 2 +-
.../parser/iwork/iwana/IWork18PackageParser.java | 2 +-
.../apache/tika/parser/iwork/IWorkParserTest.java | 6 +-
.../org/apache/tika/parser/mp4/MP4ParserTest.java | 14 +-
.../org/apache/tika/parser/crypto/TSDParser.java | 2 +-
.../org/apache/tika/parser/html/HtmlHandler.java | 2 +-
.../tika/parser/mail/MailContentHandler.java | 2 +-
.../org/apache/tika/parser/mbox/MboxParser.java | 5 +-
.../apache/tika/parser/mbox/MboxParserTest.java | 2 +-
.../apache/tika/parser/microsoft/EMFParser.java | 4 +-
.../tika/parser/microsoft/JackcessParser.java | 8 +-
.../tika/parser/microsoft/OutlookExtractor.java | 4 +-
.../parser/microsoft/rtf/RTFObjDataParser.java | 8 +-
.../rtf/jflex/RTFObjDataStreamParser.java | 8 +-
.../tika/parser/microsoft/OutlookParserTest.java | 5 +-
.../apache/tika/parser/ocr/TesseractOCRParser.java | 8 +-
.../apache/tika/parser/pdf/AbstractPDF2XHTML.java | 11 +-
.../java/org/apache/tika/parser/pdf/PDFParser.java | 4 +-
.../org/apache/tika/parser/pdf/PDFParserTest.java | 6 +-
.../apache/tika/parser/csv/TextAndCSVParser.java | 4 +-
.../apache/tika/parser/warc/WARCParserTest.java | 4 +-
.../org/apache/tika/parser/xml/XMLProfiler.java | 6 +-
.../org/apache/tika/parser/xmp/XmpExtractor.java | 2 +-
.../java/org/apache/tika/pipes/api/ParseMode.java | 2 +-
.../filter/AttachmentCountingListFilter.java | 2 +-
.../pipes/core/DigestingOpenContainersTest.java | 2 +-
.../tika/pipes/core/MetadataWriteLimiterTest.java | 22 +-
.../apache/tika/pipes/core/PipesClientTest.java | 6 +-
.../apache/tika/pipes/core/PipesServerTest.java | 2 +-
.../pipes/core/async/AsyncChaosMonkeyTest.java | 2 +-
.../configs/tika-config-write-limiter.json | 2 +-
.../tika/pipes/emitter/azblob/AZBlobEmitter.java | 2 +-
.../org/apache/tika/pipes/emitter/es/ESClient.java | 2 +-
.../apache/tika/pipes/emitter/es/ESClientTest.java | 24 +-
.../pipes/emitter/jdbc/JDBCEmitterFactory.java | 2 +-
.../pipes/emitter/opensearch/OpenSearchClient.java | 2 +-
.../apache/tika/digest/MockDigesterFactory.java | 2 +-
.../filter/AttachmentCountingListFilter.java | 2 +-
.../writefilter/StandardMetadataLimiterTest.java | 7 +-
.../tika/serialization/JsonMetadataListTest.java | 2 +-
.../tika/serialization/JsonMetadataTest.java | 2 +-
.../resources/configs/TIKA-3137-field-mapping.json | 2 +-
.../test/resources/configs/TIKA-3695-fields.json | 2 +-
.../src/test/resources/configs/TIKA-3695.json | 2 +-
.../tika/server/core/resource/TikaResource.java | 6 +-
.../server/core/TikaServerIntegrationTest.java | 8 +-
.../resources/configs/metadata-filter-include.json | 2 +-
.../tika/server/standard/MetadataResourceTest.java | 2 +-
.../server/standard/OpenNLPMetadataFilterTest.java | 4 +-
.../standard/OptimaizeMetadataFilterTest.java | 4 +-
.../standard/RecursiveMetadataFilterTest.java | 2 +-
.../standard/RecursiveMetadataResourceTest.java | 10 +-
.../tika/server/standard/UnpackerResourceTest.java | 10 +-
.../test/resources/configs/TIKA-3137-include.json | 2 +-
.../tika/xmp/convert/MSOfficeBinaryConverter.java | 2 +-
.../tika/xmp/convert/MSOfficeXMLConverter.java | 2 +-
.../tika/xmp/convert/OpenDocumentConverter.java | 2 +-
.../org/apache/tika/xmp/convert/RTFConverter.java | 2 +-
173 files changed, 1620 insertions(+), 1141 deletions(-)
create mode 100644 .skills/metadata-schema.md
create mode 100644
tika-core/src/main/java/org/apache/tika/metadata/ReservedNamespaces.java
delete mode 100644
tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-string-keys.json
create mode 100644
tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataNoUnderscoreTest.java
delete mode 100644
tika-metadata-schema/src/test/java/org/apache/tika/metadata/schema/MetadataStringKeysTest.java
