This is an automated email from the ASF dual-hosted git repository.
tballison pushed a change to branch TIKA-4856-presets
in repository https://gitbox.apache.org/repos/asf/tika.git
from 8b1eb4b9cf Merge branch 'main' into TIKA-4856-presets
add 5b78cd84bc TIKA-4873 - give embedded documents their size (#3117)
add 941424c549 fix grpc tests port toctou (#3121)
add 3fb998e05a TIKA-4868: performance improvements (#3120)
add c726a6b723 Fix tests that flake when building in parallel (#3124)
add 4a563d6fdb Add a placeholder marker (#3122)
add 54a8ed6892 TIKA-4872 - improve ocr wiring (#3119)
add 4f0e11bafa TIKA-4875: improve tika-eval performance (#3123)
add 12ddaaaca9 Merge remote-tracking branch 'origin/main' into
TIKA-4856-presets
No new revisions were added by this update.
Summary of changes:
CHANGES.txt | 136 +++++++-
docs/modules/ROOT/pages/configuration/index.adoc | 31 ++
.../ROOT/pages/configuration/metadata-filters.adoc | 5 +
.../pages/configuration/parsers/tess4j-parser.adoc | 7 +
.../parsers/tesseract-ocr-parser.adoc | 18 +-
.../pages/configuration/parsers/vlm-parsers.adoc | 14 +-
tika-app/src/main/resources/log4j2.xml | 2 +
.../java/org/apache/tika/config/ParseTimeout.java | 2 +-
...{package-info.java => TransientParseState.java} | 10 +-
.../org/apache/tika/detect/CompositeDetector.java | 2 +-
.../org/apache/tika/detect/DefaultDetector.java | 6 +
.../java/org/apache/tika/detect/MagicDetector.java | 38 ++-
.../ParsingEmbeddedDocumentExtractor.java | 9 +
.../java/org/apache/tika/io/ByteArraySource.java | 5 +
.../java/org/apache/tika/io/CachingSource.java | 9 +
.../main/java/org/apache/tika/io/FileSource.java | 5 +
.../java/org/apache/tika/io/PlaceholderSource.java | 122 +++++++
.../java/org/apache/tika/io/ReopenableSource.java | 9 +
.../java/org/apache/tika/io/TikaInputSource.java | 14 +
.../java/org/apache/tika/io/TikaInputStream.java | 60 +++-
.../main/java/org/apache/tika/mime/MagicMatch.java | 19 +-
.../main/java/org/apache/tika/mime/MimeTypes.java | 62 +++-
.../main/java/org/apache/tika/mime/Patterns.java | 27 +-
.../org/apache/tika/parser/CompositeParser.java | 32 +-
.../java/org/apache/tika/parser/ParseRecord.java | 3 +-
.../parser/enricher/CompositeContentEnricher.java | 88 +++++
.../tika/parser/enricher/ContentEnrichers.java | 194 +++++++++++
.../tika/parser/enricher/EnrichingParser.java | 21 +-
.../parser/enricher/LegacyDispatchEnricher.java | 106 ++++++
.../org/apache/tika/renderer/RenderResult.java | 3 +-
.../apache/tika/sax/FastMarkdownTextRenderer.java | 205 ++++++++++++
.../apache/tika/sax/ToMarkdownContentHandler.java | 91 ++++-
.../org/apache/tika/mime/tika-mimetypes.xml | 11 +-
.../tika/extractor/EmbeddedContentLengthTest.java | 84 +++++
.../org/apache/tika/io/DeclaredLengthTest.java | 111 ++++++
.../org/apache/tika/io/PlaceholderStreamTest.java | 63 ++++
.../org/apache/tika/mime/MimeDetectionTest.java | 14 +
.../java/org/apache/tika/mime/PatternsTest.java | 16 +
.../tika/parser/enricher/ContentEnrichersTest.java | 371 +++++++++++++++++++++
.../tika/sax/FastMarkdownTextRendererTest.java | 201 +++++++++++
.../tika/sax/ToMarkdownContentHandlerTest.java | 18 +
.../apache/tika/ml/chardetect/AdaptiveProbe.java | 12 +-
.../tika/ml/chardetect/HtmlByteStripper.java | 115 +++++--
.../ml/chardetect/HtmlByteStripperResumeTest.java | 117 +++++++
.../tika/eval/app/ExtractComparerRunner.java | 12 +-
.../apache/tika/eval/app/ExtractProfileRunner.java | 12 +-
.../org/apache/tika/eval/app/StatusReporter.java | 18 +-
.../java/org/apache/tika/eval/app/db/JDBCUtil.java | 41 +++
.../org/apache/tika/eval/app/db/JDBCUtilTest.java | 65 ++++
.../org/apache/tika/pipes/grpc/TikaGrpcServer.java | 7 +
...PipesBiDirectionalStreamingIntegrationTest.java | 21 +-
.../apache/tika/pipes/grpc/TikaGrpcServerTest.java | 55 ++-
.../resources/elasticsearch/plugins-template.json | 2 +-
.../resources/opensearch/plugins-template.json | 2 +-
.../opensearch/tika-config-opensearch.json | 2 +-
.../charsoup/core/CharSoupFeatureExtractor.java | 151 ++++++++-
.../core/CharSoupFeatureExtractorTest.java | 87 +++++
.../tika/langdetect/opennlp/OpenNLPDetector.java | 6 +-
.../tika/ml/junkdetect/UnicodeBlockRanges.java | 20 ++
.../tika/ml/junkdetect/UnicodeBlockRangesTest.java | 14 +
tika-parent/pom.xml | 34 +-
.../tika-parser-scientific-module/pom.xml | 3 +-
.../tika/inference/AbstractEmbeddingFilter.java | 68 +++-
.../apache/tika/inference/JinaEmbeddingFilter.java | 5 +
.../tika/inference/OpenAIEmbeddingFilter.java | 12 +
.../tika/inference/OpenAIEmbeddingFilterTest.java | 57 ++++
.../apache/tika/parser/vlm/OpenAIVLMParser.java | 3 +-
.../tika/parser/vlm/OpenAIVLMParserTest.java | 17 +
.../tika/detect/TestContainerAwareDetector.java | 40 ++-
.../org/apache/tika/parser/audio/CoverArt.java | 4 +-
.../tika/parser/image/AbstractImageParser.java | 58 +---
.../apache/tika/parser/image/RawTiffParser.java | 4 +
.../apache/tika/parser/image/ImageParserTest.java | 80 ++++-
.../tika/parser/image/RawTiffParserTest.java | 4 +
.../tika/parser/microsoft/MetafileRendering.java | 2 +-
.../apache/tika/parser/microsoft/OfficeParser.java | 11 +-
.../tika/parser/microsoft/WordExtractor.java | 54 ++-
.../parser/microsoft/pst/OutlookPSTParserTest.java | 3 +-
.../apache/tika/parser/ocr/TesseractOCRParser.java | 3 +-
.../apache/tika/parser/pdf/AbstractPDF2XHTML.java | 69 ++--
.../java/org/apache/tika/parser/pdf/OCR2XHTML.java | 11 +-
.../java/org/apache/tika/parser/pdf/PDF2XHTML.java | 19 +-
.../tika/parser/pdf/PDFMarkedContent2XHTML.java | 11 +-
.../java/org/apache/tika/parser/pdf/PDFParser.java | 20 +-
.../tika/parser/pdf/image/ImageGraphicsEngine.java | 2 +-
.../org/apache/tika/parser/pdf/PDFParserTest.java | 94 ++++++
.../java/org/apache/tika/parser/pkg/ZipParser.java | 14 +-
.../org/apache/tika/parser/csv/CSVSniffer.java | 176 +++++-----
.../apache/tika/pipes/api/emitter/EmitData.java | 9 +
tika-pipes/tika-pipes-config-store-ignite/pom.xml | 2 +
.../apache/tika/pipes/core/ContentBytesConfig.java | 31 +-
.../org/apache/tika/pipes/core/PipesClient.java | 69 ++++
.../tika/pipes/core/async/AsyncProcessor.java | 6 +
.../tika/pipes/core/emitter/EmitDataImpl.java | 42 ++-
.../core/serialization/EmitDataDeserializer.java | 7 +-
.../core/serialization/EmitDataSerializer.java | 6 +
.../tika/pipes/core/server/ConnectionHandler.java | 61 +++-
.../apache/tika/pipes/core/server/EmitHandler.java | 33 ++
.../apache/tika/pipes/core/server/PipesServer.java | 68 +++-
.../apache/tika/pipes/core/server/PipesWorker.java | 48 ++-
.../tika/pipes/core/server/ServerProtocolIO.java | 66 +++-
.../serialization/EmitDataContentBytesTest.java | 58 ++++
.../org/apache/tika/pipes/core/MockEnricher.java | 35 +-
.../apache/tika/pipes/core/PipesClientTest.java | 30 ++
...sic.json => tika-config-content-enrichers.json} | 1 +
.../tika/config/loader/ContentEnricherLoader.java | 69 ++++
.../apache/tika/config/loader/LoaderContext.java | 11 +
.../apache/tika/config/loader/ParserLoader.java | 64 +++-
.../apache/tika/config/loader/TikaJsonConfig.java | 1 +
.../org/apache/tika/config/loader/TikaLoader.java | 9 +
.../serdes/ParseContextSerializer.java | 7 +
.../config/loader/ContentEnricherLoaderTest.java | 113 +++++++
...malTestParser.java => EnrichingTestParser.java} | 33 +-
.../{OptInTestParser.java => TestPngEnricher.java} | 18 +-
...estParser.java => TestUnavailableEnricher.java} | 17 +-
.../TestParseContextSerialization.java | 15 +
tika-server/tika-server-core/pom.xml | 3 +-
.../server/core/resource/PipesParsingHelper.java | 65 +++-
.../tika/server/core/resource/TikaResource.java | 73 +++-
.../tika-server-core/src/main/resources/log4j2.xml | 2 +
tika-server/tika-server-standard/pom.xml | 3 +-
.../src/main/resources/log4j2.xml | 2 +
122 files changed, 4460 insertions(+), 498 deletions(-)
copy tika-core/src/main/java/org/apache/tika/config/{package-info.java =>
TransientParseState.java} (72%)
create mode 100644
tika-core/src/main/java/org/apache/tika/io/PlaceholderSource.java
create mode 100644
tika-core/src/main/java/org/apache/tika/parser/enricher/CompositeContentEnricher.java
create mode 100644
tika-core/src/main/java/org/apache/tika/parser/enricher/ContentEnrichers.java
copy
tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/textstats/BytesRefCalculator.java
=>
tika-core/src/main/java/org/apache/tika/parser/enricher/EnrichingParser.java
(66%)
create mode 100644
tika-core/src/main/java/org/apache/tika/parser/enricher/LegacyDispatchEnricher.java
create mode 100644
tika-core/src/main/java/org/apache/tika/sax/FastMarkdownTextRenderer.java
create mode 100644
tika-core/src/test/java/org/apache/tika/extractor/EmbeddedContentLengthTest.java
create mode 100644
tika-core/src/test/java/org/apache/tika/io/DeclaredLengthTest.java
create mode 100644
tika-core/src/test/java/org/apache/tika/io/PlaceholderStreamTest.java
create mode 100644
tika-core/src/test/java/org/apache/tika/parser/enricher/ContentEnrichersTest.java
create mode 100644
tika-core/src/test/java/org/apache/tika/sax/FastMarkdownTextRendererTest.java
create mode 100644
tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/HtmlByteStripperResumeTest.java
create mode 100644
tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/db/JDBCUtilTest.java
copy
tika-serialization/src/test/java/org/apache/tika/sax/UppercasingContentHandlerFactory.java
=>
tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/ContentBytesConfig.java
(51%)
create mode 100644
tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/EmitDataContentBytesTest.java
copy tika-core/src/main/java/org/apache/tika/parser/EmptyParser.java =>
tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/MockEnricher.java
(69%)
copy
tika-pipes/tika-pipes-integration-tests/src/test/resources/configs/{tika-config-basic.json
=> tika-config-content-enrichers.json} (95%)
create mode 100644
tika-serialization/src/main/java/org/apache/tika/config/loader/ContentEnricherLoader.java
create mode 100644
tika-serialization/src/test/java/org/apache/tika/config/loader/ContentEnricherLoaderTest.java
copy
tika-serialization/src/test/java/org/apache/tika/config/loader/{MinimalTestParser.java
=> EnrichingTestParser.java} (66%)
copy
tika-serialization/src/test/java/org/apache/tika/config/loader/{OptInTestParser.java
=> TestPngEnricher.java} (71%)
copy
tika-serialization/src/test/java/org/apache/tika/config/loader/{OptInTestParser.java
=> TestUnavailableEnricher.java} (71%)
