[
https://issues.apache.org/jira/browse/TIKA-5027?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=18139149#comment-18139149
]
Hudson commented on TIKA-5027:
------------------------------
SUCCESS: Integrated in Jenkins build Tika ยป tika-main-jdk17 #1777 (See
[https://ci-builds.apache.org/job/Tika/job/tika-main-jdk17/1777/])
TIKA-5027 - hashed wire junk detector (#3549) (github:
[https://github.com/apache/tika/commit/79749736e2186184261a0c51ae7378b2e1f56d62])
* (delete)
tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/UnicodeBlockRangesTest.java
* (edit) docs/modules/ROOT/pages/configuration/encoding-detectors.adoc
* (add)
tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/ChainProbe.java
* (add)
tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/JunkFilterCandidatesTest.java
* (edit) docs/modules/ROOT/pages/advanced/junk-detection-build.adoc
* (edit)
tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/HtmlContentCleaner.java
* (edit)
tika-ml/tika-ml-junkdetect/src/main/resources/org/apache/tika/ml/junkdetect/junkdetect.bin
* (delete)
tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/tools/BuildJunkAugmentationData.java
* (edit)
tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
* (add)
tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/WikipediaUnits.java
* (edit)
tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/tools/JunkDetectorTrainingConfigTest.java
* (add)
tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/HashedJunkModel.java
* (edit) tika-ml/tika-ml-junkdetect-tools/pom.xml
* (edit)
tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ProfilerBase.java
* (add)
tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/tools/HashedTrainingFixture.java
* (delete)
tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/DebugScriptRuns.java
* (edit)
tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/JunkDetectorSmokeTest.java
* (add)
tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/PairSources.java
* (edit)
tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/JunkDetectorTrainingConfig.java
* (delete)
tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainJunkModel.java
* (add)
tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainHashedJunkModel.java
* (edit)
tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/tools/TrainingProvenanceTest.java
* (add)
tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/HashedArbitration.java
* (edit)
tika-eval/tika-eval-core/src/test/java/org/apache/tika/eval/core/metadata/TikaEvalMetadataFilterTest.java
* (delete)
tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/LineScriptFractions.java
* (add)
tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/HashedJunkModelTest.java
* (edit)
tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/BuildJunkTrainingData.java
* (edit)
tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/metadata/TikaEvalMetadataFilter.java
* (add)
tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/tools/HashedTrainerTest.java
* (edit) tika-ml/tika-ml-junkdetect-tools/train-junk-model.sh
* (edit) docs/modules/ROOT/pages/advanced/junk-detection.adoc
* (edit)
tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkDetector.java
* (delete)
tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/UnicodeBlockRanges.java
* (delete)
tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/LatinLetterGateTest.java
* (edit)
tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfiler.java
* (delete)
tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/ScriptCensus.java
* (add)
tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/HashedFeatures.java
* (edit) CHANGES.txt
* (edit)
tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/db/Cols.java
* (add)
tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/GoldPage.java
* (add)
tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/JunkFamilies.java
* (delete)
tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/tools/BuildJunkAugmentationDataTest.java
* (edit)
tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/UnmodeledScriptTest.java
* (delete)
tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/BoundaryBigramAudit.java
* (edit)
tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/DecodeCorruptionDiscriminationTest.java
* (edit)
tika-ml/tika-ml-junkdetect/src/main/resources/org/apache/tika/ml/junkdetect/junkdetect.bin.provenance
* (add)
tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/HashedTrainingConfig.java
* (delete)
tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/TraceJunkFilter.java
* (edit)
tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java
* (delete)
tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/BigramTables.java
* (delete)
tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/JunkDetectorRoundTripTest.java
* (delete)
tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/TextQualityFeatures.java
* (edit)
tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetectorTest.java
* (add)
tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/HashedArbitrationTest.java
* (edit) docs/modules/ROOT/pages/advanced/charset-detection-design.adoc
* (delete)
tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/TextQualityFeaturesTest.java
* (edit)
tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainingProvenance.java
> Improve junk detector
> ---------------------
>
> Key: TIKA-5027
> URL: https://issues.apache.org/jira/browse/TIKA-5027
> Project: Tika
> Issue Type: Task
> Reporter: Tim Allison
> Priority: Major
>
> In looking at charset detection, I found a number of things we could improve
> in junk detection.
--
This message was sent by Atlassian Jira
(v10.3.26#10030026)
