This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 9a7bcad2aa TIKA-5027 follow up (#3538)
9a7bcad2aa is described below
commit 9a7bcad2aaf5840f0c56719df990041d730f42d6
Author: Tim Allison <[email protected]>
AuthorDate: Wed Nov 4 16:32:07 2026 -0500
TIKA-5027 follow up (#3538)
---
CHANGES.txt | 6 ++---
.../tika/ml/junkdetect/tools/TrainJunkModel.java | 4 +++-
.../ml/junkdetect/tools/TrainingProvenance.java | 28 ++++++++++++++++++++++
.../junkdetect/tools/TrainingProvenanceTest.java | 9 +++++++
.../tika-ml-junkdetect-tools/train-junk-model.sh | 14 +++++++----
.../apache/tika/ml/junkdetect/JunkDetector.java | 11 +++++----
6 files changed, 59 insertions(+), 13 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 289d382705..a851f5c04a 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -118,9 +118,9 @@ Release 4.3.0 - ???
* The junk detector no longer scores text in scripts its model has no
usable table
for (Yi, Mongolian, Canadian syllabics, Cherokee, Adlam and 37 others) as
junk: it scored -3 to -19 before. TextQualityScore gains getCoverage(),
and
- tika-eval reports languageness for such text as unknown. Text with no
letters
- (whitespace, digits, symbols) now scores -1 instead of +3, below clean
prose
- (TIKA-5027).
+ tika-eval reports languageness for such text as unknown.
JunkDetector.knownScripts()
+ no longer lists those scripts. Text with no letters (whitespace, digits,
symbols) now
+ scores -1 instead of +3, below clean prose (TIKA-5027).
* The file-to-markdown-docker agent skill turns OCR off with an empty
text-recognizers
list and a server block (its tesseract-ocr-parser config failed to load
since 4.2.0),
diff --git
a/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainJunkModel.java
b/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainJunkModel.java
index c7ff4cd7ad..db2fa3d3f7 100644
---
a/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainJunkModel.java
+++
b/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainJunkModel.java
@@ -204,7 +204,9 @@ public class TrainJunkModel {
Path output = null;
String gitCommit = null;
for (int i = 0; i < args.length; i++) {
- if (i + 1 >= args.length) {
+ boolean known = args[i].equals("--data-dir") ||
args[i].equals("--output")
+ || args[i].equals("--git-commit");
+ if (known && i + 1 >= args.length) {
System.err.println("ERROR: " + args[i] + " needs a value");
System.exit(1);
}
diff --git
a/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainingProvenance.java
b/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainingProvenance.java
index eeec4e5c36..a21c5a2201 100644
---
a/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainingProvenance.java
+++
b/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainingProvenance.java
@@ -66,9 +66,37 @@ public final class TrainingProvenance {
Map<String, String> kv = new LinkedHashMap<>();
kv.put("source_corpus",
sourceCorpus.toAbsolutePath().normalize().getFileName().toString());
kv.put("source_manifest_sha256", sha256(m));
+ kv.put("source_content_sha256", contentSha256(sourceCorpus));
writeKv(builtDir.resolve(SOURCE_FILE), kv);
}
+ /**
+ * SHA-256 over every regular file under {@code dir}, in path order: each
file's relative
+ * path and SHA-256. The manifest only lists languages and counts, so
this is what pins the
+ * corpus text itself.
+ */
+ static String contentSha256(Path dir) throws IOException {
+ List<Path> files;
+ try (java.util.stream.Stream<Path> s = Files.walk(dir)) {
+ files =
s.filter(Files::isRegularFile).sorted().collect(java.util.stream.Collectors.toList());
+ }
+ MessageDigest md;
+ try {
+ md = MessageDigest.getInstance("SHA-256");
+ } catch (NoSuchAlgorithmException e) {
+ throw new IllegalStateException(e);
+ }
+ for (Path f : files) {
+ String rel = dir.relativize(f).toString().replace('\\', '/');
+ md.update((rel + "\t" + sha256(f) +
"\n").getBytes(StandardCharsets.UTF_8));
+ }
+ StringBuilder sb = new StringBuilder(64);
+ for (byte b : md.digest()) {
+ sb.append(String.format(java.util.Locale.ROOT, "%02x", b));
+ }
+ return sb.toString();
+ }
+
/** Reads a key/value file written by {@link #writeKv}. */
static Map<String, String> readKv(Path p) throws IOException {
Map<String, String> kv = new LinkedHashMap<>();
diff --git
a/tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/tools/TrainingProvenanceTest.java
b/tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/tools/TrainingProvenanceTest.java
index 92d1ba3dc5..760a7f7e8d 100644
---
a/tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/tools/TrainingProvenanceTest.java
+++
b/tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/tools/TrainingProvenanceTest.java
@@ -40,7 +40,16 @@ public class TrainingProvenanceTest {
assertThrows(IOException.class, () ->
TrainingProvenance.writeSource(corpus, built));
Files.writeString(corpus.resolve("manifest.tsv"), "lang\tscript\n",
StandardCharsets.UTF_8);
+ Files.createDirectories(corpus.resolve("eng"));
+ Files.writeString(corpus.resolve("eng/sentences_wikipedia.txt"),
"1\tHello.\n", StandardCharsets.UTF_8);
TrainingProvenance.writeSource(corpus, built);
+ String content =
TrainingProvenance.readKv(built.resolve(TrainingProvenance.SOURCE_FILE))
+ .get("source_content_sha256");
+ // editing a sentence file without touching the manifest changes the
content digest
+ Files.writeString(corpus.resolve("eng/sentences_wikipedia.txt"),
"1\tHullo.\n", StandardCharsets.UTF_8);
+ assertTrue(!content.equals(TrainingProvenance.contentSha256(corpus)));
+ Files.writeString(corpus.resolve("eng/sentences_wikipedia.txt"),
"1\tHello.\n", StandardCharsets.UTF_8);
+ assertEquals(content, TrainingProvenance.contentSha256(corpus));
Files.writeString(model, "model", StandardCharsets.UTF_8);
assertThrows(IOException.class, () ->
TrainingProvenance.writeModelProvenance(
diff --git a/tika-ml/tika-ml-junkdetect-tools/train-junk-model.sh
b/tika-ml/tika-ml-junkdetect-tools/train-junk-model.sh
index 77f9ebe07d..dab8312dc2 100755
--- a/tika-ml/tika-ml-junkdetect-tools/train-junk-model.sh
+++ b/tika-ml/tika-ml-junkdetect-tools/train-junk-model.sh
@@ -36,16 +36,22 @@ SRC="${TIKA_JUNK_DATA}/${CORPUS}"
ROOT=$(git rev-parse --show-toplevel)
cd "${ROOT}"
-# An uncommitted tree is allowed (experiments), but the exact diff is kept
beside the log.
+# An uncommitted tree is allowed (experiments), but the exact diff is kept
beside the log,
+# untracked (not ignored) files included: they are built into the jar too.
+UNCOMMITTED=$(mktemp)
+git diff HEAD --binary > "${UNCOMMITTED}"
+git ls-files --others --exclude-standard -z | while IFS= read -r -d '' f; do
+ git diff --no-index --binary /dev/null "$f" >> "${UNCOMMITTED}" || true
+done
GIT=$(git rev-parse HEAD)
-if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
- DIFF_SHA=$(git diff HEAD | sha256sum | cut -c1-12)
+if [ -s "${UNCOMMITTED}" ]; then
+ DIFF_SHA=$(sha256sum < "${UNCOMMITTED}" | cut -c1-12)
GIT="${GIT}-dirty-${DIFF_SHA}"
fi
RUN="${TIKA_JUNK_DATA}/${CORPUS}-runs/$(date +%Y%m%d-%H%M%S)-${GIT:0:12}"
BUILT="${RUN}/built"
mkdir -p "${RUN}"
-git diff HEAD > "${RUN}/uncommitted.diff"
+mv "${UNCOMMITTED}" "${RUN}/uncommitted.diff"
LOG="${RUN}/train.log"
echo "corpus=${CORPUS} git=${GIT} run=${RUN}" | tee "${LOG}"
diff --git
a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkDetector.java
b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkDetector.java
index e6a4a8fc46..5a7aa0c081 100644
---
a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkDetector.java
+++
b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkDetector.java
@@ -615,13 +615,14 @@ public final class JunkDetector implements
TextQualityDetector {
// No scoreable LETTER at all (zero runs) — doc-level fallback:
// density=0 -> very negative; density=1 coherence=1 -> positive
// (unmodeled coherent script); density=1 coherence=0 -> very
negative.
- // z6 is included so a letter-free but FFFD/anomaly-heavy doc
(which can
- // no longer flood z1) is still penalized here — no path ignores
the
- // replacement ratio. z6 is 1 for anomaly-free text, so (z6 - 1)
keeps
- // it inert there: all-whitespace -1, coherent unmodeled script
~+2.4.
+ // The raw replacement ratio penalizes a letter-free but
FFFD/anomaly-heavy
+ // doc (which can no longer flood z1); it is 0 for anomaly-free
text:
+ // all-whitespace -1, coherent unmodeled script ~+2.4. Raw, not
z6, so a
+ // retrained z6 calibration cannot shift these scores.
agg.dominantScript = null;
agg.z1 = Float.NaN;
- agg.logit = -7f + 4f * agg.z7 + 6f * agg.z8 + 4f * (agg.z6 - 1f);
+ double rr = TextQualityFeatures.replacementRatio(text);
+ agg.logit = -7f + 4f * agg.z7 + 6f * agg.z8 - 4f *
(Double.isNaN(rr) ? 0f : (float) rr);
return agg;
}
agg.z1 = (float) (weightedZ1 / z1Count);
