(tika) branch main updated: TIKA-4278: colon delimiter detection is unreliable, use the next one if same confidence

[tilman]

This is an automated email from the ASF dual-hosted git repository.

tilman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git


The following commit(s) were added to refs/heads/main by this push:
     new 78bd4b29e TIKA-4278: colon delimiter detection is unreliable, use the 
next one if same confidence
78bd4b29e is described below

commit 78bd4b29eaf685eab678cf6e211274292657a30f
Author: Tilman Hausherr <til...@apache.org>
AuthorDate: Tue Jul 16 09:31:44 2024 +0200

    TIKA-4278: colon delimiter detection is unreliable, use the next one if 
same confidence
---
 .../src/main/java/org/apache/tika/parser/csv/CSVSniffer.java         | 5 +++++
 1 file changed, 5 insertions(+)

diff --git 
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java
 
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java
index 1a772c22d..6eb53eb14 100644
--- 
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java
+++ 
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java
@@ -93,6 +93,11 @@ class CSVSniffer {
         if (bestResult.getConfidence() < minConfidence) {
             return CSVResult.TEXT;
         }
+        // TIKA-4278: colon isn't reliable, e.g. govdocs1/242/242970.txt
+        if (results.size() > 1 && bestResult.getDelimiter().equals(':') &&
+                results.get(1).getConfidence() == bestResult.getConfidence()) {
+            return results.get(1);
+        }
         return bestResult;
     }