(tika) branch branch_2x updated: TIKA-4278: no confidence if the delimiter isn't in row zero

[tilman]

This is an automated email from the ASF dual-hosted git repository.

tilman pushed a commit to branch branch_2x
in repository https://gitbox.apache.org/repos/asf/tika.git


The following commit(s) were added to refs/heads/branch_2x by this push:
     new ca0ceab88 TIKA-4278: no confidence if the delimiter isn't in row zero
ca0ceab88 is described below

commit ca0ceab88a46dd3735eeece9dcd3dbaae0518822
Author: Tilman Hausherr <til...@apache.org>
AuthorDate: Mon Jul 15 20:03:36 2024 +0200

    TIKA-4278: no confidence if the delimiter isn't in row zero
---
 .../src/main/java/org/apache/tika/parser/csv/CSVSniffer.java | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git 
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java
 
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java
index 24123a56b..1a772c22d 100644
--- 
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java
+++ 
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java
@@ -134,6 +134,8 @@ class CSVSniffer {
         Map<Integer, MutableInt> rowLengthCounts = new HashMap<>();
         int charsRead = 0;
         int colCount = 0;
+        boolean rowZero = true;
+        boolean rowZeroEmpty = false;
         int encapsulated = 0; //number of cells that are encapsulated in 
dquotes (for now)
         boolean parseException = false;
 
@@ -326,7 +328,12 @@ class CSVSniffer {
             } else {
                 cnt.increment();
             }
+            if (rowZero && colCount <= 1) {
+                // row zero single column => no delimiter in first line
+                rowZeroEmpty = true;
+            }
             colCount = 0;
+            rowZero = false;
         }
 
         void unquoted(String string) {
@@ -377,6 +384,11 @@ class CSVSniffer {
                 return 0.0;
             }
 
+            if (rowZeroEmpty) {
+                // TIKA-4278: not credible that there would be no delimiter in 
row zero
+                return 0.0;
+            }
+
             //TODO: convert this to continuous vs vague heuristic step function
             double consistency = (double) max / (double) totalRows;
             return ((1d - (1d / Math.pow(totalRows, 0.3))) * consistency);