This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 3be82f50 fix(arrow): read CSV boolean columns like Java's CsvParser
(#980)
3be82f50 is described below
commit 3be82f501ba8995da57d31a8595438775cf266a1
Author: jackylee <[email protected]>
AuthorDate: Fri Oct 2 10:08:19 2026 +0800
fix(arrow): read CSV boolean columns like Java's CsvParser (#980)
---
crates/paimon/src/arrow/format/text.rs | 55 +++++++++++++++++++++++++++++++++-
1 file changed, 54 insertions(+), 1 deletion(-)
diff --git a/crates/paimon/src/arrow/format/text.rs
b/crates/paimon/src/arrow/format/text.rs
index 2f851951..d46c5411 100644
--- a/crates/paimon/src/arrow/format/text.rs
+++ b/crates/paimon/src/arrow/format/text.rs
@@ -25,7 +25,8 @@ use crate::spec::DataField;
use crate::table::{ArrowRecordBatchStream, RowRange};
use crate::Error;
use arrow_array::{
- Array, ArrayRef, BinaryArray, FixedSizeBinaryArray, LargeBinaryArray,
RecordBatch, StringArray,
+ Array, ArrayRef, BinaryArray, BooleanArray, FixedSizeBinaryArray,
LargeBinaryArray,
+ RecordBatch, StringArray,
};
use arrow_schema::{DataType, SchemaRef};
use async_trait::async_trait;
@@ -1081,6 +1082,22 @@ fn csv_cast_column(column: &ArrayRef, target: &DataType)
-> crate::Result<ArrayR
_ => unreachable!(),
});
}
+ if matches!(target, DataType::Boolean) {
+ // Java `CsvParser` reads BOOLEAN via `Boolean.parseBoolean`: only
"true"
+ // (case-insensitive) is true, every other non-null token is false, and
+ // it never yields null. arrow-cast instead maps "1"/"yes"/"t"/"on"/"y"
+ // to true and unrecognized tokens (e.g. "2", "invalid") to null, so
the
+ // same CSV file read through Rust would flip boolean values or turn
them
+ // null relative to Java. Match Java so a format table reads the same
+ // across engines. (Null fields are already `None` from CSV decoding
and
+ // stay null.)
+ let values = column.as_any().downcast_ref::<StringArray>().unwrap();
+ let booleans: BooleanArray = values
+ .iter()
+ .map(|value| value.map(|value| value.eq_ignore_ascii_case("true")))
+ .collect();
+ return Ok(Arc::new(booleans));
+ }
arrow_cast::cast(column, target).map_err(arrow_error)
}
@@ -1430,4 +1447,40 @@ mod tests {
serde_json::json!(["false", null, "true"])
);
}
+
+ #[test]
+ fn csv_boolean_matches_java_parse_boolean() {
+ // Java `CsvParser` reads BOOLEAN via `Boolean.parseBoolean`: only
"true"
+ // (case-insensitive) is true, every other non-null token is false, and
+ // null stays null. arrow-cast would instead read "1"/"yes"/"t" as true
+ // and "invalid" as null, flipping values for the same CSV file.
+ let input: ArrayRef = Arc::new(StringArray::from(vec![
+ Some("true"),
+ Some("TRUE"),
+ Some("1"),
+ Some("yes"),
+ Some("t"),
+ Some("invalid"),
+ Some("false"),
+ Some("0"),
+ None,
+ ]));
+ let result = csv_cast_column(&input, &DataType::Boolean).unwrap();
+ let booleans = result.as_any().downcast_ref::<BooleanArray>().unwrap();
+ let got: Vec<Option<bool>> = booleans.iter().collect();
+ assert_eq!(
+ got,
+ vec![
+ Some(true), // "true"
+ Some(true), // "TRUE"
+ Some(false), // "1" (arrow-cast would say true)
+ Some(false), // "yes" (arrow-cast would say true)
+ Some(false), // "t" (arrow-cast would say true)
+ Some(false), // "invalid" (arrow-cast would say null)
+ Some(false), // "false"
+ Some(false), // "0"
+ None, // null stays null
+ ]
+ );
+ }
}
