This is an automated email from the ASF dual-hosted git repository.
Jefffrey pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new a4ac705350 feat(arrow-csv): add support for parsing `Float16` (#10343)
a4ac705350 is described below
commit a4ac70535046651d86e09811aabb93efe8d455d6
Author: Glatzel <[email protected]>
AuthorDate: Wed Jul 15 18:40:49 2026 +0800
feat(arrow-csv): add support for parsing `Float16` (#10343)
# Which issue does this PR close?
- closes #10344
# Rationale for this change
`arrow-csv`'s CSV reader did not support parsing columns into
`DataType::Float16`. When a schema explicitly declared a `Float16` field
(schema inference never produces `Float16` on its own), parsing would
fail with an "Unsupported data type" error since there was no
corresponding match arm in the primitive-array builder dispatch. Since
`Float16Type` is a standard Arrow primitive type like
`Float32`/`Float64`, the CSV reader should be able to build arrays for
it when given an explicit schema.
# What changes are included in this PR?
- Added a `DataType::Float16` match arm in `parse()`
(`arrow-csv/src/reader/mod.rs`) that dispatches to
`build_primitive_array::<Float16Type>`, matching the existing handling
for `Float32`/`Float64`.
- Added a `test_float_precision` test that exercises `Float16`,
`Float32`, and `Float64` columns side by side against the same input
values, verifying:
- values that are exactly representable in binary (e.g. `1.5`, `0.25`,
`-2.5`, `0`) round-trip correctly at all three precisions,
- values with more precision than `f16`/`f32` can hold are correctly
truncated at the appropriate width while `f64` retains full precision,
- empty CSV fields decode to null across all three columns.
# Are these changes tested?
Yes, added a `test_float_precision` fn to test parsing csv with
different precision `Float16`, `Float32`, `Float64`), including
precision-loss and null handling.
- `cargo test -p arrow-csv`
# Are there any user-facing changes?
Yes. Previously, attempting to read a CSV column into a schema field
typed as `DataType::Float16` would fail with an `Parser error:
Unsupported data type Float16`.
This is now supported, consistent with the existing
`Float32`/`Float64`handling. No existing behavior changes for other
types.
---
Cargo.lock | 1 +
arrow-csv/Cargo.toml | 1 +
arrow-csv/src/reader/mod.rs | 64 +++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 66 insertions(+)
diff --git a/Cargo.lock b/Cargo.lock
index d781954e85..325baedf97 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -313,6 +313,7 @@ dependencies = [
"csv",
"csv-core",
"futures",
+ "half",
"regex",
"tempfile",
"tokio",
diff --git a/arrow-csv/Cargo.toml b/arrow-csv/Cargo.toml
index c44ec01ce3..689acf7b06 100644
--- a/arrow-csv/Cargo.toml
+++ b/arrow-csv/Cargo.toml
@@ -50,3 +50,4 @@ tempfile = "3.3"
futures = "0.3"
tokio = { version = "1.27", default-features = false, features = ["io-util"] }
bytes = "1.4"
+half = { version = "2.1", default-features = false }
diff --git a/arrow-csv/src/reader/mod.rs b/arrow-csv/src/reader/mod.rs
index aae6f66cf5..51e855f7df 100644
--- a/arrow-csv/src/reader/mod.rs
+++ b/arrow-csv/src/reader/mod.rs
@@ -801,6 +801,9 @@ fn parse(
DataType::UInt64 => {
build_primitive_array::<UInt64Type>(line_number, rows, i,
null_regex)
}
+ DataType::Float16 => {
+ build_primitive_array::<Float16Type>(line_number, rows, i,
null_regex)
+ }
DataType::Float32 => {
build_primitive_array::<Float32Type>(line_number, rows, i,
null_regex)
}
@@ -2988,4 +2991,65 @@ mod tests {
assert_eq!(c2.value(1), "something_cannot_be_inlined");
assert_eq!(c2.value(2), "bar");
}
+
+ #[test]
+ fn test_float_precision() {
+ let data = [
+ "f16,f32,f64",
+ "1.5,1.5,1.5",
+ "0.25,0.25,0.25",
+ "1.23456789,1.23456789,1.23456789",
+ "1.234567890123456,1.234567890123456,1.234567890123456",
+ "-2.5,-2.5,-2.5",
+ "0,0,0",
+ ",,",
+ ]
+ .join("\n");
+
+ let schema = Schema::new(vec![
+ Field::new("f16", DataType::Float16, true),
+ Field::new("f32", DataType::Float32, true),
+ Field::new("f64", DataType::Float64, true),
+ ]);
+
+ let mut reader = ReaderBuilder::new(Arc::new(schema))
+ .with_header(true)
+ .build(Cursor::new(data))
+ .unwrap();
+
+ let batch = reader.next().unwrap().unwrap();
+ assert_eq!(batch.num_rows(), 7);
+
+ let f16_col = batch.column(0).as_primitive::<Float16Type>();
+ let f32_col = batch.column(1).as_primitive::<Float32Type>();
+ let f64_col = batch.column(2).as_primitive::<Float64Type>();
+
+ assert_eq!(f16_col.value(0), half::f16::from_f32(1.5));
+ assert_eq!(f32_col.value(0), 1.5f32);
+ assert_eq!(f64_col.value(0), 1.5f64);
+
+ assert_eq!(f16_col.value(1), half::f16::from_f32(0.25));
+ assert_eq!(f32_col.value(1), 0.25f32);
+ assert_eq!(f64_col.value(1), 0.25f64);
+
+ assert_eq!(f16_col.value(2), half::f16::from_f32(1.234_567_9));
+ assert_eq!(f32_col.value(2), 1.234_567_9_f32);
+ assert_eq!(f64_col.value(2), 1.23456789f64);
+
+ assert_eq!(f16_col.value(3),
half::f16::from_f64(1.234567890123456f64));
+ assert_eq!(f32_col.value(3), 1.234_567_9_f32);
+ assert_eq!(f64_col.value(3), 1.234567890123456f64);
+
+ assert_eq!(f16_col.value(4), half::f16::from_f32(-2.5));
+ assert_eq!(f32_col.value(4), -2.5f32);
+ assert_eq!(f64_col.value(4), -2.5f64);
+
+ assert_eq!(f16_col.value(5), half::f16::from_f32(0.0));
+ assert_eq!(f32_col.value(5), 0.0f32);
+ assert_eq!(f64_col.value(5), 0.0f64);
+
+ assert!(f16_col.is_null(6));
+ assert!(f32_col.is_null(6));
+ assert!(f64_col.is_null(6));
+ }
}
