This is an automated email from the ASF dual-hosted git repository.
Jefffrey pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 8acab7b537 feat: Implement decimal <-> float16 casts (#10008)
8acab7b537 is described below
commit 8acab7b5371470deff6c211899295d2bb3030dfc
Author: Neil Conway <[email protected]>
AuthorDate: Sun May 24 21:57:45 2026 -0400
feat: Implement decimal <-> float16 casts (#10008)
# Which issue does this PR close?
- Closes #9123.
# Rationale for this change
Arrow supports casts between decimal and float64/float32; for
consistency and completeness, we should also support casts between
decimal and float16.
In DataFusion, this will be particularly useful: once
https://github.com/apache/datafusion/issues/14612 is fixed,
`arrow_cast(0.0, 'Float16')` will no longer work, unless we first add
support for decimal -> float16 casts in arrow-rs.
# What changes are included in this PR?
* Add support for decimal -> float16 cast
* Add support for float16 -> decimal cast
* Add unit tests for new behavior
* Update docs/comment on supported casts
# Are these changes tested?
Yes; new tests added.
# Are there any user-facing changes?
Yes; new casts are now supported. Otherwise no changes.
---
arrow-cast/src/cast/mod.rs | 282 ++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 278 insertions(+), 4 deletions(-)
diff --git a/arrow-cast/src/cast/mod.rs b/arrow-cast/src/cast/mod.rs
index 6b4355ae3a..12442bd9fd 100644
--- a/arrow-cast/src/cast/mod.rs
+++ b/arrow-cast/src/cast/mod.rs
@@ -188,7 +188,7 @@ pub fn can_cast_types(from_type: &DataType, to_type:
&DataType) -> bool {
) => true,
// signed numeric to decimal
(
- Int8 | Int16 | Int32 | Int64 | Float32 | Float64,
+ Int8 | Int16 | Int32 | Int64 | Float16 | Float32 | Float64,
Decimal32(_, _) | Decimal64(_, _) | Decimal128(_, _) |
Decimal256(_, _),
) => true,
// decimal to unsigned numeric
@@ -199,7 +199,7 @@ pub fn can_cast_types(from_type: &DataType, to_type:
&DataType) -> bool {
// decimal to signed numeric
(
Decimal32(_, _) | Decimal64(_, _) | Decimal128(_, _) |
Decimal256(_, _),
- Null | Int8 | Int16 | Int32 | Int64 | Float32 | Float64,
+ Null | Int8 | Int16 | Int32 | Int64 | Float16 | Float32 | Float64,
) => true,
// decimal to string
(
@@ -649,9 +649,9 @@ fn timestamp_to_date32<T: ArrowTimestampType>(
/// * `Time32 and `Time64`: precision lost when going to higher interval
/// * `Timestamp` and `Date{32|64}`: precision lost when going to higher
interval
/// * Temporal to/from backing Primitive: zero-copy with data type change
-/// * `Float32/Float64` to `Decimal(precision, scale)` rounds to the `scale`
decimals
+/// * `Float16/Float32/Float64` to `Decimal(precision, scale)` rounds to the
`scale` decimals
/// (i.e. casting `6.4999` to `Decimal(10, 1)` becomes `6.5`).
-/// * `Decimal` to `Float32/Float64` is lossy and values outside the
representable
+/// * `Decimal` to `Float16/Float32/Float64` is lossy and values outside the
representable
/// range become `INFINITY` or `-INFINITY` without error.
///
/// Unsupported Casts (check with `can_cast_types` before calling):
@@ -2329,6 +2329,13 @@ where
Int16 => cast_decimal_to_integer::<D, Int16Type>(array, base, *scale,
cast_options),
Int32 => cast_decimal_to_integer::<D, Int32Type>(array, base, *scale,
cast_options),
Int64 => cast_decimal_to_integer::<D, Int64Type>(array, base, *scale,
cast_options),
+ Float16 => cast_decimal_to_float::<D, Float16Type, _>(array, |x| {
+ half::f16::from_f64(single_decimal_to_float_lossy::<D, F>(
+ &as_float,
+ x,
+ <i32 as From<i8>>::from(*scale),
+ ))
+ }),
Float32 => cast_decimal_to_float::<D, Float32Type, _>(array, |x| {
single_decimal_to_float_lossy::<D, F>(&as_float, x, <i32 as
From<i8>>::from(*scale))
as f32
@@ -2426,6 +2433,12 @@ where
base,
cast_options,
),
+ Float16 => cast_floating_point_to_decimal::<_, D>(
+ array.as_primitive::<Float16Type>(),
+ *precision,
+ *scale,
+ cast_options,
+ ),
Float32 => cast_floating_point_to_decimal::<_, D>(
array.as_primitive::<Float32Type>(),
*precision,
@@ -3366,6 +3379,84 @@ mod tests {
}
}
+ #[test]
+ fn test_cast_float16_to_decimals() {
+ let array = Float16Array::from(vec![
+ Some(f16::from_f32(1.25)),
+ Some(f16::from_f32(-2.5)),
+ Some(f16::from_f32(1.125)),
+ Some(f16::from_f32(-1.125)),
+ Some(f16::from_f32(0.0)),
+ None,
+ ]);
+
+ generate_cast_test_case!(
+ &array,
+ Decimal32Array,
+ &DataType::Decimal32(9, 2),
+ vec![
+ Some(125_i32),
+ Some(-250_i32),
+ Some(113_i32),
+ Some(-113_i32),
+ Some(0_i32),
+ None
+ ]
+ );
+ generate_cast_test_case!(
+ &array,
+ Decimal64Array,
+ &DataType::Decimal64(18, 2),
+ vec![
+ Some(125_i64),
+ Some(-250_i64),
+ Some(113_i64),
+ Some(-113_i64),
+ Some(0_i64),
+ None
+ ]
+ );
+ generate_cast_test_case!(
+ &array,
+ Decimal128Array,
+ &DataType::Decimal128(38, 2),
+ vec![
+ Some(125_i128),
+ Some(-250_i128),
+ Some(113_i128),
+ Some(-113_i128),
+ Some(0_i128),
+ None
+ ]
+ );
+ generate_cast_test_case!(
+ &array,
+ Decimal256Array,
+ &DataType::Decimal256(76, 2),
+ vec![
+ Some(i256::from_i128(125_i128)),
+ Some(i256::from_i128(-250_i128)),
+ Some(i256::from_i128(113_i128)),
+ Some(i256::from_i128(-113_i128)),
+ Some(i256::from_i128(0_i128)),
+ None
+ ]
+ );
+
+ let array = Float16Array::from(vec![
+ Some(f16::from_f32(1250.0)),
+ Some(f16::from_f32(-1250.0)),
+ Some(f16::from_f32(1249.0)),
+ None,
+ ]);
+ generate_cast_test_case!(
+ &array,
+ Decimal128Array,
+ &DataType::Decimal128(5, -2),
+ vec![Some(13_i128), Some(-13_i128), Some(12_i128), None]
+ );
+ }
+
#[test]
fn test_cast_decimal128_to_decimal128_overflow() {
let input_type = DataType::Decimal128(38, 3);
@@ -3620,6 +3711,19 @@ mod tests {
&DataType::Int64,
vec![Some(1_i64), Some(2_i64), Some(3_i64), None, Some(5_i64)]
);
+ // f16
+ generate_cast_test_case!(
+ array,
+ Float16Array,
+ &DataType::Float16,
+ vec![
+ Some(f16::from_f32(1.25)),
+ Some(f16::from_f32(2.25)),
+ Some(f16::from_f32(3.25)),
+ None,
+ Some(f16::from_f32(5.25))
+ ]
+ );
// f32
generate_cast_test_case!(
array,
@@ -3847,6 +3951,19 @@ mod tests {
&DataType::Int64,
vec![Some(1_i64), Some(2_i64), Some(3_i64), None, Some(5_i64)]
);
+ // f16
+ generate_cast_test_case!(
+ &array,
+ Float16Array,
+ &DataType::Float16,
+ vec![
+ Some(f16::from_f32(1.25)),
+ Some(f16::from_f32(2.25)),
+ Some(f16::from_f32(3.25)),
+ None,
+ Some(f16::from_f32(5.25))
+ ]
+ );
// f32
generate_cast_test_case!(
&array,
@@ -3957,6 +4074,60 @@ mod tests {
);
}
+ #[test]
+ fn test_cast_decimal128_to_float16_overflow() {
+ let array = create_decimal128_array(
+ vec![
+ Some(6_550_400_i128),
+ Some(100_000_000_i128),
+ Some(-100_000_000_i128),
+ None,
+ ],
+ 10,
+ 2,
+ )
+ .unwrap();
+
+ generate_cast_test_case!(
+ &array,
+ Float16Array,
+ &DataType::Float16,
+ vec![
+ Some(f16::from_f64(65504.0)),
+ Some(f16::INFINITY),
+ Some(f16::NEG_INFINITY),
+ None
+ ]
+ );
+ }
+
+ #[test]
+ fn test_cast_decimal256_to_float16_overflow() {
+ let array = create_decimal256_array(
+ vec![
+ Some(i256::from_i128(6_550_400_i128)),
+ Some(i256::from_i128(100_000_000_i128)),
+ Some(i256::from_i128(-100_000_000_i128)),
+ None,
+ ],
+ 10,
+ 2,
+ )
+ .unwrap();
+
+ generate_cast_test_case!(
+ &array,
+ Float16Array,
+ &DataType::Float16,
+ vec![
+ Some(f16::from_f64(65504.0)),
+ Some(f16::INFINITY),
+ Some(f16::NEG_INFINITY),
+ None
+ ]
+ );
+ }
+
#[test]
fn test_cast_decimal_to_numeric_negative_scale() {
let value_array: Vec<Option<i256>> = vec![
@@ -3975,6 +4146,19 @@ mod tests {
vec![Some(1_250), Some(2_250), Some(3_250), None, Some(5_250)]
);
+ let value_array: Vec<Option<i128>> = vec![Some(12), Some(-12), None];
+ let array = create_decimal128_array(value_array, 10, -2).unwrap();
+ generate_cast_test_case!(
+ &array,
+ Float16Array,
+ &DataType::Float16,
+ vec![
+ Some(f16::from_f32(1200.0)),
+ Some(f16::from_f32(-1200.0)),
+ None
+ ]
+ );
+
let value_array: Vec<Option<i32>> = vec![Some(125), Some(225),
Some(325), None, Some(525)];
let array = create_decimal32_array(value_array, 8, -2).unwrap();
generate_cast_test_case!(
@@ -10164,6 +10348,96 @@ mod tests {
);
}
+ #[test]
+ fn test_cast_float16_to_decimal128_precision_overflow() {
+ let array = Float16Array::from(vec![f16::from_f32(1.1)]);
+ let array = Arc::new(array) as ArrayRef;
+ let casted_array = cast_with_options(
+ &array,
+ &DataType::Decimal128(2, 2),
+ &CastOptions {
+ safe: true,
+ format_options: FormatOptions::default(),
+ },
+ );
+ assert!(casted_array.is_ok());
+ assert!(casted_array.unwrap().is_null(0));
+
+ let casted_array = cast_with_options(
+ &array,
+ &DataType::Decimal128(2, 2),
+ &CastOptions {
+ safe: false,
+ format_options: FormatOptions::default(),
+ },
+ );
+ let err = casted_array.unwrap_err().to_string();
+ let expected_error = "Invalid argument error: 1.10 is too large to
store in a Decimal128 of precision 2. Max is 0.99";
+ assert_eq!(err, expected_error);
+ }
+
+ #[test]
+ fn test_cast_float16_to_decimal256_precision_overflow() {
+ let array = Float16Array::from(vec![f16::from_f32(1.1)]);
+ let array = Arc::new(array) as ArrayRef;
+ let casted_array = cast_with_options(
+ &array,
+ &DataType::Decimal256(2, 2),
+ &CastOptions {
+ safe: true,
+ format_options: FormatOptions::default(),
+ },
+ );
+ assert!(casted_array.is_ok());
+ assert!(casted_array.unwrap().is_null(0));
+
+ let casted_array = cast_with_options(
+ &array,
+ &DataType::Decimal256(2, 2),
+ &CastOptions {
+ safe: false,
+ format_options: FormatOptions::default(),
+ },
+ );
+ let err = casted_array.unwrap_err().to_string();
+ let expected_error = "Invalid argument error: 1.10 is too large to
store in a Decimal256 of precision 2. Max is 0.99";
+ assert_eq!(err, expected_error);
+ }
+
+ #[test]
+ fn test_cast_float16_to_decimal128_non_finite() {
+ let array = Float16Array::from(vec![f16::NAN, f16::INFINITY,
f16::NEG_INFINITY]);
+ let array = Arc::new(array) as ArrayRef;
+ let casted_array = cast_with_options(
+ &array,
+ &DataType::Decimal128(38, 2),
+ &CastOptions {
+ safe: true,
+ format_options: FormatOptions::default(),
+ },
+ )
+ .unwrap();
+
+ assert!(casted_array.is_null(0));
+ assert!(casted_array.is_null(1));
+ assert!(casted_array.is_null(2));
+
+ let casted_array = cast_with_options(
+ &array,
+ &DataType::Decimal128(38, 2),
+ &CastOptions {
+ safe: false,
+ format_options: FormatOptions::default(),
+ },
+ );
+ let err = casted_array.unwrap_err().to_string();
+ let expected_error = "Cannot cast to Decimal128(38, 2)";
+ assert!(
+ err.contains(expected_error),
+ "did not find expected error '{expected_error}' in actual error
'{err}'"
+ );
+ }
+
#[test]
fn test_cast_floating_point_to_decimal256_precision_overflow() {
let array = Float64Array::from(vec![1.1]);
