This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 442dfd55 fix(variant): truncate decimals and floats in variant_get int
casts (#945)
442dfd55 is described below
commit 442dfd553570b4256799b8fcc8ba361350205b05
Author: jackylee <[email protected]>
AuthorDate: Fri Sep 25 20:50:22 2026 +0800
fix(variant): truncate decimals and floats in variant_get int casts (#945)
---
.../datafusion/src/variant_functions.rs | 66 +++++++++++++++++++++-
crates/paimon/src/variant.rs | 60 +++++++++++++++++++-
2 files changed, 122 insertions(+), 4 deletions(-)
diff --git a/crates/integrations/datafusion/src/variant_functions.rs
b/crates/integrations/datafusion/src/variant_functions.rs
index 6eb8ab85..bf658fcc 100644
--- a/crates/integrations/datafusion/src/variant_functions.rs
+++ b/crates/integrations/datafusion/src/variant_functions.rs
@@ -454,13 +454,47 @@ fn cast_to_i64(variant: VariantRef<'_>) -> DFResult<i64> {
.map_err(|_| invalid_cast()),
VariantKind::Decimal => {
let decimal = variant.get_decimal().map_err(to_df_error)?;
- rescale_decimal(decimal.unscaled, decimal.scale, 0)
- .and_then(|v| i64::try_from(v).map_err(|_| invalid_cast()))
+ decimal_to_i64_truncating(decimal.unscaled, decimal.scale)
+ }
+ VariantKind::Double =>
f64_to_i64_truncating(variant.get_double().map_err(to_df_error)?),
+ VariantKind::Float => {
+ f64_to_i64_truncating(variant.get_float().map_err(to_df_error)? as
f64)
}
_ => Err(invalid_cast()),
}
}
+/// Cast a variant decimal to i64 the way Spark/Java `VariantGet` does: drop
the fractional
+/// digits toward zero (`RoundingMode.DOWN`), then require the result to fit.
Rust integer
+/// division truncates toward zero, so this matches Java `DOWN` for negatives
too.
+fn decimal_to_i64_truncating(unscaled: i128, scale: i8) -> DFResult<i64> {
+ let integral = if scale <= 0 {
+ let factor = 10_i128
+ .checked_pow(-(scale as i32) as u32)
+ .ok_or_else(invalid_cast)?;
+ unscaled.checked_mul(factor).ok_or_else(invalid_cast)?
+ } else {
+ let factor = 10_i128.checked_pow(scale as
u32).ok_or_else(invalid_cast)?;
+ unscaled / factor
+ };
+ i64::try_from(integral).map_err(|_| invalid_cast())
+}
+
+/// Cast a variant float/double to i64 by truncating toward zero, rejecting
NaN, infinity,
+/// and out-of-range values -- mirroring Java `VariantGet`'s
`RoundingMode.DOWN` plus its
+/// integral-fit check.
+fn f64_to_i64_truncating(value: f64) -> DFResult<i64> {
+ let truncated = value.trunc();
+ // The representable i64 range as f64 is [-2^63, 2^63); 2^63 is i64::MAX +
1.
+ if truncated.is_finite()
+ &&
(-9_223_372_036_854_775_808.0..9_223_372_036_854_775_808.0).contains(&truncated)
+ {
+ Ok(truncated as i64)
+ } else {
+ Err(invalid_cast())
+ }
+}
+
fn cast_to_f64(variant: VariantRef<'_>) -> DFResult<f64> {
match variant.kind().map_err(to_df_error)? {
VariantKind::Long
@@ -874,4 +908,32 @@ mod tests {
.to_string()
.contains("variant_get type argument must be a string literal"));
}
+
+ #[tokio::test]
+ async fn variant_get_int_truncates_decimals_and_floats_like_spark() {
+ // Spark/Java cast a JSON number to an integer with RoundingMode.DOWN
(truncate
+ // toward zero). Rust previously rejected any decimal with a
fractional part and
+ // every float, so this whole query used to error / return NULL.
+ let batch = collect_one(
+ r#"SELECT
+ variant_get(parse_json('{"a":19.99}'), '$.a', 'int') AS pos_dec,
+ variant_get(parse_json('{"a":-3.9}'), '$.a', 'int') AS neg_dec,
+ try_variant_get(parse_json('{"a":19.99}'), '$.a', 'int') AS
try_dec,
+ variant_get(parse_json('{"a":2.5e0}'), '$.a', 'int') AS flt
+ "#,
+ )
+ .await;
+ let col = |i: usize| {
+ batch
+ .column(i)
+ .as_any()
+ .downcast_ref::<Int32Array>()
+ .unwrap()
+ .value(0)
+ };
+ assert_eq!(col(0), 19, "19.99 truncates to 19");
+ assert_eq!(col(1), -3, "-3.9 truncates toward zero to -3, not -4");
+ assert_eq!(col(2), 19, "try_variant_get truncates too");
+ assert_eq!(col(3), 2, "a fractional number truncates to 2");
+ }
}
diff --git a/crates/paimon/src/variant.rs b/crates/paimon/src/variant.rs
index b7530841..2199e621 100644
--- a/crates/paimon/src/variant.rs
+++ b/crates/paimon/src/variant.rs
@@ -2713,13 +2713,43 @@ fn cast_variant_to_i64(variant: VariantRef<'_>) ->
Option<i64> {
VariantKind::String => variant.get_string().ok()?.parse::<i64>().ok(),
VariantKind::Decimal => {
let decimal = variant.get_decimal().ok()?;
- rescale_decimal_exact(decimal.unscaled, decimal.scale, 0)
- .and_then(|value| i64::try_from(value).ok())
+ decimal_to_i64_truncating(decimal.unscaled, decimal.scale)
}
+ VariantKind::Double =>
f64_to_i64_truncating(variant.get_double().ok()?),
+ VariantKind::Float => f64_to_i64_truncating(variant.get_float().ok()?
as f64),
_ => None,
}
}
+/// Cast a variant decimal to i64 the way Spark/Java `VariantGet` does: drop
the fractional
+/// digits toward zero (`RoundingMode.DOWN`), returning `None` when it
overflows i64. Rust
+/// integer division truncates toward zero, so this matches Java `DOWN` for
negatives too.
+fn decimal_to_i64_truncating(unscaled: i128, scale: i8) -> Option<i64> {
+ let integral = if scale <= 0 {
+ let factor = 10_i128.checked_pow((-(scale as i32)) as u32)?;
+ unscaled.checked_mul(factor)?
+ } else {
+ let factor = 10_i128.checked_pow(scale as u32)?;
+ unscaled / factor
+ };
+ i64::try_from(integral).ok()
+}
+
+/// Cast a variant float/double to i64 by truncating toward zero, returning
`None` for NaN,
+/// infinity, and out-of-range values -- mirroring Java `VariantGet`'s
`RoundingMode.DOWN`
+/// plus its integral-fit check.
+fn f64_to_i64_truncating(value: f64) -> Option<i64> {
+ let truncated = value.trunc();
+ // The representable i64 range as f64 is [-2^63, 2^63); 2^63 is i64::MAX +
1.
+ if truncated.is_finite()
+ &&
(-9_223_372_036_854_775_808.0..9_223_372_036_854_775_808.0).contains(&truncated)
+ {
+ Some(truncated as i64)
+ } else {
+ None
+ }
+}
+
fn cast_variant_to_f64(variant: VariantRef<'_>) -> Option<f64> {
match variant.kind().ok()? {
VariantKind::Long
@@ -3510,4 +3540,30 @@ mod tests {
VariantKind::Double
);
}
+
+ #[test]
+ fn cast_variant_to_i64_truncates_decimals_and_floats() {
+ // Pushdown-extraction counterpart of the variant_get UDF int cast:
Spark/Java
+ // truncate a JSON number toward zero (RoundingMode.DOWN) when casting
to an
+ // integer. This path previously returned None (NULL / "cannot cast")
for any
+ // decimal with a fractional part and every float.
+ let pos = GenericVariant::parse_json(r#"{"a":19.99}"#).unwrap();
+ assert_eq!(
+ cast_variant_to_i64(pos.get_path("$.a").unwrap().unwrap()),
+ Some(19),
+ "19.99 truncates to 19"
+ );
+ let neg = GenericVariant::parse_json(r#"{"a":-3.9}"#).unwrap();
+ assert_eq!(
+ cast_variant_to_i64(neg.get_path("$.a").unwrap().unwrap()),
+ Some(-3),
+ "-3.9 truncates toward zero to -3, not -4"
+ );
+ let dbl = GenericVariant::parse_json(r#"{"a":2.5e0}"#).unwrap();
+ assert_eq!(
+ cast_variant_to_i64(dbl.get_path("$.a").unwrap().unwrap()),
+ Some(2),
+ "2.5 truncates to 2"
+ );
+ }
}
