JingsongLi commented on code in PR #907:
URL: https://github.com/apache/paimon-rust/pull/907#discussion_r4067960775
##########
crates/paimon/src/arrow/nested_evolution.rs:
##########
@@ -64,6 +66,62 @@ pub(crate) fn evolve_column(
return Ok(source.clone());
}
+ if let (
+ ArrowDataType::Decimal128(_, source_scale),
+ ArrowDataType::Decimal128(target_precision, target_scale),
+ ) = (source.data_type(), &target_arrow)
+ {
+ if source_scale > target_scale {
+ // Arrow rounds Decimal128 scale reductions. Paimon schema casts,
+ // like Python's unsafe Arrow cast, truncate toward zero instead.
+ let shift = u32::try_from(i16::from(*source_scale) -
i16::from(*target_scale))
+ .map_err(|_| crate::Error::DataInvalid {
+ message: "invalid decimal scale reduction".to_string(),
+ source: None,
+ })?;
+ let divisor = 10_i128
+ .checked_pow(shift)
+ .ok_or_else(|| crate::Error::DataInvalid {
+ message: "decimal scale reduction exceeds Decimal128
range".to_string(),
+ source: None,
+ })?;
+ let decimals = source
+ .as_any()
+ .downcast_ref::<Decimal128Array>()
+ .ok_or_else(|| crate::Error::DataInvalid {
+ message: format!("expected Decimal128 array, got {:?}",
source.data_type()),
+ source: None,
+ })?;
+ let reduced = Decimal128Array::from(
+ (0..decimals.len())
+ .map(|index| {
+ decimals
+ .is_valid(index)
+ .then(|| decimals.value(index) / divisor)
+ })
+ .collect::<Vec<_>>(),
+ )
+ .with_precision_and_scale(*target_precision, *target_scale)
Review Comment:
Fixed in aeba0cdf. Arrow safe cast now nulls values that exceed the target
precision after rounding. The regression covers precision+scale reduction,
rounding-induced overflow, negative overflow, and
validate_decimal_precision(3); it failed before the fix and passes now. Full
Rust lib suite: 3009 passed, 6 ignored.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
