kazuyukitanimura commented on code in PR #6830:
URL: https://github.com/apache/datafusion-comet/pull/6830#discussion_r4272952763
##########
native/spark-expr/src/agg_funcs/correlation.rs:
##########
@@ -594,6 +640,199 @@ mod groups_tests {
}
}
+ fn assert_divide_by_zero<T: std::fmt::Debug>(result: Result<T>) {
+ let Err(DataFusionError::External(error)) = result else {
+ panic!("expected a structured Spark divide-by-zero error, got
{result:?}");
+ };
+ assert!(matches!(
+ error.downcast_ref::<SparkError>(),
+ Some(SparkError::DivideByZero)
+ ));
+ }
+
+ fn partial_state(input: &[ArrayRef]) -> Vec<ArrayRef> {
+ let mut partial =
+ CorrelationAccumulator::try_new(true,
zero_divisor(EvalMode::Legacy)).unwrap();
+ partial.update_batch(input).unwrap();
+ partial
+ .state()
+ .unwrap()
+ .iter()
+ .map(|s| s.to_array_of_size(1).unwrap())
+ .collect()
+ }
+
+ #[test]
+ fn correlation_zero_denominator_respects_eval_mode() {
+ // https://github.com/apache/datafusion-comet/issues/6481
+ // x is constant at 0.1, which binary floating point cannot represent
+ // exactly. Spark divides ck by sqrt(xMk * yMk) = 0, so its Divide
+ // returns null in LEGACY mode and raises DIVIDE_BY_ZERO in ANSI mode.
+ let y: Vec<f64> = (0..6).map(f64::from).collect();
+ let x = [0.1; 6];
+ let input = |range: std::ops::Range<usize>| -> Vec<ArrayRef> {
+ vec![
+ Arc::new(Float64Array::from(y[range.clone()].to_vec())),
+ Arc::new(Float64Array::from(x[range].to_vec())),
+ ]
+ };
+ for eval_mode in [EvalMode::Legacy, EvalMode::Ansi] {
+ for null_on_divide_by_zero in [false, true] {
Review Comment:
Done in 0cc2c7f75. null_on_divide_by_zero is fixed to true there, with a
comment that only the count == 1 branch reads it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
