fwojciec commented on code in PR #20391:
URL: https://github.com/apache/datafusion/pull/20391#discussion_r2814708455
##########
datafusion/physical-plan/src/filter.rs:
##########
@@ -2053,4 +2064,43 @@ mod tests {
Ok(())
}
+
+ /// Regression test: columns with Absent min/max statistics must remain
+ /// Absent after FilterExec, not be converted to Exact(NULL). The latter
+ /// causes `estimate_disjoint_inputs` to incorrectly conclude join inputs
+ /// are disjoint (ScalarValue's PartialOrd sorts NULLs last), producing
+ /// zero cardinality and forcing Partitioned join mode.
Review Comment:
Done (apologies, should have just "Committed suggestion") - the result is
the same though.
##########
datafusion/physical-plan/src/filter.rs:
##########
@@ -771,10 +771,21 @@ fn collect_new_statistics(
};
};
let (lower, upper) = interval.into_bounds();
- let (min_value, max_value) = if lower.eq(&upper) {
- (Precision::Exact(lower), Precision::Exact(upper))
+ let bounds_equal =
+ !lower.is_null() && !upper.is_null() && lower.eq(&upper);
+ let min_value = if lower.is_null() {
+ Precision::Absent
+ } else if bounds_equal {
+ Precision::Exact(lower)
} else {
- (Precision::Inexact(lower), Precision::Inexact(upper))
+ Precision::Inexact(lower)
+ };
+ let max_value = if upper.is_null() {
+ Precision::Absent
+ } else if bounds_equal {
+ Precision::Exact(upper)
+ } else {
+ Precision::Inexact(upper)
Review Comment:
Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
