isidentical commented on code in PR #3837:
URL: https://github.com/apache/arrow-datafusion/pull/3837#discussion_r996295008
##########
datafusion/core/src/physical_plan/join_utils.rs:
##########
@@ -447,6 +450,59 @@ fn estimate_inner_join_cardinality(
}
}
+/// Estimate the number of maximum distinct values that can be present in the
+/// given column from its statistics.
+///
+/// If distinct_count is available, uses it directly. If the column numeric,
and
+/// has min/max values, then they might be used as a fallback option.
Otherwise,
+/// returns None.
+fn max_distinct_count(num_rows: usize, stats: ColumnStatistics) ->
Option<usize> {
+ match (stats.distinct_count, stats.max_value, stats.min_value) {
+ (Some(_), _, _) => stats.distinct_count,
+ (_, Some(max), Some(min)) => {
+ // Note that float support is intentionally omitted here, since
the computation
+ // of a range between two float values is not trivial and the
result would be
+ // highly inaccurate.
+ let numeric_range = get_int_range(min, max)?;
+
+ // The number can never be greater than the number of rows we have
(minus
+ // the nulls, since they don't count as distinct values).
+ let ceiling = num_rows - stats.null_count.unwrap_or(0);
+ Some(numeric_range.min(ceiling))
+ }
+ _ => None,
+ }
+}
+
+/// Return the numeric range between the given min and max values.
+fn get_int_range(min: ScalarValue, max: ScalarValue) -> Option<usize> {
+ let delta = &max.sub(&min).ok()?;
+ match delta {
+ ScalarValue::Int8(Some(delta)) if *delta >= 0 => Some(*delta as usize),
+ ScalarValue::Int16(Some(delta)) if *delta >= 0 => Some(*delta as
usize),
+ ScalarValue::Int32(Some(delta)) if *delta >= 0 => Some(*delta as
usize),
+ ScalarValue::Int64(Some(delta)) if *delta >= 0 => Some(*delta as
usize),
+ ScalarValue::UInt8(Some(delta)) => Some(*delta as usize),
+ ScalarValue::UInt16(Some(delta)) => Some(*delta as usize),
+ ScalarValue::UInt32(Some(delta)) => Some(*delta as usize),
+ ScalarValue::UInt64(Some(delta)) => Some(*delta as usize),
+ _ => None,
+ }
+ // The delta (directly) is not the real range, since it does not include
the
+ // first term if it is different from the final term.
+ // E.g. (min=2, max=4) -> (4 - 2) -> 2, but the actual result should be 3
(1, 2, 3).
Review Comment:
Yes, and we should be already handling it (check out the comment in the
following line).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
