asolimando commented on code in PR #24536:
URL: https://github.com/apache/datafusion/pull/24536#discussion_r3832129394
##########
datafusion/physical-plan/src/joins/utils.rs:
##########
Review Comment:
This is stale now
##########
datafusion/physical-plan/src/joins/utils.rs:
##########
@@ -466,12 +470,42 @@ pub(crate) fn estimate_join_statistics(
join_type: &JoinType,
schema: &Schema,
) -> Result<Statistics> {
+ // Width of one output row, from the sides this join emits. Without it a
join reports
+ // no size and `hash_join_single_partition_threshold` falls back to
counting rows.
+ let width = |stats: &Statistics| match (
+ stats.total_byte_size.get_value(),
+ stats.num_rows.get_value(),
+ ) {
+ (Some(bytes), Some(rows)) if *rows > 0 => Some(*bytes as f64 / *rows
as f64),
+ _ => None,
+ };
+ // The boolean a mark join appends is one bit per row.
+ const MARK_COLUMN_WIDTH: f64 = 1.0 / 8.0;
+ let output_width = match join_type {
+ JoinType::LeftSemi | JoinType::LeftAnti => width(&left_stats),
+ JoinType::RightSemi | JoinType::RightAnti => width(&right_stats),
+ JoinType::Inner | JoinType::Left | JoinType::Right | JoinType::Full =>
{
+ width(&left_stats)
+ .zip(width(&right_stats))
+ .map(|(left, right)| left + right)
+ }
+ JoinType::LeftMark => width(&left_stats).map(|w| w +
MARK_COLUMN_WIDTH),
+ JoinType::RightMark => width(&right_stats).map(|w| w +
MARK_COLUMN_WIDTH),
+ };
+
let join_stats =
estimate_join_cardinality(join_type, left_stats, right_stats, on,
null_equality);
let (num_rows, total_byte_size, column_statistics) = match join_stats {
Some(stats) => (
Precision::Inexact(stats.num_rows),
- stats.total_byte_size,
+ match (stats.total_byte_size, output_width) {
+ // Only fill a gap: a size the join derived from its column
statistics
+ // knows which columns it keeps, which an average row width
cannot.
+ (Precision::Absent, Some(width)) => {
+ Precision::Inexact((stats.num_rows as f64 * width) as
usize)
Review Comment:
Minor: maybe we could round here instead of truncating?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
