asolimando commented on code in PR #27574:
URL: https://github.com/apache/datafusion/pull/27574#discussion_r4635064259
##########
datafusion/sqllogictest/test_files/parquet_statistics.slt:
##########
@@ -409,6 +409,50 @@ FROM (SELECT 1 AS x, 2 AS y FROM literal_ndv_empty);
statement ok
DROP TABLE literal_ndv_empty;
+# The years of the date bounds limit the groups of the final aggregate. The
+# key is nullable and its NULL count is unknown, so one NULL group is added to
Review Comment:
`d` has `Null=Exact(0)`, so the NULL count is known here. The extra group
comes from `expression_bounds_statistics` dropping `null_count`. Could
`date_part` keep the input NULL count? Otherwise, maybe update this comment.
##########
datafusion/physical-plan/src/aggregates/mod.rs:
##########
@@ -1584,30 +1586,42 @@ impl AggregateExec {
/// - Per-set products are summed across all grouping sets
/// - Requires NDV stats for ALL active group-by columns; if any lacks
stats,
/// falls back to `input_rows` (or `Absent` if that is also unknown)
+ /// - A plain `GROUP BY` with final output also accepts integer bounds for
+ /// a key without NDV (see [`Self::final_group_key_count`])
fn statistics_inner(
&self,
child_statistics: &Statistics,
partition: Option<usize>,
) -> Result<Statistics> {
- // TODO stats: group expressions:
- // - once expressions will be able to compute their own stats, use it
here
- // - case where we group by on a column for which with have the
`distinct` stat
// TODO stats: aggr expression:
// - aggregations sometimes also preserve invariants such as min,
max...
+ // A partial aggregate can emit a group more than once.
+ let use_range_bounds = self.group_by().is_single()
+ && self.mode.output_mode() == AggregateOutputMode::Final;
+ let mut final_group_count = use_range_bounds.then_some(1usize);
+ let input_schema = self.input.schema();
let column_statistics = {
// self.schema: [<group by exprs>, <aggregate exprs>]
let mut column_statistics =
Statistics::unknown_column(&self.schema());
for (idx, (expr, _)) in self.group_by().expr.iter().enumerate() {
- if let Some(col) = expr.downcast_ref::<Column>() {
- let child_col_stats =
- &child_statistics.column_statistics[col.index()];
- column_statistics[idx].max_value =
child_col_stats.max_value.clone();
- column_statistics[idx].min_value =
child_col_stats.min_value.clone();
- column_statistics[idx].distinct_count =
- child_col_stats.distinct_count;
- }
+ let key_stats = Self::group_key_statistics(
+ expr.as_ref(),
+ child_statistics,
+ &input_schema,
+ );
+ final_group_count = final_group_count.and_then(|product| {
+ let groups = Self::final_group_key_count(
+ expr.as_ref(),
+ &key_stats,
+ &input_schema,
+ )?;
+ Some(product.saturating_mul(groups))
+ });
+ column_statistics[idx].max_value = key_stats.max_value;
+ column_statistics[idx].min_value = key_stats.min_value;
+ column_statistics[idx].distinct_count =
key_stats.distinct_count;
Review Comment:
The Partial output does not keep `null_count`, so the Final aggregate adds a
NULL group that Single mode does not add. A nullable key with no NULLs in [1,
7] gets 8 rows with 2 partitions and 7 rows with 1 partition. The query returns
7.
```
statement ok
COPY (SELECT CASE WHEN value < 0 THEN NULL ELSE CAST(value % 7 + 1 AS INT)
END AS k FROM range(0, 10000))
TO 'test_files/scratch/parquet_statistics/k.parquet' STORED AS PARQUET;
statement ok
CREATE EXTERNAL TABLE kt STORED AS PARQUET
LOCATION 'test_files/scratch/parquet_statistics/k.parquet';
statement ok
SET datafusion.explain.show_statistics = true;
statement ok
SET datafusion.execution.target_partitions = 2;
# FinalPartitioned: Rows=Inexact(8)
query TT
EXPLAIN SELECT k, count(*) FROM kt GROUP BY k;
statement ok
SET datafusion.execution.target_partitions = 1;
# Single: Rows=Inexact(7)
query TT
EXPLAIN SELECT k, count(*) FROM kt GROUP BY k;
```
Copying `null_count` for a plain `GROUP BY` (not for grouping sets) should
fix it.
##########
datafusion/physical-expr/src/projection.rs:
##########
@@ -937,6 +987,70 @@ fn project_column_statistics_through_expr(
}
}
+/// Bounds `expr` by [`PhysicalExpr::evaluate_bounds`] over the bounds of its
+/// inputs. These bounds enclose the non-NULL values of `expr`, but they can be
+/// loose: `sin` reports `[-1, 1]` for every input. The bounds are therefore
+/// inexact, so that they never answer `MIN` or `MAX` queries. All other
+/// statistics are unknown.
+fn expression_bounds_statistics(
+ expr: &dyn PhysicalExpr,
+ column_stats: &[ColumnStatistics],
+ input_schema: Option<&Schema>,
+) -> ColumnStatistics {
+ let input_bounds = expr
+ .children()
+ .into_iter()
+ .map(|child| {
+ let stats = project_column_statistics_through_expr(
+ child.as_ref(),
+ column_stats,
+ input_schema,
+ );
+ match (stats.min_value.get_value(), stats.max_value.get_value()) {
+ (Some(min), Some(max)) if !min.is_null() && !max.is_null() => {
+ Interval::try_new(min.clone(), max.clone()).ok()
+ }
+ _ => None,
+ }
+ })
+ .collect::<Option<Vec<_>>>();
+ let bounds = input_bounds
+ .filter(|input_bounds| has_interval_support(expr, input_bounds))
+ .and_then(|input_bounds| {
+ let input_bounds = input_bounds.iter().collect::<Vec<_>>();
+ expr.evaluate_bounds(&input_bounds).ok()
+ });
+ match bounds {
+ Some(bounds) if !bounds.is_unbounded() => {
+ let (min, max) = bounds.into_bounds();
+ ColumnStatistics::new_unknown()
+ .with_min_value(Precision::Inexact(min))
+ .with_max_value(Precision::Inexact(max))
+ }
+ _ => ColumnStatistics::new_unknown(),
+ }
+}
+
+/// Whether `expr` can compute bounds from `input_bounds`. Arithmetic follows
+/// the rules of interval analysis in [`check_support`]. Scalar functions
+/// decide in their own [`PhysicalExpr::evaluate_bounds`].
+///
+/// [`check_support`]: crate::intervals::utils::check_support
+fn has_interval_support(expr: &dyn PhysicalExpr, input_bounds: &[Interval]) ->
bool {
+ let supported_inputs = || {
+ input_bounds
+ .iter()
+ .all(|bounds| is_datatype_supported(&bounds.data_type()))
+ };
+ if let Some(binary) = expr.downcast_ref::<BinaryExpr>() {
+ is_operator_supported(binary.op()) && supported_inputs()
+ } else if expr.is::<NegativeExpr>() {
+ supported_inputs()
+ } else {
+ expr.is::<ScalarFunctionExpr>()
Review Comment:
`radians_bounds` returns a fixed (-pi, pi), which is wrong (`radians(720)`
is about 12.57). Since this PR uses these bounds for statistics, the join below
is estimated at 0 rows but returns 301. The merge base estimates 301.
```
statement ok
COPY (SELECT CAST(value AS DOUBLE) AS a FROM range(0, 721))
TO 'test_files/scratch/parquet_statistics/deg.parquet' STORED AS PARQUET;
statement ok
CREATE EXTERNAL TABLE deg STORED AS PARQUET
LOCATION 'test_files/scratch/parquet_statistics/deg.parquet';
statement ok
COPY (SELECT radians(CAST(value AS DOUBLE)) AS x FROM range(300, 601))
TO 'test_files/scratch/parquet_statistics/rad.parquet' STORED AS PARQUET;
statement ok
CREATE EXTERNAL TABLE rad STORED AS PARQUET
LOCATION 'test_files/scratch/parquet_statistics/rad.parquet';
statement ok
SET datafusion.explain.show_statistics = true;
query TT
EXPLAIN SELECT * FROM (SELECT radians(a) AS r FROM deg) s JOIN rad ON s.r =
rad.x;
```
```
HashJoinExec: ..., statistics=[Rows=Inexact(0), ...
DataSourceExec: ..., projection=[radians(a@0) as r], ...
Min=Inexact(Float64(-3.1415926535897936))
Max=Inexact(Float64(3.1415926535897936))
```
Computing `x * pi / 180` from the input interval in `radians_bounds` should
fix it. The other functions in `bounds.rs` look fine.
##########
datafusion/physical-plan/src/aggregates/mod.rs:
##########
@@ -1800,7 +1816,7 @@ impl AggregateExec {
num_rows
}
} else {
- match (ndv, limit) {
+ match (group_count, limit) {
(Some(n), Some(l)) => Precision::Inexact(n.min(l)),
(Some(n), None) => Precision::Inexact(n),
Review Comment:
When the input row count is unknown, this now returns the range size as the
estimate, for example `Inexact(2147483647)` for bounds [1, i32::MAX]. Before,
it returned Absent. The range size is only an upper bound, so I would use it
only to cap a known row count.
##########
datafusion/physical-plan/src/aggregates/mod.rs:
##########
@@ -1849,6 +1863,48 @@ impl AggregateExec {
Some(total)
}
+ /// Estimates the contribution of one key to a plain `GROUP BY` with final
+ /// output. A key without NDV uses the number of values between its integer
+ /// bounds, plus one NULL group unless the key has no NULLs.
+ ///
+ /// **Example:** `GROUP BY date_part('year', d)` where `d` is in
+ /// `[1992-01-02, 1998-12-01]` and the key is nullable with an unknown
+ /// NULL count → 1998 - 1992 + 1 = 7 years, + 1 NULL group = 8
+ fn final_group_key_count(
+ expr: &dyn PhysicalExpr,
+ stats: &ColumnStatistics,
+ input_schema: &Schema,
+ ) -> Option<usize> {
+ if let Some(&ndv) = stats.distinct_count.get_value() {
Review Comment:
For an integer key, the range size is an upper bound for the NDV. With
bounds [1, 7] and an Inexact NDV of 1000, this returns 1000. Maybe use the
smaller of the two?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]