asolimando commented on code in PR #27574:
URL: https://github.com/apache/datafusion/pull/27574#discussion_r4635064259


##########
datafusion/sqllogictest/test_files/parquet_statistics.slt:
##########
@@ -409,6 +409,50 @@ FROM (SELECT 1 AS x, 2 AS y FROM literal_ndv_empty);
 statement ok
 DROP TABLE literal_ndv_empty;
 
+# The years of the date bounds limit the groups of the final aggregate. The
+# key is nullable and its NULL count is unknown, so one NULL group is added to

Review Comment:
   `d` has `Null=Exact(0)`, so the NULL count is known here. The extra group 
comes from `expression_bounds_statistics` dropping `null_count`. Could 
`date_part` keep the input NULL count? Otherwise, maybe update this comment.



##########
datafusion/physical-plan/src/aggregates/mod.rs:
##########
@@ -1584,30 +1586,42 @@ impl AggregateExec {
     /// - Per-set products are summed across all grouping sets
     /// - Requires NDV stats for ALL active group-by columns; if any lacks 
stats,
     ///   falls back to `input_rows` (or `Absent` if that is also unknown)
+    /// - A plain `GROUP BY` with final output also accepts integer bounds for
+    ///   a key without NDV (see [`Self::final_group_key_count`])
     fn statistics_inner(
         &self,
         child_statistics: &Statistics,
         partition: Option<usize>,
     ) -> Result<Statistics> {
-        // TODO stats: group expressions:
-        // - once expressions will be able to compute their own stats, use it 
here
-        // - case where we group by on a column for which with have the 
`distinct` stat
         // TODO stats: aggr expression:
         // - aggregations sometimes also preserve invariants such as min, 
max...
 
+        // A partial aggregate can emit a group more than once.
+        let use_range_bounds = self.group_by().is_single()
+            && self.mode.output_mode() == AggregateOutputMode::Final;
+        let mut final_group_count = use_range_bounds.then_some(1usize);
+        let input_schema = self.input.schema();
         let column_statistics = {
             // self.schema: [<group by exprs>, <aggregate exprs>]
             let mut column_statistics = 
Statistics::unknown_column(&self.schema());
 
             for (idx, (expr, _)) in self.group_by().expr.iter().enumerate() {
-                if let Some(col) = expr.downcast_ref::<Column>() {
-                    let child_col_stats =
-                        &child_statistics.column_statistics[col.index()];
-                    column_statistics[idx].max_value = 
child_col_stats.max_value.clone();
-                    column_statistics[idx].min_value = 
child_col_stats.min_value.clone();
-                    column_statistics[idx].distinct_count =
-                        child_col_stats.distinct_count;
-                }
+                let key_stats = Self::group_key_statistics(
+                    expr.as_ref(),
+                    child_statistics,
+                    &input_schema,
+                );
+                final_group_count = final_group_count.and_then(|product| {
+                    let groups = Self::final_group_key_count(
+                        expr.as_ref(),
+                        &key_stats,
+                        &input_schema,
+                    )?;
+                    Some(product.saturating_mul(groups))
+                });
+                column_statistics[idx].max_value = key_stats.max_value;
+                column_statistics[idx].min_value = key_stats.min_value;
+                column_statistics[idx].distinct_count = 
key_stats.distinct_count;

Review Comment:
   The Partial output does not keep `null_count`, so the Final aggregate adds a 
NULL group that Single mode does not add. A nullable key with no NULLs in [1, 
7] gets 8 rows with 2 partitions and 7 rows with 1 partition. The query returns 
7.
   
   ```
   statement ok
   COPY (SELECT CASE WHEN value < 0 THEN NULL ELSE CAST(value % 7 + 1 AS INT) 
END AS k FROM range(0, 10000))
   TO 'test_files/scratch/parquet_statistics/k.parquet' STORED AS PARQUET;
   
   statement ok
   CREATE EXTERNAL TABLE kt STORED AS PARQUET
   LOCATION 'test_files/scratch/parquet_statistics/k.parquet';
   
   statement ok
   SET datafusion.explain.show_statistics = true;
   
   statement ok
   SET datafusion.execution.target_partitions = 2;
   
   # FinalPartitioned: Rows=Inexact(8)
   query TT
   EXPLAIN SELECT k, count(*) FROM kt GROUP BY k;
   
   statement ok
   SET datafusion.execution.target_partitions = 1;
   
   # Single: Rows=Inexact(7)
   query TT
   EXPLAIN SELECT k, count(*) FROM kt GROUP BY k;
   ```
   
   Copying `null_count` for a plain `GROUP BY` (not for grouping sets) should 
fix it.



##########
datafusion/physical-expr/src/projection.rs:
##########
@@ -937,6 +987,70 @@ fn project_column_statistics_through_expr(
     }
 }
 
+/// Bounds `expr` by [`PhysicalExpr::evaluate_bounds`] over the bounds of its
+/// inputs. These bounds enclose the non-NULL values of `expr`, but they can be
+/// loose: `sin` reports `[-1, 1]` for every input. The bounds are therefore
+/// inexact, so that they never answer `MIN` or `MAX` queries. All other
+/// statistics are unknown.
+fn expression_bounds_statistics(
+    expr: &dyn PhysicalExpr,
+    column_stats: &[ColumnStatistics],
+    input_schema: Option<&Schema>,
+) -> ColumnStatistics {
+    let input_bounds = expr
+        .children()
+        .into_iter()
+        .map(|child| {
+            let stats = project_column_statistics_through_expr(
+                child.as_ref(),
+                column_stats,
+                input_schema,
+            );
+            match (stats.min_value.get_value(), stats.max_value.get_value()) {
+                (Some(min), Some(max)) if !min.is_null() && !max.is_null() => {
+                    Interval::try_new(min.clone(), max.clone()).ok()
+                }
+                _ => None,
+            }
+        })
+        .collect::<Option<Vec<_>>>();
+    let bounds = input_bounds
+        .filter(|input_bounds| has_interval_support(expr, input_bounds))
+        .and_then(|input_bounds| {
+            let input_bounds = input_bounds.iter().collect::<Vec<_>>();
+            expr.evaluate_bounds(&input_bounds).ok()
+        });
+    match bounds {
+        Some(bounds) if !bounds.is_unbounded() => {
+            let (min, max) = bounds.into_bounds();
+            ColumnStatistics::new_unknown()
+                .with_min_value(Precision::Inexact(min))
+                .with_max_value(Precision::Inexact(max))
+        }
+        _ => ColumnStatistics::new_unknown(),
+    }
+}
+
+/// Whether `expr` can compute bounds from `input_bounds`. Arithmetic follows
+/// the rules of interval analysis in [`check_support`]. Scalar functions
+/// decide in their own [`PhysicalExpr::evaluate_bounds`].
+///
+/// [`check_support`]: crate::intervals::utils::check_support
+fn has_interval_support(expr: &dyn PhysicalExpr, input_bounds: &[Interval]) -> 
bool {
+    let supported_inputs = || {
+        input_bounds
+            .iter()
+            .all(|bounds| is_datatype_supported(&bounds.data_type()))
+    };
+    if let Some(binary) = expr.downcast_ref::<BinaryExpr>() {
+        is_operator_supported(binary.op()) && supported_inputs()
+    } else if expr.is::<NegativeExpr>() {
+        supported_inputs()
+    } else {
+        expr.is::<ScalarFunctionExpr>()

Review Comment:
   `radians_bounds` returns a fixed (-pi, pi), which is wrong (`radians(720)` 
is about 12.57). Since this PR uses these bounds for statistics, the join below 
is estimated at 0 rows but returns 301. The merge base estimates 301.
   
   ```
   statement ok
   COPY (SELECT CAST(value AS DOUBLE) AS a FROM range(0, 721))
   TO 'test_files/scratch/parquet_statistics/deg.parquet' STORED AS PARQUET;
   
   statement ok
   CREATE EXTERNAL TABLE deg STORED AS PARQUET
   LOCATION 'test_files/scratch/parquet_statistics/deg.parquet';
   
   statement ok
   COPY (SELECT radians(CAST(value AS DOUBLE)) AS x FROM range(300, 601))
   TO 'test_files/scratch/parquet_statistics/rad.parquet' STORED AS PARQUET;
   
   statement ok
   CREATE EXTERNAL TABLE rad STORED AS PARQUET
   LOCATION 'test_files/scratch/parquet_statistics/rad.parquet';
   
   statement ok
   SET datafusion.explain.show_statistics = true;
   
   query TT
   EXPLAIN SELECT * FROM (SELECT radians(a) AS r FROM deg) s JOIN rad ON s.r = 
rad.x;
   ```
   
   ```
   HashJoinExec: ..., statistics=[Rows=Inexact(0), ...
     DataSourceExec: ..., projection=[radians(a@0) as r], ... 
Min=Inexact(Float64(-3.1415926535897936)) 
Max=Inexact(Float64(3.1415926535897936))
   ```
   
   Computing `x * pi / 180` from the input interval in `radians_bounds` should 
fix it. The other functions in `bounds.rs` look fine.



##########
datafusion/physical-plan/src/aggregates/mod.rs:
##########
@@ -1800,7 +1816,7 @@ impl AggregateExec {
                 num_rows
             }
         } else {
-            match (ndv, limit) {
+            match (group_count, limit) {
                 (Some(n), Some(l)) => Precision::Inexact(n.min(l)),
                 (Some(n), None) => Precision::Inexact(n),

Review Comment:
   When the input row count is unknown, this now returns the range size as the 
estimate, for example `Inexact(2147483647)` for bounds [1, i32::MAX]. Before, 
it returned Absent. The range size is only an upper bound, so I would use it 
only to cap a known row count.



##########
datafusion/physical-plan/src/aggregates/mod.rs:
##########
@@ -1849,6 +1863,48 @@ impl AggregateExec {
         Some(total)
     }
 
+    /// Estimates the contribution of one key to a plain `GROUP BY` with final
+    /// output. A key without NDV uses the number of values between its integer
+    /// bounds, plus one NULL group unless the key has no NULLs.
+    ///
+    /// **Example:** `GROUP BY date_part('year', d)` where `d` is in
+    /// `[1992-01-02, 1998-12-01]` and the key is nullable with an unknown
+    /// NULL count → 1998 - 1992 + 1 = 7 years, + 1 NULL group = 8
+    fn final_group_key_count(
+        expr: &dyn PhysicalExpr,
+        stats: &ColumnStatistics,
+        input_schema: &Schema,
+    ) -> Option<usize> {
+        if let Some(&ndv) = stats.distinct_count.get_value() {

Review Comment:
   For an integer key, the range size is an upper bound for the NDV. With 
bounds [1, 7] and an Inexact NDV of 1000, this returns 1000. Maybe use the 
smaller of the two?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to