sunchao commented on code in PR #7598:
URL: https://github.com/apache/datafusion-comet/pull/7598#discussion_r4575305000


##########
native/core/src/execution/operators/dynamic_filter/min_max.rs:
##########
@@ -0,0 +1,315 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Connect one partial MIN/MAX aggregate's improving bound to its Parquet 
reader.
+
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use arrow::datatypes::DataType;
+use datafusion::common::config::ConfigOptions;
+use datafusion::common::tree_node::TreeNodeRecursion;
+use datafusion::common::{internal_err, Result, Statistics};
+use datafusion::datasource::source::DataSourceExec;
+use datafusion::execution::TaskContext;
+use datafusion::functions_aggregate::min_max::{Max, Min};
+use datafusion::physical_expr::expressions::{Column, 
DynamicFilterPhysicalExpr};
+use datafusion::physical_expr::{OrderingRequirements, PhysicalExpr};
+use datafusion::physical_plan::aggregates::{AggregateExec, AggregateMode};
+use 
datafusion::physical_plan::distribution_requirements::InputDistributionRequirements;
+use datafusion::physical_plan::execution_plan::CardinalityEffect;
+use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, 
MetricBuilder, MetricsSet};
+use datafusion::physical_plan::statistics::{ChildStats, StatisticsArgs};
+use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
+use datafusion::physical_plan::{
+    ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, 
ExecutionPlanProperties,
+    PlanProperties, ReplaceChildrenOptions, SendableRecordBatchStream,
+};
+use futures::TryStreamExt;
+
+use super::parquet_reader::{parquet_file_source, 
try_attach_parquet_reader_filter};
+
+mod row_group_limit;
+
+/// Own an unexecuted aggregate template and metric handles. Each execution 
owns
+/// a fresh aggregate, accumulator bounds, and reader predicate; none of that 
live
+/// state is retained by the template or shared with another Spark task.
+#[derive(Debug)]
+pub(crate) struct MinMaxReaderFilterExec {
+    template: AggregateExec,
+    config: ConfigOptions,
+    metrics: ExecutionPlanMetricsSet,
+}
+
+struct RuntimeAggregate {
+    aggregate: Arc<dyn ExecutionPlan>,
+    reader_filter_attached: bool,
+}
+
+impl MinMaxReaderFilterExec {
+    // Spark planner integration is a separate step of 
apache/datafusion-comet#5774.
+    #[cfg_attr(not(test), allow(dead_code))]
+    pub(crate) fn try_new(
+        aggregate: &AggregateExec,
+        config: &ConfigOptions,
+    ) -> Result<Option<Self>> {
+        if !config.optimizer.enable_dynamic_filter_pushdown
+            || !config.optimizer.enable_aggregate_dynamic_filter_pushdown
+            || aggregate.mode() != &AggregateMode::Partial
+            || !aggregate.group_expr().is_true_no_grouping()
+            || aggregate.limit_options().is_some()
+            || aggregate.filter_expr().iter().any(Option::is_some)
+            || aggregate.input().output_partitioning().partition_count() != 1
+            || aggregate.input().fetch().is_some()
+        {
+            return Ok(None);
+        }
+        let [expression] = aggregate.aggr_expr() else {
+            return Ok(None);
+        };
+        let function = expression.fun().inner();
+        if !(function.is::<Min>() || function.is::<Max>())
+            || expression.is_distinct()
+            || !expression.order_bys().is_empty()
+        {
+            return Ok(None);
+        }
+        let arguments = expression.expressions();
+        let [key] = arguments.as_slice() else {
+            return Ok(None);
+        };
+        if !key.is::<Column>()
+            || !matches!(
+                key.data_type(aggregate.input().schema().as_ref())?,
+                DataType::Int8 | DataType::Int16 | DataType::Int32 | 
DataType::Int64
+            )
+        {
+            return Ok(None);
+        }
+        // Keep expressions, residuals, limits, and native/JVM boundaries out 
of
+        // this first path. The shared helper still handles reader column 
mapping
+        // and per-file schema guards once this direct scan is accepted.
+        let Some(scan) = aggregate.input().downcast_ref::<DataSourceExec>() 
else {
+            return Ok(None);
+        };
+        let Some((_, source)) = parquet_file_source(scan) else {
+            return Ok(None);
+        };
+        if source.filter().is_some()
+            || source
+                .projection()
+                .is_some_and(|projection| projection.iter().any(|expr| 
!expr.expr.is::<Column>()))
+        {
+            // Pruning before a computed scan projection could change its 
stateful
+            // evaluation or suppress errors, even for an unselected output 
column.
+            return Ok(None);
+        }
+        Ok(Some(Self::new(aggregate, config.clone())?))
+    }
+
+    fn new(aggregate: &AggregateExec, config: ConfigOptions) -> Result<Self> {
+        Ok(Self {
+            template: Self::fresh_aggregate(aggregate, 
Arc::clone(aggregate.input()))?,
+            config,
+            metrics: ExecutionPlanMetricsSet::new(),
+        })
+    }
+
+    fn fresh_aggregate(
+        template: &AggregateExec,
+        input: Arc<dyn ExecutionPlan>,
+    ) -> Result<AggregateExec> {
+        // AggregateExec::clone/replace_children keep shared accumulator 
bounds.
+        // Replacing just its dynamic expression would leave those bounds 
stale.
+        AggregateExec::try_new(
+            *template.mode(),
+            template.group_expr().clone(),
+            template.aggr_expr().to_vec(),
+            template.filter_expr().to_vec(),
+            input,
+            template.input_schema(),
+        )
+    }
+
+    fn build_runtime_aggregate(&self) -> Result<RuntimeAggregate> {
+        let aggregate: Arc<dyn ExecutionPlan> = Arc::new(Self::fresh_aggregate(
+            &self.template,
+            Arc::clone(self.template.input()),
+        )?);
+        let predicate = aggregate
+            .dynamic_expressions_produced()
+            .into_iter()
+            .find_map(|expr| {
+                (expr as Arc<dyn std::any::Any + Send + Sync>)
+                    .downcast::<DynamicFilterPhysicalExpr>()
+                    .ok()
+            });
+        let Some(predicate) = predicate else {
+            return Ok(RuntimeAggregate {
+                aggregate,
+                reader_filter_attached: false,
+            });
+        };
+        let reader =
+            try_attach_parquet_reader_filter(self.template.input(), predicate, 
&self.config)?;

Review Comment:
   [P2] Preserve validation of required fields filled by a NULL default
   
   The attached schema guard accepts any projected column whose adapter 
rewrites it to a literal. A missing required field with a typed NULL default 
therefore leaves runtime pruning enabled. If the live MIN bound eliminates the 
file, the scan never produces the batch that would validate the required field.
   
   I reproduced this with the native wrapper and the same ordinary scan on this 
commit. Use one partition, batch size two, and a projection of nullable `key: 
Int32` followed by required `payload: Int32`. The first file has keys [0, 1] 
and valid payloads. The second has keys [10, 11], omits `payload`, and the scan 
supplies an `Int32(None)` default. Disable decoder and page filtering. Ordinary 
partial MIN reports `Column 'payload' is declared as non-nullable but contains 
null values`. With the wrapper, the first file sets the live MIN to 0 and 
execution returns 0 without reporting the error.
   
   Allow a literal through the guard for a nullable field or when the literal 
is non-null. This is the missing-field/default path, separate from the 
physically nullable column case covered by the current regression.



##########
native/core/src/execution/operators/dynamic_filter/min_max/row_group_limit.rs:
##########
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Limit the work performed by DataFusion's live row-group pruner.
+
+use std::sync::Arc;
+
+use datafusion::datasource::physical_plan::FileSource;
+use datafusion::datasource::source::DataSourceExec;
+use datafusion::physical_plan::ExecutionPlan;
+
+use super::super::parquet_reader::concrete_parquet_source;
+use crate::parquet::parquet_exec::METADATA_SIZE_HINT;
+
+// DataFusion 55.2 rechecks the entire unread tail at every row-group boundary,
+// even when the predicate has not changed. Until that work is bounded 
upstream,
+// cap each eligible file rather than merely throttling aggregate updates.
+pub(super) const MAX_ROW_GROUPS_PER_FILE: usize = 64;
+
+pub(super) async fn within_row_group_limit(
+    input: &Arc<dyn ExecutionPlan>,
+    partition: usize,
+) -> bool {
+    let Some(scan) = input.downcast_ref::<DataSourceExec>() else {
+        return false;
+    };
+    let Some((config, source, _)) = concrete_parquet_source(scan) else {
+        return false;
+    };
+    let Some(factory) = source.parquet_file_reader_factory() else {
+        return false;
+    };
+    let Some(group) = config.file_groups.get(partition) else {
+        return false;
+    };
+    // Use the native reader's footer cache where available. If any file is
+    // oversized, keep ordinary aggregation for this execution. Let normal
+    // decoding report metadata errors in its original order and context.
+    for file in group.files() {
+        let Ok(mut reader) = factory.create_reader(
+            partition,
+            file.clone(),
+            file.metadata_size_hint.or(Some(METADATA_SIZE_HINT)),
+            source.metrics(),
+        ) else {
+            return false;
+        };
+        match reader.get_metadata(None).await {

Review Comment:
   [P2] Reuse preflight footers that do not fit the shared cache
   
   Preflight drops the metadata after counting the row groups. The scan opens 
the file again, so the 512 KiB hint reduces requests for each read but cannot 
remove the second read when the cache cannot hold that footer.
   
   On this commit, I ran ordinary partial MIN and the native wrapper over the 
same one-row, one-row-group Parquet file with a 64 MiB footer key/value and 
DataFusion's default 50 MiB metadata cache. The instrumented reader counted one 
footer load and 67,633,854 requested bytes for ordinary MIN, and two footer 
loads and 135,267,665 bytes with the wrapper. Both returned 1. In a second test 
with 16 one-row files and caching disabled, the wrapper read 32 footers versus 
16. With enough cache space both paths read 16.
   
   Carry the loaded footer into the scan, or skip the preflight when the scan 
cannot reuse its read.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to