sunchao commented on code in PR #7598: URL: https://github.com/apache/datafusion-comet/pull/7598#discussion_r4575305000
########## native/core/src/execution/operators/dynamic_filter/min_max.rs: ########## @@ -0,0 +1,315 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Connect one partial MIN/MAX aggregate's improving bound to its Parquet reader. + +use std::fmt::Formatter; +use std::sync::Arc; + +use arrow::datatypes::DataType; +use datafusion::common::config::ConfigOptions; +use datafusion::common::tree_node::TreeNodeRecursion; +use datafusion::common::{internal_err, Result, Statistics}; +use datafusion::datasource::source::DataSourceExec; +use datafusion::execution::TaskContext; +use datafusion::functions_aggregate::min_max::{Max, Min}; +use datafusion::physical_expr::expressions::{Column, DynamicFilterPhysicalExpr}; +use datafusion::physical_expr::{OrderingRequirements, PhysicalExpr}; +use datafusion::physical_plan::aggregates::{AggregateExec, AggregateMode}; +use datafusion::physical_plan::distribution_requirements::InputDistributionRequirements; +use datafusion::physical_plan::execution_plan::CardinalityEffect; +use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricBuilder, MetricsSet}; +use datafusion::physical_plan::statistics::{ChildStats, StatisticsArgs}; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::physical_plan::{ + ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, + PlanProperties, ReplaceChildrenOptions, SendableRecordBatchStream, +}; +use futures::TryStreamExt; + +use super::parquet_reader::{parquet_file_source, try_attach_parquet_reader_filter}; + +mod row_group_limit; + +/// Own an unexecuted aggregate template and metric handles. Each execution owns +/// a fresh aggregate, accumulator bounds, and reader predicate; none of that live +/// state is retained by the template or shared with another Spark task. +#[derive(Debug)] +pub(crate) struct MinMaxReaderFilterExec { + template: AggregateExec, + config: ConfigOptions, + metrics: ExecutionPlanMetricsSet, +} + +struct RuntimeAggregate { + aggregate: Arc<dyn ExecutionPlan>, + reader_filter_attached: bool, +} + +impl MinMaxReaderFilterExec { + // Spark planner integration is a separate step of apache/datafusion-comet#5774. + #[cfg_attr(not(test), allow(dead_code))] + pub(crate) fn try_new( + aggregate: &AggregateExec, + config: &ConfigOptions, + ) -> Result<Option<Self>> { + if !config.optimizer.enable_dynamic_filter_pushdown + || !config.optimizer.enable_aggregate_dynamic_filter_pushdown + || aggregate.mode() != &AggregateMode::Partial + || !aggregate.group_expr().is_true_no_grouping() + || aggregate.limit_options().is_some() + || aggregate.filter_expr().iter().any(Option::is_some) + || aggregate.input().output_partitioning().partition_count() != 1 + || aggregate.input().fetch().is_some() + { + return Ok(None); + } + let [expression] = aggregate.aggr_expr() else { + return Ok(None); + }; + let function = expression.fun().inner(); + if !(function.is::<Min>() || function.is::<Max>()) + || expression.is_distinct() + || !expression.order_bys().is_empty() + { + return Ok(None); + } + let arguments = expression.expressions(); + let [key] = arguments.as_slice() else { + return Ok(None); + }; + if !key.is::<Column>() + || !matches!( + key.data_type(aggregate.input().schema().as_ref())?, + DataType::Int8 | DataType::Int16 | DataType::Int32 | DataType::Int64 + ) + { + return Ok(None); + } + // Keep expressions, residuals, limits, and native/JVM boundaries out of + // this first path. The shared helper still handles reader column mapping + // and per-file schema guards once this direct scan is accepted. + let Some(scan) = aggregate.input().downcast_ref::<DataSourceExec>() else { + return Ok(None); + }; + let Some((_, source)) = parquet_file_source(scan) else { + return Ok(None); + }; + if source.filter().is_some() + || source + .projection() + .is_some_and(|projection| projection.iter().any(|expr| !expr.expr.is::<Column>())) + { + // Pruning before a computed scan projection could change its stateful + // evaluation or suppress errors, even for an unselected output column. + return Ok(None); + } + Ok(Some(Self::new(aggregate, config.clone())?)) + } + + fn new(aggregate: &AggregateExec, config: ConfigOptions) -> Result<Self> { + Ok(Self { + template: Self::fresh_aggregate(aggregate, Arc::clone(aggregate.input()))?, + config, + metrics: ExecutionPlanMetricsSet::new(), + }) + } + + fn fresh_aggregate( + template: &AggregateExec, + input: Arc<dyn ExecutionPlan>, + ) -> Result<AggregateExec> { + // AggregateExec::clone/replace_children keep shared accumulator bounds. + // Replacing just its dynamic expression would leave those bounds stale. + AggregateExec::try_new( + *template.mode(), + template.group_expr().clone(), + template.aggr_expr().to_vec(), + template.filter_expr().to_vec(), + input, + template.input_schema(), + ) + } + + fn build_runtime_aggregate(&self) -> Result<RuntimeAggregate> { + let aggregate: Arc<dyn ExecutionPlan> = Arc::new(Self::fresh_aggregate( + &self.template, + Arc::clone(self.template.input()), + )?); + let predicate = aggregate + .dynamic_expressions_produced() + .into_iter() + .find_map(|expr| { + (expr as Arc<dyn std::any::Any + Send + Sync>) + .downcast::<DynamicFilterPhysicalExpr>() + .ok() + }); + let Some(predicate) = predicate else { + return Ok(RuntimeAggregate { + aggregate, + reader_filter_attached: false, + }); + }; + let reader = + try_attach_parquet_reader_filter(self.template.input(), predicate, &self.config)?; Review Comment: [P2] Preserve validation of required fields filled by a NULL default The attached schema guard accepts any projected column whose adapter rewrites it to a literal. A missing required field with a typed NULL default therefore leaves runtime pruning enabled. If the live MIN bound eliminates the file, the scan never produces the batch that would validate the required field. I reproduced this with the native wrapper and the same ordinary scan on this commit. Use one partition, batch size two, and a projection of nullable `key: Int32` followed by required `payload: Int32`. The first file has keys [0, 1] and valid payloads. The second has keys [10, 11], omits `payload`, and the scan supplies an `Int32(None)` default. Disable decoder and page filtering. Ordinary partial MIN reports `Column 'payload' is declared as non-nullable but contains null values`. With the wrapper, the first file sets the live MIN to 0 and execution returns 0 without reporting the error. Allow a literal through the guard for a nullable field or when the literal is non-null. This is the missing-field/default path, separate from the physically nullable column case covered by the current regression. ########## native/core/src/execution/operators/dynamic_filter/min_max/row_group_limit.rs: ########## @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Limit the work performed by DataFusion's live row-group pruner. + +use std::sync::Arc; + +use datafusion::datasource::physical_plan::FileSource; +use datafusion::datasource::source::DataSourceExec; +use datafusion::physical_plan::ExecutionPlan; + +use super::super::parquet_reader::concrete_parquet_source; +use crate::parquet::parquet_exec::METADATA_SIZE_HINT; + +// DataFusion 55.2 rechecks the entire unread tail at every row-group boundary, +// even when the predicate has not changed. Until that work is bounded upstream, +// cap each eligible file rather than merely throttling aggregate updates. +pub(super) const MAX_ROW_GROUPS_PER_FILE: usize = 64; + +pub(super) async fn within_row_group_limit( + input: &Arc<dyn ExecutionPlan>, + partition: usize, +) -> bool { + let Some(scan) = input.downcast_ref::<DataSourceExec>() else { + return false; + }; + let Some((config, source, _)) = concrete_parquet_source(scan) else { + return false; + }; + let Some(factory) = source.parquet_file_reader_factory() else { + return false; + }; + let Some(group) = config.file_groups.get(partition) else { + return false; + }; + // Use the native reader's footer cache where available. If any file is + // oversized, keep ordinary aggregation for this execution. Let normal + // decoding report metadata errors in its original order and context. + for file in group.files() { + let Ok(mut reader) = factory.create_reader( + partition, + file.clone(), + file.metadata_size_hint.or(Some(METADATA_SIZE_HINT)), + source.metrics(), + ) else { + return false; + }; + match reader.get_metadata(None).await { Review Comment: [P2] Reuse preflight footers that do not fit the shared cache Preflight drops the metadata after counting the row groups. The scan opens the file again, so the 512 KiB hint reduces requests for each read but cannot remove the second read when the cache cannot hold that footer. On this commit, I ran ordinary partial MIN and the native wrapper over the same one-row, one-row-group Parquet file with a 64 MiB footer key/value and DataFusion's default 50 MiB metadata cache. The instrumented reader counted one footer load and 67,633,854 requested bytes for ordinary MIN, and two footer loads and 135,267,665 bytes with the wrapper. Both returned 1. In a second test with 16 one-row files and caching disabled, the wrapper read 32 footers versus 16. With enough cache space both paths read 16. Carry the loaded footer into the scan, or skip the preflight when the scan cannot reuse its read. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
