This is an automated email from the ASF dual-hosted git repository. agrove pushed a commit to branch comet-parquet-exec in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit 8e47562f340dc91ab7aae24523e4bb72e3028555 Author: Matt Butrovich <[email protected]> AuthorDate: Sat Nov 9 08:39:41 2024 -0500 Enable filter pushdown with TableParquetOptions. --- native/core/src/execution/datafusion/planner.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/native/core/src/execution/datafusion/planner.rs b/native/core/src/execution/datafusion/planner.rs index aa858e44..f6bf3314 100644 --- a/native/core/src/execution/datafusion/planner.rs +++ b/native/core/src/execution/datafusion/planner.rs @@ -84,7 +84,6 @@ use datafusion::{ }; use datafusion_physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctionExpr}; -use datafusion::datasource::file_format::FileFormat; use datafusion::datasource::listing::PartitionedFile; use datafusion::datasource::physical_plan::parquet::ParquetExecBuilder; use datafusion::datasource::physical_plan::FileScanConfig; @@ -104,6 +103,7 @@ use datafusion_comet_spark_expr::{ Cast, CreateNamedStruct, DateTruncExpr, GetArrayStructFields, GetStructField, HourExpr, IfExpr, ListExtract, MinuteExpr, RLike, SecondExpr, TimestampTruncExpr, ToJson, }; +use datafusion_common::config::TableParquetOptions; use datafusion_common::scalar::ScalarStructBuilder; use datafusion_common::{ tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion, TreeNodeRewriter}, @@ -1028,8 +1028,13 @@ impl PhysicalPlanner { .with_file(file) .with_projection(Some(projection_vector)); - let builder = - ParquetExecBuilder::new(file_scan_config).with_predicate(test_data_filters); + let mut table_parquet_options = TableParquetOptions::new(); + table_parquet_options.global.pushdown_filters = true; + table_parquet_options.global.reorder_filters = true; + + let builder = ParquetExecBuilder::new(file_scan_config) + .with_predicate(test_data_filters) + .with_table_parquet_options(table_parquet_options); let scan = builder.build(); return Ok((vec![], Arc::new(scan))); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
