This is an automated email from the ASF dual-hosted git repository. agrove pushed a commit to branch comet-parquet-exec in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit 2027755f9ac30b3a42852fd4693b5b5cf8a7da61 Merge: 0b0d6e86 fb68558a Author: Matt Butrovich <[email protected]> AuthorDate: Sat Nov 9 10:39:54 2024 -0500 Merge branch 'native_parquet2' into df-parquet-exec native/core/src/execution/datafusion/planner.rs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --cc native/core/src/execution/datafusion/planner.rs index 7d8bbbb7,d49d8dbb..6db33fad --- a/native/core/src/execution/datafusion/planner.rs +++ b/native/core/src/execution/datafusion/planner.rs @@@ -1024,18 -1021,20 +1025,23 @@@ impl PhysicalPlanner .runtime_env() .register_object_store(&url, Arc::new(object_store)); - let file = PartitionedFile::from_path(path.path().to_string())?; + let files: Vec<PartitionedFile> = paths + .iter() + .map(|path| PartitionedFile::from_path(path.path().to_string()).unwrap()) + .collect(); let file_scan_config = - FileScanConfig::new(object_store_url, Arc::clone(&data_schema_arrow)) - .with_file(file) + FileScanConfig::new(object_store_url, data_schema_arrow.clone()) + .with_file_groups(vec![files]) .with_projection(Some(projection_vector)); - let builder = - ParquetExecBuilder::new(file_scan_config).with_predicate(test_data_filters); + let mut table_parquet_options = TableParquetOptions::new(); + table_parquet_options.global.pushdown_filters = true; + table_parquet_options.global.reorder_filters = true; + + let builder = ParquetExecBuilder::new(file_scan_config) + .with_predicate(test_data_filters) + .with_table_parquet_options(table_parquet_options); let scan = builder.build(); return Ok((vec![], Arc::new(scan))); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
