This is an automated email from the ASF dual-hosted git repository. agrove pushed a commit to branch comet-parquet-exec in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit d7396e4f878085b57f3f09ace1e93b549f14a8c8 Merge: 08308407 95d69fa3 Author: Andy Grove <[email protected]> AuthorDate: Sat Nov 9 09:36:12 2024 -0700 merge native/core/src/execution/datafusion/planner.rs | 30 ++++++++++++++++--------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --cc native/core/src/execution/datafusion/planner.rs index 2c7b36ef,c121e93d..c711b03d --- a/native/core/src/execution/datafusion/planner.rs +++ b/native/core/src/execution/datafusion/planner.rs @@@ -1030,21 -1032,18 +1033,26 @@@ impl PhysicalPlanner .map(|path| PartitionedFile::from_path(path.path().to_string()).unwrap()) .collect(); + // partition the files + // TODO really should partition the row groups + + let mut file_groups = vec![vec![]; partition_count]; + files.iter().enumerate().for_each(|(idx, file)| { + file_groups[partition_count % idx].push(file.clone()); + }); + let file_scan_config = - FileScanConfig::new(object_store_url, data_schema_arrow.clone()) + FileScanConfig::new(object_store_url, Arc::clone(&data_schema_arrow)) - .with_file_groups(vec![files]) + .with_file_groups(file_groups) .with_projection(Some(projection_vector)); - let builder = - ParquetExecBuilder::new(file_scan_config).with_predicate(test_data_filters); + let mut table_parquet_options = TableParquetOptions::new(); + table_parquet_options.global.pushdown_filters = true; + table_parquet_options.global.reorder_filters = true; + + let builder = ParquetExecBuilder::new(file_scan_config) + .with_predicate(test_data_filters) + .with_table_parquet_options(table_parquet_options); let scan = builder.build(); return Ok((vec![], Arc::new(scan))); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
