This is an automated email from the ASF dual-hosted git repository. houqp pushed a commit to branch qp_partition_scan in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
commit 524670a4ca6772c71e9a71233ac0d735ac622457 Author: QP Hou <[email protected]> AuthorDate: Wed Oct 25 23:29:40 2023 -0700 reduce partition scan object list using partition columns --- datafusion/core/src/datasource/listing/helpers.rs | 44 ++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/datafusion/core/src/datasource/listing/helpers.rs b/datafusion/core/src/datasource/listing/helpers.rs index 476c58b698..62e2c93496 100644 --- a/datafusion/core/src/datasource/listing/helpers.rs +++ b/datafusion/core/src/datasource/listing/helpers.rs @@ -330,7 +330,49 @@ pub async fn pruned_partition_list<'a>( )); } - let partitions = list_partitions(store, table_path, partition_cols.len()).await?; + use datafusion_expr::expr::*; + use datafusion_expr::*; + + let mut partitioned_table_path: Option<ListingTableUrl> = None; + let mut list_partiton_depth = partition_cols.len(); + + for filter in filters { + match filter { + Expr::BinaryExpr(BinaryExpr { left, op, right }) => match op { + Operator::Eq => match (left.as_ref(), right.as_ref()) { + ( + &Expr::Column(Column { + ref name, + relation: _, + }), + &Expr::Literal(ref lit), + ) => { + if name == &partition_cols[0].0 { + partitioned_table_path = Some( + ListingTableUrl::parse(format!( + "{}/{}={}", + table_path.as_str(), + name, + lit, + )) + .unwrap(), + ); + list_partiton_depth -= 1; + } + } + _ => {} + }, + _ => {} + }, + _ => {} + } + } + let partitions = list_partitions( + store, + partitioned_table_path.as_ref().unwrap_or(table_path), + list_partiton_depth, + ) + .await?; debug!("Listed {} partitions", partitions.len()); let pruned =
