This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 50e9d78da feat: `ParquetExec` predicate preservation (#5495)
50e9d78da is described below
commit 50e9d78da179fcee28d7fbd30e32de08f1a73d3d
Author: Marco Neumann <[email protected]>
AuthorDate: Tue Mar 7 22:44:57 2023 +0100
feat: `ParquetExec` predicate preservation (#5495)
- Add public getter for `ParquetExec::predicate`. This should allow
phys. optimizer passes to inspect the actual predicate.
- Use the actual predicate (not the pruning one which may contain less
data) for serialization. This should avoid some confusion where
predicates (when they are not used for pruning) are lost.
---
datafusion/core/src/physical_plan/file_format/parquet.rs | 5 +++++
datafusion/proto/src/physical_plan/mod.rs | 8 ++++----
2 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/datafusion/core/src/physical_plan/file_format/parquet.rs
b/datafusion/core/src/physical_plan/file_format/parquet.rs
index cbd43fabe..2c1bfc9ca 100644
--- a/datafusion/core/src/physical_plan/file_format/parquet.rs
+++ b/datafusion/core/src/physical_plan/file_format/parquet.rs
@@ -174,6 +174,11 @@ impl ParquetExec {
&self.base_config
}
+ /// Optional predicate.
+ pub fn predicate(&self) -> Option<&Arc<dyn PhysicalExpr>> {
+ self.predicate.as_ref()
+ }
+
/// Optional reference to this parquet scan's pruning predicate
pub fn pruning_predicate(&self) -> Option<&Arc<PruningPredicate>> {
self.pruning_predicate.as_ref()
diff --git a/datafusion/proto/src/physical_plan/mod.rs
b/datafusion/proto/src/physical_plan/mod.rs
index b9b23a3ce..96c2611b0 100644
--- a/datafusion/proto/src/physical_plan/mod.rs
+++ b/datafusion/proto/src/physical_plan/mod.rs
@@ -956,15 +956,15 @@ impl AsExecutionPlan for PhysicalPlanNode {
)),
})
} else if let Some(exec) = plan.downcast_ref::<ParquetExec>() {
- let pruning_expr = exec
- .pruning_predicate()
- .map(|pred| pred.orig_expr().clone().try_into())
+ let predicate = exec
+ .predicate()
+ .map(|pred| pred.clone().try_into())
.transpose()?;
Ok(protobuf::PhysicalPlanNode {
physical_plan_type: Some(PhysicalPlanType::ParquetScan(
protobuf::ParquetScanExecNode {
base_conf: Some(exec.base_config().try_into()?),
- predicate: pruning_expr,
+ predicate,
},
)),
})
