This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new d48fdb00d7 Optimize EliminateFilter to avoid unnecessary copies #10288
(#10302)
d48fdb00d7 is described below
commit d48fdb00d7c0e074c0ebe7a359d0f392cd757ac0
Author: Dmitry Bugakov <[email protected]>
AuthorDate: Tue Apr 30 22:39:29 2024 +0200
Optimize EliminateFilter to avoid unnecessary copies #10288 (#10302)
---
datafusion/optimizer/src/eliminate_filter.rs | 68 ++++++++++++----------
.../optimizer/tests/optimizer_integration.rs | 15 +++++
2 files changed, 52 insertions(+), 31 deletions(-)
diff --git a/datafusion/optimizer/src/eliminate_filter.rs
b/datafusion/optimizer/src/eliminate_filter.rs
index 2bf5cfa303..c294bc68f0 100644
--- a/datafusion/optimizer/src/eliminate_filter.rs
+++ b/datafusion/optimizer/src/eliminate_filter.rs
@@ -17,13 +17,12 @@
//! [`EliminateFilter`] replaces `where false` or `where null` with an empty
relation.
-use crate::optimizer::ApplyOrder;
-use datafusion_common::{Result, ScalarValue};
-use datafusion_expr::{
- logical_plan::{EmptyRelation, LogicalPlan},
- Expr, Filter,
-};
+use datafusion_common::tree_node::Transformed;
+use datafusion_common::{internal_err, Result, ScalarValue};
+use datafusion_expr::logical_plan::tree_node::unwrap_arc;
+use datafusion_expr::{EmptyRelation, Expr, Filter, LogicalPlan};
+use crate::optimizer::ApplyOrder;
use crate::{OptimizerConfig, OptimizerRule};
/// Optimization rule that eliminate the scalar value (true/false/null) filter
@@ -44,31 +43,10 @@ impl EliminateFilter {
impl OptimizerRule for EliminateFilter {
fn try_optimize(
&self,
- plan: &LogicalPlan,
+ _plan: &LogicalPlan,
_config: &dyn OptimizerConfig,
) -> Result<Option<LogicalPlan>> {
- match plan {
- LogicalPlan::Filter(Filter {
- predicate: Expr::Literal(ScalarValue::Boolean(v)),
- input,
- ..
- }) => {
- match *v {
- // input also can be filter, apply again
- Some(true) => Ok(Some(
- self.try_optimize(input, _config)?
- .unwrap_or_else(|| input.as_ref().clone()),
- )),
- Some(false) | None => {
- Ok(Some(LogicalPlan::EmptyRelation(EmptyRelation {
- produce_one_row: false,
- schema: input.schema().clone(),
- })))
- }
- }
- }
- _ => Ok(None),
- }
+ internal_err!("Should have called EliminateFilter::rewrite")
}
fn name(&self) -> &str {
@@ -78,17 +56,45 @@ impl OptimizerRule for EliminateFilter {
fn apply_order(&self) -> Option<ApplyOrder> {
Some(ApplyOrder::TopDown)
}
+
+ fn supports_rewrite(&self) -> bool {
+ true
+ }
+
+ fn rewrite(
+ &self,
+ plan: LogicalPlan,
+ _config: &dyn OptimizerConfig,
+ ) -> Result<Transformed<LogicalPlan>> {
+ match plan {
+ LogicalPlan::Filter(Filter {
+ predicate: Expr::Literal(ScalarValue::Boolean(v)),
+ input,
+ ..
+ }) => match v {
+ Some(true) => Ok(Transformed::yes(unwrap_arc(input))),
+ Some(false) | None =>
Ok(Transformed::yes(LogicalPlan::EmptyRelation(
+ EmptyRelation {
+ produce_one_row: false,
+ schema: input.schema().clone(),
+ },
+ ))),
+ },
+ _ => Ok(Transformed::no(plan)),
+ }
+ }
}
#[cfg(test)]
mod tests {
- use crate::eliminate_filter::EliminateFilter;
+ use std::sync::Arc;
+
use datafusion_common::{Result, ScalarValue};
use datafusion_expr::{
col, lit, logical_plan::builder::LogicalPlanBuilder, sum, Expr,
LogicalPlan,
};
- use std::sync::Arc;
+ use crate::eliminate_filter::EliminateFilter;
use crate::test::*;
fn assert_optimized_plan_equal(plan: LogicalPlan, expected: &str) ->
Result<()> {
diff --git a/datafusion/optimizer/tests/optimizer_integration.rs
b/datafusion/optimizer/tests/optimizer_integration.rs
index 180c792066..2430d2d52e 100644
--- a/datafusion/optimizer/tests/optimizer_integration.rs
+++ b/datafusion/optimizer/tests/optimizer_integration.rs
@@ -275,6 +275,21 @@ fn test_same_name_but_not_ambiguous() {
assert_eq!(expected, format!("{plan:?}"));
}
+#[test]
+fn eliminate_nested_filters() {
+ let sql = "\
+ SELECT col_int32 FROM test \
+ WHERE (1=1) AND (col_int32 > 0) \
+ AND (1=1) AND (1=0 OR 1=1)";
+
+ let plan = test_sql(sql).unwrap();
+ let expected = "\
+ Filter: test.col_int32 > Int32(0)\
+ \n TableScan: test projection=[col_int32]";
+
+ assert_eq!(expected, format!("{plan:?}"));
+}
+
fn test_sql(sql: &str) -> Result<LogicalPlan> {
// parse the SQL
let dialect = GenericDialect {}; // or AnsiDialect, or your own dialect ...
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
