This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 818ce3f01e refactor: Incorporate RewriteDisjunctivePredicate rule into
SimplifyExpressions (#13032)
818ce3f01e is described below
commit 818ce3f01efe1213a9a1eda5dff1542bb9d457f7
Author: Emil Ejbyfeldt <[email protected]>
AuthorDate: Tue Oct 22 17:27:55 2024 +0200
refactor: Incorporate RewriteDisjunctivePredicate rule into
SimplifyExpressions (#13032)
* Elliminate common factors in disjunctions
This adds a rewrite rule that elliminates common factors in OR. This is
already implmented in RewriteDisjunctivePredicate but this
implementation is simpler and will apply in more cases.
* Remove RewriteDisjunctivePredicate rule
* Fix cse test
* Update datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
Co-authored-by: Andrew Lamb <[email protected]>
---------
Co-authored-by: Andrew Lamb <[email protected]>
---
datafusion/expr/src/utils.rs | 48 +++
datafusion/optimizer/src/lib.rs | 1 -
datafusion/optimizer/src/optimizer.rs | 2 -
datafusion/optimizer/src/push_down_filter.rs | 4 +-
.../optimizer/src/rewrite_disjunctive_predicate.rs | 430 ---------------------
.../src/simplify_expressions/expr_simplifier.rs | 73 +++-
datafusion/sqllogictest/test_files/cse.slt | 8 +-
datafusion/sqllogictest/test_files/explain.slt | 2 -
8 files changed, 126 insertions(+), 442 deletions(-)
diff --git a/datafusion/expr/src/utils.rs b/datafusion/expr/src/utils.rs
index 9ee13f1e06..86562daf69 100644
--- a/datafusion/expr/src/utils.rs
+++ b/datafusion/expr/src/utils.rs
@@ -1101,6 +1101,54 @@ fn split_conjunction_impl<'a>(expr: &'a Expr, mut exprs:
Vec<&'a Expr>) -> Vec<&
}
}
+/// Iteratate parts in a conjunctive [`Expr`] such as `A AND B AND C` => `[A,
B, C]`
+///
+/// See [`split_conjunction_owned`] for more details and an example.
+pub fn iter_conjunction(expr: &Expr) -> impl Iterator<Item = &Expr> {
+ let mut stack = vec![expr];
+ std::iter::from_fn(move || {
+ while let Some(expr) = stack.pop() {
+ match expr {
+ Expr::BinaryExpr(BinaryExpr {
+ right,
+ op: Operator::And,
+ left,
+ }) => {
+ stack.push(right);
+ stack.push(left);
+ }
+ Expr::Alias(Alias { expr, .. }) => stack.push(expr),
+ other => return Some(other),
+ }
+ }
+ None
+ })
+}
+
+/// Iteratate parts in a conjunctive [`Expr`] such as `A AND B AND C` => `[A,
B, C]`
+///
+/// See [`split_conjunction_owned`] for more details and an example.
+pub fn iter_conjunction_owned(expr: Expr) -> impl Iterator<Item = Expr> {
+ let mut stack = vec![expr];
+ std::iter::from_fn(move || {
+ while let Some(expr) = stack.pop() {
+ match expr {
+ Expr::BinaryExpr(BinaryExpr {
+ right,
+ op: Operator::And,
+ left,
+ }) => {
+ stack.push(*right);
+ stack.push(*left);
+ }
+ Expr::Alias(Alias { expr, .. }) => stack.push(*expr),
+ other => return Some(other),
+ }
+ }
+ None
+ })
+}
+
/// Splits an owned conjunctive [`Expr`] such as `A AND B AND C` => `[A, B, C]`
///
/// This is often used to "split" filter expressions such as `col1 = 5
diff --git a/datafusion/optimizer/src/lib.rs b/datafusion/optimizer/src/lib.rs
index 3b1df3510d..f310838311 100644
--- a/datafusion/optimizer/src/lib.rs
+++ b/datafusion/optimizer/src/lib.rs
@@ -51,7 +51,6 @@ pub mod propagate_empty_relation;
pub mod push_down_filter;
pub mod push_down_limit;
pub mod replace_distinct_aggregate;
-pub mod rewrite_disjunctive_predicate;
pub mod scalar_subquery_to_join;
pub mod simplify_expressions;
pub mod single_distinct_to_groupby;
diff --git a/datafusion/optimizer/src/optimizer.rs
b/datafusion/optimizer/src/optimizer.rs
index 08dcefa22f..373c877187 100644
--- a/datafusion/optimizer/src/optimizer.rs
+++ b/datafusion/optimizer/src/optimizer.rs
@@ -51,7 +51,6 @@ use crate::propagate_empty_relation::PropagateEmptyRelation;
use crate::push_down_filter::PushDownFilter;
use crate::push_down_limit::PushDownLimit;
use crate::replace_distinct_aggregate::ReplaceDistinctWithAggregate;
-use crate::rewrite_disjunctive_predicate::RewriteDisjunctivePredicate;
use crate::scalar_subquery_to_join::ScalarSubqueryToJoin;
use crate::simplify_expressions::SimplifyExpressions;
use crate::single_distinct_to_groupby::SingleDistinctToGroupBy;
@@ -255,7 +254,6 @@ impl Optimizer {
// run it again after running the optimizations that potentially
converted
// subqueries to joins
Arc::new(SimplifyExpressions::new()),
- Arc::new(RewriteDisjunctivePredicate::new()),
Arc::new(EliminateDuplicatedExpr::new()),
Arc::new(EliminateFilter::new()),
Arc::new(EliminateCrossJoin::new()),
diff --git a/datafusion/optimizer/src/push_down_filter.rs
b/datafusion/optimizer/src/push_down_filter.rs
index 2e3bca5b0b..ac81f3efaa 100644
--- a/datafusion/optimizer/src/push_down_filter.rs
+++ b/datafusion/optimizer/src/push_down_filter.rs
@@ -1213,7 +1213,7 @@ mod tests {
};
use crate::optimizer::Optimizer;
- use crate::rewrite_disjunctive_predicate::RewriteDisjunctivePredicate;
+ use crate::simplify_expressions::SimplifyExpressions;
use crate::test::*;
use crate::OptimizerContext;
use datafusion_expr::test::function_stub::sum;
@@ -1235,7 +1235,7 @@ mod tests {
expected: &str,
) -> Result<()> {
let optimizer = Optimizer::with_rules(vec![
- Arc::new(RewriteDisjunctivePredicate::new()),
+ Arc::new(SimplifyExpressions::new()),
Arc::new(PushDownFilter::new()),
]);
let optimized_plan =
diff --git a/datafusion/optimizer/src/rewrite_disjunctive_predicate.rs
b/datafusion/optimizer/src/rewrite_disjunctive_predicate.rs
deleted file mode 100644
index a6b633fdb8..0000000000
--- a/datafusion/optimizer/src/rewrite_disjunctive_predicate.rs
+++ /dev/null
@@ -1,430 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! [`RewriteDisjunctivePredicate`] rewrites predicates to reduce redundancy
-
-use crate::optimizer::ApplyOrder;
-use crate::{OptimizerConfig, OptimizerRule};
-use datafusion_common::tree_node::Transformed;
-use datafusion_common::Result;
-use datafusion_expr::expr::BinaryExpr;
-use datafusion_expr::logical_plan::Filter;
-use datafusion_expr::{Expr, LogicalPlan, Operator};
-
-/// Optimizer pass that rewrites predicates of the form
-///
-/// ```text
-/// (A = B AND <expr1>) OR (A = B AND <expr2>) OR ... (A = B AND <exprN>)
-/// ```
-///
-/// Into
-/// ```text
-/// (A = B) AND (<expr1> OR <expr2> OR ... <exprN> )
-/// ```
-///
-/// Predicates connected by `OR` typically not able to be broken down
-/// and distributed as well as those connected by `AND`.
-///
-/// The idea is to rewrite predicates into `good_predicate1 AND
-/// good_predicate2 AND ...` where `good_predicate` means the
-/// predicate has special support in the execution engine.
-///
-/// Equality join predicates (e.g. `col1 = col2`), or single column
-/// expressions (e.g. `col = 5`) are examples of predicates with
-/// special support.
-///
-/// # TPCH Q19
-///
-/// This optimization is admittedly somewhat of a niche usecase. It's
-/// main use is that it appears in TPCH Q19 and is required to avoid a
-/// CROSS JOIN.
-///
-/// Specifically, Q19 has a WHERE clause that looks like
-///
-/// ```sql
-/// where
-/// (
-/// p_partkey = l_partkey
-/// and p_brand = ‘[BRAND1]’
-/// and p_container in ( ‘SM CASE’, ‘SM BOX’, ‘SM PACK’, ‘SM PKG’)
-/// and l_quantity >= [QUANTITY1] and l_quantity <= [QUANTITY1] + 10
-/// and p_size between 1 and 5
-/// and l_shipmode in (‘AIR’, ‘AIR REG’)
-/// and l_shipinstruct = ‘DELIVER IN PERSON’
-/// )
-/// or
-/// (
-/// p_partkey = l_partkey
-/// and p_brand = ‘[BRAND2]’
-/// and p_container in (‘MED BAG’, ‘MED BOX’, ‘MED PKG’, ‘MED PACK’)
-/// and l_quantity >= [QUANTITY2] and l_quantity <= [QUANTITY2] + 10
-/// and p_size between 1 and 10
-/// and l_shipmode in (‘AIR’, ‘AIR REG’)
-/// and l_shipinstruct = ‘DELIVER IN PERSON’
-/// )
-/// or
-/// (
-/// p_partkey = l_partkey
-/// and p_brand = ‘[BRAND3]’
-/// and p_container in ( ‘LG CASE’, ‘LG BOX’, ‘LG PACK’, ‘LG PKG’)
-/// and l_quantity >= [QUANTITY3] and l_quantity <= [QUANTITY3] + 10
-/// and p_size between 1 and 15
-/// and l_shipmode in (‘AIR’, ‘AIR REG’)
-/// and l_shipinstruct = ‘DELIVER IN PERSON’
-/// )
-/// )
-/// ```
-///
-/// Naively planning this query will result in a CROSS join with that
-/// single large OR filter. However, rewriting it using the rewrite in
-/// this pass results in a proper join predicate, `p_partkey = l_partkey`:
-///
-/// ```sql
-/// where
-/// p_partkey = l_partkey
-/// and l_shipmode in (‘AIR’, ‘AIR REG’)
-/// and l_shipinstruct = ‘DELIVER IN PERSON’
-/// and (
-/// (
-/// and p_brand = ‘[BRAND1]’
-/// and p_container in ( ‘SM CASE’, ‘SM BOX’, ‘SM PACK’, ‘SM PKG’)
-/// and l_quantity >= [QUANTITY1] and l_quantity <= [QUANTITY1] + 10
-/// and p_size between 1 and 5
-/// )
-/// or
-/// (
-/// and p_brand = ‘[BRAND2]’
-/// and p_container in (‘MED BAG’, ‘MED BOX’, ‘MED PKG’, ‘MED PACK’)
-/// and l_quantity >= [QUANTITY2] and l_quantity <= [QUANTITY2] + 10
-/// and p_size between 1 and 10
-/// )
-/// or
-/// (
-/// and p_brand = ‘[BRAND3]’
-/// and p_container in ( ‘LG CASE’, ‘LG BOX’, ‘LG PACK’, ‘LG PKG’)
-/// and l_quantity >= [QUANTITY3] and l_quantity <= [QUANTITY3] + 10
-/// and p_size between 1 and 15
-/// )
-/// )
-/// ```
-///
-#[derive(Default, Debug)]
-pub struct RewriteDisjunctivePredicate;
-
-impl RewriteDisjunctivePredicate {
- pub fn new() -> Self {
- Self
- }
-}
-
-impl OptimizerRule for RewriteDisjunctivePredicate {
- fn name(&self) -> &str {
- "rewrite_disjunctive_predicate"
- }
-
- fn apply_order(&self) -> Option<ApplyOrder> {
- Some(ApplyOrder::TopDown)
- }
-
- fn supports_rewrite(&self) -> bool {
- true
- }
-
- fn rewrite(
- &self,
- plan: LogicalPlan,
- _config: &dyn OptimizerConfig,
- ) -> Result<Transformed<LogicalPlan>> {
- match plan {
- LogicalPlan::Filter(filter) => {
- let predicate = predicate(filter.predicate)?;
- let rewritten_predicate = rewrite_predicate(predicate);
- let rewritten_expr = normalize_predicate(rewritten_predicate);
- Ok(Transformed::yes(LogicalPlan::Filter(Filter::try_new(
- rewritten_expr,
- filter.input,
- )?)))
- }
- _ => Ok(Transformed::no(plan)),
- }
- }
-}
-
-#[derive(Clone, PartialEq, Debug)]
-enum Predicate {
- And { args: Vec<Predicate> },
- Or { args: Vec<Predicate> },
- Other { expr: Box<Expr> },
-}
-
-fn predicate(expr: Expr) -> Result<Predicate> {
- match expr {
- Expr::BinaryExpr(BinaryExpr { left, op, right }) => match op {
- Operator::And => {
- let args = vec![predicate(*left)?, predicate(*right)?];
- Ok(Predicate::And { args })
- }
- Operator::Or => {
- let args = vec![predicate(*left)?, predicate(*right)?];
- Ok(Predicate::Or { args })
- }
- _ => Ok(Predicate::Other {
- expr: Box::new(Expr::BinaryExpr(BinaryExpr::new(left, op,
right))),
- }),
- },
- _ => Ok(Predicate::Other {
- expr: Box::new(expr),
- }),
- }
-}
-
-fn normalize_predicate(predicate: Predicate) -> Expr {
- match predicate {
- Predicate::And { args } => {
- assert!(args.len() >= 2);
- args.into_iter()
- .map(normalize_predicate)
- .reduce(Expr::and)
- .expect("had more than one arg")
- }
- Predicate::Or { args } => {
- assert!(args.len() >= 2);
- args.into_iter()
- .map(normalize_predicate)
- .reduce(Expr::or)
- .expect("had more than one arg")
- }
- Predicate::Other { expr } => *expr,
- }
-}
-
-fn rewrite_predicate(predicate: Predicate) -> Predicate {
- match predicate {
- Predicate::And { args } => {
- let mut rewritten_args = Vec::with_capacity(args.len());
- for arg in args.into_iter() {
- rewritten_args.push(rewrite_predicate(arg));
- }
- rewritten_args = flatten_and_predicates(rewritten_args);
- Predicate::And {
- args: rewritten_args,
- }
- }
- Predicate::Or { args } => {
- let mut rewritten_args = vec![];
- for arg in args.into_iter() {
- rewritten_args.push(rewrite_predicate(arg));
- }
- rewritten_args = flatten_or_predicates(rewritten_args);
- delete_duplicate_predicates(rewritten_args)
- }
- Predicate::Other { expr } => Predicate::Other { expr },
- }
-}
-
-fn flatten_and_predicates(
- and_predicates: impl IntoIterator<Item = Predicate>,
-) -> Vec<Predicate> {
- let mut flattened_predicates = vec![];
- for predicate in and_predicates {
- match predicate {
- Predicate::And { args } => {
- flattened_predicates.append(&mut flatten_and_predicates(args));
- }
- _ => {
- flattened_predicates.push(predicate);
- }
- }
- }
- flattened_predicates
-}
-
-fn flatten_or_predicates(
- or_predicates: impl IntoIterator<Item = Predicate>,
-) -> Vec<Predicate> {
- let mut flattened_predicates = vec![];
- for predicate in or_predicates {
- match predicate {
- Predicate::Or { args } => {
- flattened_predicates.append(&mut flatten_or_predicates(args));
- }
- _ => {
- flattened_predicates.push(predicate);
- }
- }
- }
- flattened_predicates
-}
-
-fn delete_duplicate_predicates(or_predicates: Vec<Predicate>) -> Predicate {
- let mut shortest_exprs: Vec<Predicate> = vec![];
- let mut shortest_exprs_len = 0;
- // choose the shortest AND predicate
- for or_predicate in or_predicates.iter() {
- match or_predicate {
- Predicate::And { args } => {
- let args_num = args.len();
- if shortest_exprs.is_empty() || args_num < shortest_exprs_len {
- shortest_exprs.clone_from(args);
- shortest_exprs_len = args_num;
- }
- }
- _ => {
- // if there is no AND predicate, it must be the shortest
expression.
- shortest_exprs = vec![or_predicate.clone()];
- break;
- }
- }
- }
-
- // dedup shortest_exprs
- shortest_exprs.dedup();
-
- // Check each element in shortest_exprs to see if it's in all the OR
arguments.
- let mut exist_exprs: Vec<Predicate> = vec![];
- for expr in shortest_exprs.iter() {
- let found = or_predicates.iter().all(|or_predicate| match or_predicate
{
- Predicate::And { args } => args.contains(expr),
- _ => or_predicate == expr,
- });
- if found {
- exist_exprs.push((*expr).clone());
- }
- }
- if exist_exprs.is_empty() {
- return Predicate::Or {
- args: or_predicates,
- };
- }
-
- // Rebuild the OR predicate.
- // (A AND B) OR A will be optimized to A.
- let mut new_or_predicates = vec![];
- for or_predicate in or_predicates.into_iter() {
- match or_predicate {
- Predicate::And { mut args } => {
- args.retain(|expr| !exist_exprs.contains(expr));
- if !args.is_empty() {
- if args.len() == 1 {
- new_or_predicates.push(args.remove(0));
- } else {
- new_or_predicates.push(Predicate::And { args });
- }
- } else {
- new_or_predicates.clear();
- break;
- }
- }
- _ => {
- if exist_exprs.contains(&or_predicate) {
- new_or_predicates.clear();
- break;
- }
- }
- }
- }
- if !new_or_predicates.is_empty() {
- if new_or_predicates.len() == 1 {
- exist_exprs.push(new_or_predicates.remove(0));
- } else {
- exist_exprs.push(Predicate::Or {
- args: flatten_or_predicates(new_or_predicates),
- });
- }
- }
-
- if exist_exprs.len() == 1 {
- exist_exprs.remove(0)
- } else {
- Predicate::And {
- args: flatten_and_predicates(exist_exprs),
- }
- }
-}
-
-#[cfg(test)]
-mod tests {
- use crate::rewrite_disjunctive_predicate::{
- normalize_predicate, predicate, rewrite_predicate, Predicate,
- };
-
- use datafusion_common::{Result, ScalarValue};
- use datafusion_expr::{and, col, lit, or};
-
- #[test]
- fn test_rewrite_predicate() -> Result<()> {
- let equi_expr = col("t1.a").eq(col("t2.b"));
- let gt_expr = col("t1.c").gt(lit(ScalarValue::Int8(Some(1))));
- let lt_expr = col("t1.d").lt(lit(ScalarValue::Int8(Some(2))));
- let expr = or(
- and(equi_expr.clone(), gt_expr.clone()),
- and(equi_expr.clone(), lt_expr.clone()),
- );
- let predicate = predicate(expr)?;
- assert_eq!(
- predicate,
- Predicate::Or {
- args: vec![
- Predicate::And {
- args: vec![
- Predicate::Other {
- expr: Box::new(equi_expr.clone())
- },
- Predicate::Other {
- expr: Box::new(gt_expr.clone())
- },
- ]
- },
- Predicate::And {
- args: vec![
- Predicate::Other {
- expr: Box::new(equi_expr.clone())
- },
- Predicate::Other {
- expr: Box::new(lt_expr.clone())
- },
- ]
- },
- ]
- }
- );
- let rewritten_predicate = rewrite_predicate(predicate);
- assert_eq!(
- rewritten_predicate,
- Predicate::And {
- args: vec![
- Predicate::Other {
- expr: Box::new(equi_expr.clone())
- },
- Predicate::Or {
- args: vec![
- Predicate::Other {
- expr: Box::new(gt_expr.clone())
- },
- Predicate::Other {
- expr: Box::new(lt_expr.clone())
- },
- ]
- },
- ]
- }
- );
- let rewritten_expr = normalize_predicate(rewritten_predicate);
- assert_eq!(rewritten_expr, and(equi_expr, or(gt_expr, lt_expr)));
- Ok(())
- }
-}
diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
index 2bac71a6ae..f9dfadc708 100644
--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
@@ -32,14 +32,18 @@ use datafusion_common::{
tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRewriter},
};
use datafusion_common::{internal_err, DFSchema, DataFusionError, Result,
ScalarValue};
-use datafusion_expr::expr::{InList, InSubquery, WindowFunction};
use datafusion_expr::simplify::ExprSimplifyResult;
use datafusion_expr::{
and, lit, or, BinaryExpr, Case, ColumnarValue, Expr, Like, Operator,
Volatility,
WindowFunctionDefinition,
};
use datafusion_expr::{expr::ScalarFunction,
interval_arithmetic::NullableInterval};
+use datafusion_expr::{
+ expr::{InList, InSubquery, WindowFunction},
+ utils::{iter_conjunction, iter_conjunction_owned},
+};
use datafusion_physical_expr::{create_physical_expr,
execution_props::ExecutionProps};
+use indexmap::IndexSet;
use crate::analyzer::type_coercion::TypeCoercionRewriter;
use crate::simplify_expressions::guarantees::GuaranteeRewriter;
@@ -850,6 +854,27 @@ impl<'a, S: SimplifyInfo> TreeNodeRewriter for
Simplifier<'a, S> {
op: Or,
right,
}) if is_op_with(And, &left, &right) => Transformed::yes(*right),
+ // Eliminate common factors in conjunctions e.g
+ // (A AND B) OR (A AND C) -> A AND (B OR C)
+ Expr::BinaryExpr(BinaryExpr {
+ left,
+ op: Or,
+ right,
+ }) if has_common_conjunction(&left, &right) => {
+ let lhs: IndexSet<Expr> =
iter_conjunction_owned(*left).collect();
+ let (common, rhs): (Vec<_>, Vec<_>) =
+ iter_conjunction_owned(*right).partition(|e|
lhs.contains(e));
+
+ let new_rhs = rhs.into_iter().reduce(and);
+ let new_lhs = lhs.into_iter().filter(|e|
!common.contains(e)).reduce(and);
+ let common_conjunction =
common.into_iter().reduce(and).unwrap();
+
+ let new_expr = match (new_lhs, new_rhs) {
+ (Some(lhs), Some(rhs)) => and(common_conjunction, or(lhs,
rhs)),
+ (_, _) => common_conjunction,
+ };
+ Transformed::yes(new_expr)
+ }
//
// Rules for AND
@@ -1656,6 +1681,11 @@ impl<'a, S: SimplifyInfo> TreeNodeRewriter for
Simplifier<'a, S> {
}
}
+fn has_common_conjunction(lhs: &Expr, rhs: &Expr) -> bool {
+ let lhs: HashSet<&Expr> = iter_conjunction(lhs).collect();
+ iter_conjunction(rhs).any(|e| lhs.contains(&e))
+}
+
// TODO: We might not need this after defer pattern for Box is stabilized.
https://github.com/rust-lang/rust/issues/87121
fn are_inlist_and_eq_and_match_neg(
left: &Expr,
@@ -3743,6 +3773,47 @@ mod tests {
assert_eq!(expr, expected);
assert_eq!(num_iter, 2);
}
+
+ fn boolean_test_schema() -> DFSchemaRef {
+ Schema::new(vec![
+ Field::new("A", DataType::Boolean, false),
+ Field::new("B", DataType::Boolean, false),
+ Field::new("C", DataType::Boolean, false),
+ Field::new("D", DataType::Boolean, false),
+ ])
+ .to_dfschema_ref()
+ .unwrap()
+ }
+
+ #[test]
+ fn simplify_common_factor_conjuction_in_disjunction() {
+ let props = ExecutionProps::new();
+ let schema = boolean_test_schema();
+ let simplifier =
+
ExprSimplifier::new(SimplifyContext::new(&props).with_schema(schema));
+
+ let a = || col("A");
+ let b = || col("B");
+ let c = || col("C");
+ let d = || col("D");
+
+ // (A AND B) OR (A AND C) -> A AND (B OR C)
+ let expr = a().and(b()).or(a().and(c()));
+ let expected = a().and(b().or(c()));
+
+ assert_eq!(expected, simplifier.simplify(expr).unwrap());
+
+ // (A AND B) OR (A AND C) OR (A AND D) -> A AND (B OR C OR D)
+ let expr = a().and(b()).or(a().and(c())).or(a().and(d()));
+ let expected = a().and(b().or(c()).or(d()));
+ assert_eq!(expected, simplifier.simplify(expr).unwrap());
+
+ // A OR (B AND C AND A) -> A
+ let expr = a().or(b().and(c().and(a())));
+ let expected = a();
+ assert_eq!(expected, simplifier.simplify(expr).unwrap());
+ }
+
#[test]
fn test_simplify_udaf() {
let udaf =
AggregateUDF::new_from_impl(SimplifyMockUdaf::new_with_simplify());
diff --git a/datafusion/sqllogictest/test_files/cse.slt
b/datafusion/sqllogictest/test_files/cse.slt
index 9f0f654179..c95e9a1309 100644
--- a/datafusion/sqllogictest/test_files/cse.slt
+++ b/datafusion/sqllogictest/test_files/cse.slt
@@ -199,18 +199,18 @@ physical_plan
# Surely only once but also conditionally evaluated subexpressions
query TT
EXPLAIN SELECT
- (a = 1 OR random() = 0) AND (a = 1 OR random() = 1) AS c1,
- (a = 2 AND random() = 0) OR (a = 2 AND random() = 1) AS c2,
+ (a = 1 OR random() = 0) AND (a = 2 OR random() = 1) AS c1,
+ (a = 2 AND random() = 0) OR (a = 1 AND random() = 1) AS c2,
CASE WHEN a + 3 = 0 THEN a + 3 + random() ELSE 0 END AS c3,
CASE WHEN a + 4 = 0 THEN 0 ELSE a + 4 + random() END AS c4
FROM t1
----
logical_plan
-01)Projection: (__common_expr_1 OR random() = Float64(0)) AND (__common_expr_1
OR random() = Float64(1)) AS c1, __common_expr_2 AND random() = Float64(0) OR
__common_expr_2 AND random() = Float64(1) AS c2, CASE WHEN __common_expr_3 =
Float64(0) THEN __common_expr_3 + random() ELSE Float64(0) END AS c3, CASE WHEN
__common_expr_4 = Float64(0) THEN Float64(0) ELSE __common_expr_4 + random()
END AS c4
+01)Projection: (__common_expr_1 OR random() = Float64(0)) AND (__common_expr_2
OR random() = Float64(1)) AS c1, __common_expr_2 AND random() = Float64(0) OR
__common_expr_1 AND random() = Float64(1) AS c2, CASE WHEN __common_expr_3 =
Float64(0) THEN __common_expr_3 + random() ELSE Float64(0) END AS c3, CASE WHEN
__common_expr_4 = Float64(0) THEN Float64(0) ELSE __common_expr_4 + random()
END AS c4
02)--Projection: t1.a = Float64(1) AS __common_expr_1, t1.a = Float64(2) AS
__common_expr_2, t1.a + Float64(3) AS __common_expr_3, t1.a + Float64(4) AS
__common_expr_4
03)----TableScan: t1 projection=[a]
physical_plan
-01)ProjectionExec: expr=[(__common_expr_1@0 OR random() = 0) AND
(__common_expr_1@0 OR random() = 1) as c1, __common_expr_2@1 AND random() = 0
OR __common_expr_2@1 AND random() = 1 as c2, CASE WHEN __common_expr_3@2 = 0
THEN __common_expr_3@2 + random() ELSE 0 END as c3, CASE WHEN __common_expr_4@3
= 0 THEN 0 ELSE __common_expr_4@3 + random() END as c4]
+01)ProjectionExec: expr=[(__common_expr_1@0 OR random() = 0) AND
(__common_expr_2@1 OR random() = 1) as c1, __common_expr_2@1 AND random() = 0
OR __common_expr_1@0 AND random() = 1 as c2, CASE WHEN __common_expr_3@2 = 0
THEN __common_expr_3@2 + random() ELSE 0 END as c3, CASE WHEN __common_expr_4@3
= 0 THEN 0 ELSE __common_expr_4@3 + random() END as c4]
02)--ProjectionExec: expr=[a@0 = 1 as __common_expr_1, a@0 = 2 as
__common_expr_2, a@0 + 3 as __common_expr_3, a@0 + 4 as __common_expr_4]
03)----MemoryExec: partitions=1, partition_sizes=[0]
diff --git a/datafusion/sqllogictest/test_files/explain.slt
b/datafusion/sqllogictest/test_files/explain.slt
index b1962ffcc1..54340604ad 100644
--- a/datafusion/sqllogictest/test_files/explain.slt
+++ b/datafusion/sqllogictest/test_files/explain.slt
@@ -189,7 +189,6 @@ logical_plan after decorrelate_predicate_subquery SAME TEXT
AS ABOVE
logical_plan after scalar_subquery_to_join SAME TEXT AS ABOVE
logical_plan after extract_equijoin_predicate SAME TEXT AS ABOVE
logical_plan after simplify_expressions SAME TEXT AS ABOVE
-logical_plan after rewrite_disjunctive_predicate SAME TEXT AS ABOVE
logical_plan after eliminate_duplicated_expr SAME TEXT AS ABOVE
logical_plan after eliminate_filter SAME TEXT AS ABOVE
logical_plan after eliminate_cross_join SAME TEXT AS ABOVE
@@ -216,7 +215,6 @@ logical_plan after decorrelate_predicate_subquery SAME TEXT
AS ABOVE
logical_plan after scalar_subquery_to_join SAME TEXT AS ABOVE
logical_plan after extract_equijoin_predicate SAME TEXT AS ABOVE
logical_plan after simplify_expressions SAME TEXT AS ABOVE
-logical_plan after rewrite_disjunctive_predicate SAME TEXT AS ABOVE
logical_plan after eliminate_duplicated_expr SAME TEXT AS ABOVE
logical_plan after eliminate_filter SAME TEXT AS ABOVE
logical_plan after eliminate_cross_join SAME TEXT AS ABOVE
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
