Repository: spark Updated Branches: refs/heads/master 8ec25cd67 -> c3be2cd34
[SPARK-25092] Add RewriteExceptAll and RewriteIntersectAll in the list of nonExcludableRules ## What changes were proposed in this pull request? Add RewriteExceptAll and RewriteIntersectAll in the list of nonExcludableRules as the rewrites are essential for the functioning of EXCEPT ALL and INTERSECT ALL feature. ## How was this patch tested? Added test in OptimizerRuleExclusionSuite. Closes #22080 from dilipbiswal/exceptall_rewrite_exclusion. Authored-by: Dilip Biswal <dbis...@us.ibm.com> Signed-off-by: Xiao Li <gatorsm...@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c3be2cd3 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c3be2cd3 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c3be2cd3 Branch: refs/heads/master Commit: c3be2cd347c42972d9c499b6fd9a6f988f80af12 Parents: 8ec25cd Author: Dilip Biswal <dbis...@us.ibm.com> Authored: Sat Aug 11 22:51:11 2018 -0700 Committer: Xiao Li <gatorsm...@gmail.com> Committed: Sat Aug 11 22:51:11 2018 -0700 ---------------------------------------------------------------------- .../org/apache/spark/sql/catalyst/optimizer/Optimizer.scala | 6 ++++-- .../sql/catalyst/optimizer/OptimizerRuleExclusionSuite.scala | 6 ++++-- .../spark/sql/catalyst/optimizer/SetOperationSuite.scala | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/c3be2cd3/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index 105623c..2ff6768 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -135,7 +135,7 @@ abstract class Optimizer(sessionCatalog: SessionCatalog) Batch("Subquery", Once, OptimizeSubqueries) :: Batch("Replace Operators", fixedPoint, - RewriteExcepAll, + RewriteExceptAll, RewriteIntersectAll, ReplaceIntersectWithSemiJoin, ReplaceExceptWithFilter, @@ -189,6 +189,8 @@ abstract class Optimizer(sessionCatalog: SessionCatalog) ReplaceIntersectWithSemiJoin.ruleName :: ReplaceExceptWithFilter.ruleName :: ReplaceExceptWithAntiJoin.ruleName :: + RewriteExceptAll.ruleName :: + RewriteIntersectAll.ruleName :: ReplaceDistinctWithAggregate.ruleName :: PullupCorrelatedPredicates.ruleName :: RewritePredicateSubquery.ruleName :: Nil @@ -1462,7 +1464,7 @@ object ReplaceExceptWithAntiJoin extends Rule[LogicalPlan] { * }}} */ -object RewriteExcepAll extends Rule[LogicalPlan] { +object RewriteExceptAll extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = plan transform { case Except(left, right, true) => assert(left.output.size == right.output.size) http://git-wip-us.apache.org/repos/asf/spark/blob/c3be2cd3/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerRuleExclusionSuite.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerRuleExclusionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerRuleExclusionSuite.scala index 30c80d2..eee8dc3 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerRuleExclusionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerRuleExclusionSuite.scala @@ -80,12 +80,14 @@ class OptimizerRuleExclusionSuite extends PlanTest { "DummyRuleName")) } - test("Try to exclude a non-excludable rule") { + test("Try to exclude some non-excludable rules") { verifyExcludedRules( new SimpleTestOptimizer(), Seq( ReplaceIntersectWithSemiJoin.ruleName, - PullupCorrelatedPredicates.ruleName)) + PullupCorrelatedPredicates.ruleName, + RewriteExceptAll.ruleName, + RewriteIntersectAll.ruleName)) } test("Custom optimizer") { http://git-wip-us.apache.org/repos/asf/spark/blob/c3be2cd3/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SetOperationSuite.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SetOperationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SetOperationSuite.scala index cb744be..da3923f 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SetOperationSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SetOperationSuite.scala @@ -148,7 +148,7 @@ class SetOperationSuite extends PlanTest { test("EXCEPT ALL rewrite") { val input = Except(testRelation, testRelation2, isAll = true) - val rewrittenPlan = RewriteExcepAll(input) + val rewrittenPlan = RewriteExceptAll(input) val planFragment = testRelation.select(Literal(1L).as("vcol"), 'a, 'b, 'c) .union(testRelation2.select(Literal(-1L).as("vcol"), 'd, 'e, 'f)) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
