This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push: new 007e1c9 [SPARK-35906][SQL][FOLLOWUP] Recursive remove sort if the maximum number of rows less than or equal to 1 007e1c9 is described below commit 007e1c93848fd5c9254524220d269669afc9b666 Author: Yuming Wang <yumw...@ebay.com> AuthorDate: Wed Jul 7 14:27:00 2021 +0900 [SPARK-35906][SQL][FOLLOWUP] Recursive remove sort if the maximum number of rows less than or equal to 1 ### What changes were proposed in this pull request? Make it recursive remove sort if the maximum number of rows less than or equal to 1. For example: ```sql select a from (select a from values(0, 1) t(a, b) order by a) order by a ``` ### Why are the changes needed? Fix Once strategy's idempotence is broken for batch Eliminate Sorts. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Unit test. Closes #33240 from wangyum/SPARK-35906-2. Authored-by: Yuming Wang <yumw...@ebay.com> Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> (cherry picked from commit ddc5cb9051bad39684dcfe671491d14d3ca7b576) Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> --- .../scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala | 2 +- .../org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index fd3b7a1..c79fd7a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -1214,7 +1214,7 @@ object EliminateSorts extends Rule[LogicalPlan] { _.containsPattern(SORT))(applyLocally) private val applyLocally: PartialFunction[LogicalPlan, LogicalPlan] = { - case Sort(_, _, child) if child.maxRows.exists(_ <= 1L) => child + case Sort(_, _, child) if child.maxRows.exists(_ <= 1L) => recursiveRemoveSort(child) case s @ Sort(orders, _, child) if orders.isEmpty || orders.exists(_.child.foldable) => val newOrders = orders.filterNot(_.child.foldable) if (newOrders.isEmpty) { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala index 79070b0..6dc464c 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala @@ -427,5 +427,9 @@ class EliminateSortsSuite extends AnalysisTest { comparePlans( Optimize.execute(testRelation.groupBy()(count(1).as("cnt")).orderBy('cnt.asc)).analyze, testRelation.groupBy()(count(1).as("cnt")).analyze) + + comparePlans( + Optimize.execute(testRelation.limit(Literal(1)).orderBy('a.asc).orderBy('a.asc)).analyze, + testRelation.limit(Literal(1)).analyze) } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org