cloud-fan commented on code in PR #44429:
URL: https://github.com/apache/spark/pull/44429#discussion_r1432896673
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala:
##########
@@ -1555,38 +1556,30 @@ object CombineFilters extends Rule[LogicalPlan] with
PredicateHelper {
* Note that changes in the final output ordering may affect the file size
(SPARK-32318).
* This rule handles the following cases:
* 1) if the sort order is empty or the sort order does not have any reference
- * 2) if the Sort operator is a local sort and the child is already sorted
- * 3) if there is another Sort operator separated by 0...n Project, Filter,
Repartition or
+ * 2) if there is another Sort operator separated by 0...n Project, Filter,
Repartition or
* RepartitionByExpression, RebalancePartitions (with deterministic
expressions) operators
- * 4) if the Sort operator is within Join separated by 0...n Project, Filter,
Repartition or
+ * 3) if the Sort operator is within Join separated by 0...n Project, Filter,
Repartition or
* RepartitionByExpression, RebalancePartitions (with deterministic
expressions) operators only
* and the Join condition is deterministic
- * 5) if the Sort operator is within GroupBy separated by 0...n Project,
Filter, Repartition or
+ * 4) if the Sort operator is within GroupBy separated by 0...n Project,
Filter, Repartition or
* RepartitionByExpression, RebalancePartitions (with deterministic
expressions) operators only
* and the aggregate function is order irrelevant
*/
object EliminateSorts extends Rule[LogicalPlan] {
- def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
- _.containsPattern(SORT))(applyLocally)
-
- private val applyLocally: PartialFunction[LogicalPlan, LogicalPlan] = {
+ def apply(plan: LogicalPlan): LogicalPlan =
plan.transformUpWithPruning(_.containsPattern(SORT)) {
case s @ Sort(orders, _, child) if orders.isEmpty ||
orders.exists(_.child.foldable) =>
val newOrders = orders.filterNot(_.child.foldable)
if (newOrders.isEmpty) {
- applyLocally.lift(child).getOrElse(child)
+ child
} else {
s.copy(order = newOrders)
}
- case Sort(orders, false, child) if
SortOrder.orderingSatisfies(child.outputOrdering, orders) =>
Review Comment:
This is the expensive part as it need to calculate the ordering of children.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
