Github user gatorsmile commented on a diff in the pull request:
https://github.com/apache/spark/pull/19257#discussion_r156579907
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala
---
@@ -248,13 +252,83 @@ case class EnsureRequirements(conf: SQLConf) extends
Rule[SparkPlan] {
operator.withNewChildren(children)
}
+ /**
+ * When the physical operators are created for JOIN, the ordering of
join keys is based on order
+ * in which the join keys appear in the user query. That might not match
with the output
+ * partitioning of the join node's children (thus leading to extra sort
/ shuffle being
+ * introduced). This rule will change the ordering of the join keys to
match with the
+ * partitioning of the join nodes' children.
+ */
+ def reorderJoinPredicates(plan: SparkPlan): SparkPlan = {
+ def reorderJoinKeys(
+ leftKeys: Seq[Expression],
+ rightKeys: Seq[Expression],
+ leftPartitioning: Partitioning,
+ rightPartitioning: Partitioning): (Seq[Expression],
Seq[Expression]) = {
+
+ def reorder(expectedOrderOfKeys: Seq[Expression],
+ currentOrderOfKeys: Seq[Expression]): (Seq[Expression],
Seq[Expression]) = {
--- End diff --
indents.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
