cloud-fan closed pull request #23330: [SPARK-26352][SQL][FOLLOWUP-2.4] Fix
missing sameOutput in branch-2.4
URL: https://github.com/apache/spark/pull/23330
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/joins.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/joins.scala
index 0b6471289a471..2feb4720f9f92 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/joins.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/joins.scala
@@ -100,7 +100,7 @@ object ReorderJoin extends Rule[LogicalPlan] with
PredicateHelper {
createOrderedJoin(input, conditions)
}
- if (p.sameOutput(reordered)) {
+ if (sameOutput(p, reordered)) {
reordered
} else {
// Reordering the joins have changed the order of the columns.
@@ -108,6 +108,21 @@ object ReorderJoin extends Rule[LogicalPlan] with
PredicateHelper {
Project(p.output, reordered)
}
}
+
+ /**
+ * Returns true iff output of both plans are semantically the same, ie.:
+ * - they contain the same number of `Attribute`s;
+ * - references are the same;
+ * - the order is equal too.
+ * NOTE: this is copied over from SPARK-25691 from master.
+ */
+ def sameOutput(plan1: LogicalPlan, plan2: LogicalPlan): Boolean = {
+ val output1 = plan1.output
+ val output2 = plan2.output
+ output1.length == output2.length && output1.zip(output2).forall {
+ case (a1, a2) => a1.semanticEquals(a2)
+ }
+ }
}
/**
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala
index c94a8b9e318f6..38a70f0691dd4 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala
@@ -291,8 +291,8 @@ class JoinReorderSuite extends PlanTest with
StatsEstimationTestBase {
val optimized = Optimize.execute(analyzed)
val expected = groundTruthBestPlan.analyze
- assert(analyzed.sameOutput(expected)) // if this fails, the expected plan
itself is incorrect
- assert(analyzed.sameOutput(optimized))
+ assert(sameOutput(analyzed, expected)) // if this fails, the expected plan
itself is incorrect
+ assert(sameOutput(analyzed, optimized))
compareJoinOrder(optimized, expected)
}
@@ -300,4 +300,19 @@ class JoinReorderSuite extends PlanTest with
StatsEstimationTestBase {
private def outputsOf(plans: LogicalPlan*): Seq[Attribute] = {
plans.map(_.output).reduce(_ ++ _)
}
+
+ /**
+ * Returns true iff output of both plans are semantically the same, ie.:
+ * - they contain the same number of `Attribute`s;
+ * - references are the same;
+ * - the order is equal too.
+ * NOTE: this is copied over from SPARK-25691 from master.
+ */
+ def sameOutput(plan1: LogicalPlan, plan2: LogicalPlan): Boolean = {
+ val output1 = plan1.output
+ val output2 = plan2.output
+ output1.length == output2.length && output1.zip(output2).forall {
+ case (a1, a2) => a1.semanticEquals(a2)
+ }
+ }
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
