Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/20670#discussion_r171276798
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/InferFiltersFromConstraintsSuite.scala
---
@@ -192,4 +192,17 @@ class InferFiltersFromConstraintsSuite extends
PlanTest {
comparePlans(Optimize.execute(original.analyze), correct.analyze)
}
+
+ test("SPARK-23405: left-semi equal-join should filter out null join keys
on both sides") {
+ val x = testRelation.subquery('x)
+ val y = testRelation.subquery('y)
+ val condition = Some("x.a".attr === "y.a".attr)
+ val originalQuery = x.join(y, LeftSemi, condition).analyze
+ val left = x.where(IsNotNull('a))
+ val right = y.where(IsNotNull('a))
+ val correctAnswer = left.join(right, LeftSemi, condition)
+ .analyze
--- End diff --
this doesn't need to be in a new line
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
