ulysses-you commented on a change in pull request #27008: [SPARK-30353][SQL]
Add IsNotNull check in SimplifyBinaryComparison optimization
URL: https://github.com/apache/spark/pull/27008#discussion_r365255572
##########
File path:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala
##########
@@ -122,4 +128,51 @@ class BinaryComparisonSimplificationSuite extends
PlanTest with PredicateHelper
comparePlans(optimized, correctAnswer)
}
+
+ test("Simplify null and nonnull with filter constraints") {
+ val a = Symbol("a")
+ Seq(a === a, a <= a, a >= a, a < a, a > a).foreach { condition =>
+ val plan = nonNullableRelation.where(condition).analyze
+ val actual = Optimize.execute(plan)
+ val correctAnswer = nonNullableRelation.analyze
+ comparePlans(actual, correctAnswer)
+ }
+
+ // infer filter constraints will add IsNotNull
+ Seq(a === a, a <= a, a >= a).foreach { condition =>
+ val plan = nullableRelation.where(condition).analyze
+ val actual = Optimize.execute(plan)
+ val correctAnswer = nullableRelation.where('a.isNotNull).analyze
+ comparePlans(actual, correctAnswer)
+ }
+
+ Seq(a < a, a > a).foreach { condition =>
+ val plan = nullableRelation.where(condition).analyze
+ val actual = Optimize.execute(plan)
+ val correctAnswer = nullableRelation.analyze
+ comparePlans(actual, correctAnswer)
+ }
+ }
+
+ test("Simplify nullable without constraints propagation") {
+ withSQLConf(SQLConf.CONSTRAINT_PROPAGATION_ENABLED.key -> "false") {
+ val a = Symbol("a")
+ Seq(And(a === a, a.isNotNull),
+ And(a <= a, a.isNotNull),
+ And(a >= a, a.isNotNull)).foreach { condition =>
+ val plan = nullableRelation.where(condition).analyze
+ val actual = Optimize.execute(plan)
+ val correctAnswer = nullableRelation.where('a.isNotNull).analyze
+ comparePlans(actual, correctAnswer)
+ }
+
+ Seq(And(a < a, a.isNotNull), And(a > a, a.isNotNull))
+ .foreach { condition =>
+ val plan = nullableRelation.where(condition).analyze
+ val actual = Optimize.execute(plan)
+ val correctAnswer = nullableRelation.analyze
Review comment:
`False` has been remove by `PruneFilters`, so result is just empty
LocalRelation.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
