This is an automated email from the ASF dual-hosted git repository.
yumwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new e01ca3c76ed [SPARK-42597][SQL][FOLLOW-UP] Only rewrite `EqualNullSafe`
when the left side is non-nullable
e01ca3c76ed is described below
commit e01ca3c76ed7031819237deb6ab34d13141bda8e
Author: Yuming Wang <yumw...@ebay.com>
AuthorDate: Sat Apr 15 09:10:02 2023 +0800
[SPARK-42597][SQL][FOLLOW-UP] Only rewrite `EqualNullSafe` when the left
side is non-nullable
### What changes were proposed in this pull request?
This PR makes it only rewrite `EqualNullSafe` when the left side is
non-nullable when unwrapping date type to timestamp type.
### Why are the changes needed?
1. The current rewrite is incorrect for `EqualNullSafe` if left side is
nullable.
3. Even if we fix the rewrite, it will contain `If` and can't be pushed
down to the data source:
```
EqualNullSafe(Cast(ts, DateType), date) if Cast(ts, DateType).nullable
-> If(IsNull(ts), FalseLiteral, And(GreaterThanOrEqual(ts, Cast(date,
TimestampType)), LessThan(ts, Cast(date + 1, TimestampType))))
```
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Update existing unit test.
Closes #40743 from wangyum/SPARK-42597-dev.
Lead-authored-by: Yuming Wang <yumw...@ebay.com>
Co-authored-by: Yuming Wang <wgy...@gmail.com>
Signed-off-by: Yuming Wang <yumw...@ebay.com>
---
.../sql/catalyst/optimizer/UnwrapCastInBinaryComparison.scala | 5 ++++-
.../sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala | 7 +++----
2 files changed, 7 insertions(+), 5 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparison.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparison.scala
index 7e5242d968d..54b1dd419fb 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparison.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparison.scala
@@ -315,7 +315,10 @@ object UnwrapCastInBinaryComparison extends
Rule[LogicalPlan] {
GreaterThanOrEqual(fromExp, Cast(dateAddOne, fromExp.dataType, tz,
evalMode))
case _: GreaterThanOrEqual =>
GreaterThanOrEqual(fromExp, Cast(date, fromExp.dataType, tz, evalMode))
- case Equality(_, _) =>
+ case _: EqualTo =>
+ And(GreaterThanOrEqual(fromExp, Cast(date, fromExp.dataType, tz,
evalMode)),
+ LessThan(fromExp, Cast(dateAddOne, fromExp.dataType, tz, evalMode)))
+ case EqualNullSafe(left, _) if !left.nullable =>
And(GreaterThanOrEqual(fromExp, Cast(date, fromExp.dataType, tz,
evalMode)),
LessThan(fromExp, Cast(dateAddOne, fromExp.dataType, tz, evalMode)))
case _: LessThan =>
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala
index 8ceb26b906b..0f0acb669c2 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala
@@ -46,7 +46,7 @@ class UnwrapCastInBinaryComparisonSuite extends PlanTest with
ExpressionEvalHelp
val f2: BoundReference = $"b".float.canBeNull.at(1)
val f3: BoundReference = $"c".decimal(5, 2).canBeNull.at(2)
val f4: BoundReference = $"d".boolean.canBeNull.at(3)
- val f5: BoundReference = $"e".timestamp.canBeNull.at(4)
+ val f5: BoundReference = $"e".timestamp.notNull.at(4)
val f6: BoundReference = $"f".timestampNTZ.canBeNull.at(5)
test("unwrap casts when literal == max") {
@@ -392,9 +392,8 @@ class UnwrapCastInBinaryComparisonSuite extends PlanTest
with ExpressionEvalHelp
(f5 >= castTimestamp(dateLit) && f5 < castTimestamp(dateAddOne)) ||
(f6 >= castTimestampNTZ(dateLit) && f6 < castTimestampNTZ(dateAddOne)))
assertEquivalent(
- castDate(f5) <=> dateLit || castDate(f6) === dateLit,
- (f5 >= castTimestamp(dateLit) && f5 < castTimestamp(dateAddOne)) ||
- (f6 >= castTimestampNTZ(dateLit) && f6 < castTimestampNTZ(dateAddOne)))
+ castDate(f5) <=> dateLit || castDate(f6) <=> dateLit,
+ (f5 >= castTimestamp(dateLit) && f5 < castTimestamp(dateAddOne)) ||
castDate(f6) <=> dateLit)
assertEquivalent(
dateLit < castDate(f5) || dateLit < castDate(f6),
castTimestamp(dateAddOne) <= f5 || castTimestampNTZ(dateAddOne) <= f6)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
