This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new a6b089fa00c2 [SPARK-45909][SQL] Remove `NumericType` cast if it can
safely up-cast in `IsNotNull`
a6b089fa00c2 is described below
commit a6b089fa00c2736bafd7bd374f401c392da9cf80
Author: Yuming Wang <[email protected]>
AuthorDate: Mon Nov 13 10:50:55 2023 -0800
[SPARK-45909][SQL] Remove `NumericType` cast if it can safely up-cast in
`IsNotNull`
### What changes were proposed in this pull request?
Similar to SPARK-37922. We can remove the cast if it can safely up-cast in
`IsNotNull`.
### Why are the changes needed?
Improve the query performance.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Unit test.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #43785 from wangyum/SPARK-45909.
Authored-by: Yuming Wang <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../spark/sql/catalyst/optimizer/expressions.scala | 2 ++
.../sql/catalyst/optimizer/SimplifyCastsSuite.scala | 21 +++++++++++++++++++++
2 files changed, 23 insertions(+)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
index 2c73e25cb5ed..9e9e6bd905b9 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
@@ -1063,6 +1063,8 @@ object SimplifyCasts extends Rule[LogicalPlan] {
if fromKey == toKey && fromValue == toValue => e
case _ => c
}
+ case IsNotNull(Cast(e, dataType: NumericType, _, _)) if
isWiderCast(e.dataType, dataType) =>
+ IsNotNull(e)
}
// Returns whether the from DataType can be safely casted to the to DataType
without losing
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
index 741b1bb8c082..2eb4830bca98 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
@@ -117,4 +117,25 @@ class SimplifyCastsSuite extends PlanTest {
input.select($"d".cast(LongType).cast(StringType).as("casted")).analyze),
input.select($"d".cast(LongType).cast(StringType).as("casted")).analyze)
}
+
+ test("SPARK-45909: Remove the cast if it can safely up-cast in IsNotNull") {
+ val input = LocalRelation($"a".int, $"b".decimal(18, 0))
+ // Remove cast
+ comparePlans(
+ Optimize.execute(
+ input.select($"a".cast(DecimalType(18, 1)).isNotNull.as("v")).analyze),
+ input.select($"a".isNotNull.as("v")).analyze)
+ comparePlans(
+
Optimize.execute(input.select($"a".cast(LongType).isNotNull.as("v")).analyze),
+ input.select($"a".isNotNull.as("v")).analyze)
+ comparePlans(
+
Optimize.execute(input.select($"b".cast(LongType).isNotNull.as("v")).analyze),
+ input.select($"b".isNotNull.as("v")).analyze)
+
+ // Can not remove cast
+ comparePlans(
+ Optimize.execute(
+ input.select($"a".cast(DecimalType(2, 1)).as("v")).analyze),
+ input.select($"a".cast(DecimalType(2, 1)).as("v")).analyze)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
