This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new a6b089fa00c2 [SPARK-45909][SQL] Remove `NumericType` cast if it can safely up-cast in `IsNotNull` a6b089fa00c2 is described below commit a6b089fa00c2736bafd7bd374f401c392da9cf80 Author: Yuming Wang <yumw...@ebay.com> AuthorDate: Mon Nov 13 10:50:55 2023 -0800 [SPARK-45909][SQL] Remove `NumericType` cast if it can safely up-cast in `IsNotNull` ### What changes were proposed in this pull request? Similar to SPARK-37922. We can remove the cast if it can safely up-cast in `IsNotNull`. ### Why are the changes needed? Improve the query performance. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Unit test. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #43785 from wangyum/SPARK-45909. Authored-by: Yuming Wang <yumw...@ebay.com> Signed-off-by: Dongjoon Hyun <dh...@apple.com> --- .../spark/sql/catalyst/optimizer/expressions.scala | 2 ++ .../sql/catalyst/optimizer/SimplifyCastsSuite.scala | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala index 2c73e25cb5ed..9e9e6bd905b9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala @@ -1063,6 +1063,8 @@ object SimplifyCasts extends Rule[LogicalPlan] { if fromKey == toKey && fromValue == toValue => e case _ => c } + case IsNotNull(Cast(e, dataType: NumericType, _, _)) if isWiderCast(e.dataType, dataType) => + IsNotNull(e) } // Returns whether the from DataType can be safely casted to the to DataType without losing diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala index 741b1bb8c082..2eb4830bca98 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala @@ -117,4 +117,25 @@ class SimplifyCastsSuite extends PlanTest { input.select($"d".cast(LongType).cast(StringType).as("casted")).analyze), input.select($"d".cast(LongType).cast(StringType).as("casted")).analyze) } + + test("SPARK-45909: Remove the cast if it can safely up-cast in IsNotNull") { + val input = LocalRelation($"a".int, $"b".decimal(18, 0)) + // Remove cast + comparePlans( + Optimize.execute( + input.select($"a".cast(DecimalType(18, 1)).isNotNull.as("v")).analyze), + input.select($"a".isNotNull.as("v")).analyze) + comparePlans( + Optimize.execute(input.select($"a".cast(LongType).isNotNull.as("v")).analyze), + input.select($"a".isNotNull.as("v")).analyze) + comparePlans( + Optimize.execute(input.select($"b".cast(LongType).isNotNull.as("v")).analyze), + input.select($"b".isNotNull.as("v")).analyze) + + // Can not remove cast + comparePlans( + Optimize.execute( + input.select($"a".cast(DecimalType(2, 1)).as("v")).analyze), + input.select($"a".cast(DecimalType(2, 1)).as("v")).analyze) + } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org