This is an automated email from the ASF dual-hosted git repository.
zhli pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 0ef2f8216 [VL] Support KnownNullable for Spark 3.5 (#6193)
0ef2f8216 is described below
commit 0ef2f8216b03f5f279d80c71baac30dbdb94199f
Author: Zhen Li <[email protected]>
AuthorDate: Mon Jun 24 16:35:14 2024 +0800
[VL] Support KnownNullable for Spark 3.5 (#6193)
[VL] Support KnownNullable for Spark 3.5.
---
.../gluten/execution/ScalarFunctionsValidateSuite.scala | 14 +++++++++++++-
.../org/apache/gluten/sql/shims/spark35/Spark35Shims.scala | 4 +++-
2 files changed, 16 insertions(+), 2 deletions(-)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
index 11eaa3289..75b60addf 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
@@ -16,6 +16,7 @@
*/
package org.apache.gluten.execution
+import org.apache.spark.sql.execution.ProjectExec
import org.apache.spark.sql.types._
import java.sql.Timestamp
@@ -1145,7 +1146,18 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateTest {
runQueryAndCompare(
"SELECT a, window.start, window.end, count(*) as cnt FROM" +
" string_timestamp GROUP by a, window(b, '5 minutes') ORDER BY a,
start;") {
- checkGlutenOperatorMatch[ProjectExecTransformer]
+ df =>
+ val executedPlan = getExecutedPlan(df)
+ assert(
+ executedPlan.exists(plan =>
plan.isInstanceOf[ProjectExecTransformer]),
+ s"Expect ProjectExecTransformer exists " +
+ s"in executedPlan:\n ${executedPlan.last}"
+ )
+ assert(
+ !executedPlan.exists(plan => plan.isInstanceOf[ProjectExec]),
+ s"Expect ProjectExec doesn't exist " +
+ s"in executedPlan:\n ${executedPlan.last}"
+ )
}
}
}
diff --git
a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
index 95571f166..f6feae01a 100644
---
a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
+++
b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
@@ -76,7 +76,9 @@ class Spark35Shims extends SparkShims {
Sig[SplitPart](ExpressionNames.SPLIT_PART),
Sig[Sec](ExpressionNames.SEC),
Sig[Csc](ExpressionNames.CSC),
- Sig[Empty2Null](ExpressionNames.EMPTY2NULL))
+ Sig[KnownNullable](ExpressionNames.KNOWN_NULLABLE),
+ Sig[Empty2Null](ExpressionNames.EMPTY2NULL)
+ )
}
override def aggregateExpressionMappings: Seq[Sig] = {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
